From dd1615b4a7d562f9be3ab6460287ee8d0a217dfe Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 26 Aug 2021 16:03:17 -0400 Subject: [PATCH 01/30] Fixes for imgutil --- imgutil/imgutil | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/imgutil/imgutil b/imgutil/imgutil index 62905c9b..9829450c 100644 --- a/imgutil/imgutil +++ b/imgutil/imgutil @@ -358,7 +358,9 @@ def create_yumconf(sourcedir, addrepos): yumconf.write('enabled=1\ngpgcheck=0\n\n') addrepoidx = 1 for repo in addrepos.split(','): - yumconf.write('[addrepo-{0}]\n', addrepoidx) + if not repo: + continue + yumconf.write('[addrepo-{0}]\n'.format(addrepoidx)) yumconf.write('name=Add-on repository {0}\n'.format(addrepoidx)) yumconf.write('baseurl={0}\n'.format(repo)) yumconf.write('enabled=1\ngpgcheck\0\n\n') @@ -708,13 +710,6 @@ def check_root(installroot): # Ensure that the target is an adequate filesystem to # be root mkdirp(installroot) - testpath = os.path.join(installroot, '.testcap') - with open(testpath, 'w') as tp: - tp.write('') - try: - subprocess.check_call(['setcap', 'cap_net_raw+p', testpath]) - finally: - os.remove(testpath) def fingerprint_source_suse(files, sourcepath, args): From e35ce77bc7fc052f148f9313e541bdb414a61ab5 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 27 Aug 2021 08:46:23 -0400 Subject: [PATCH 02/30] Fix SUSE imgutil --- imgutil/imgutil | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/imgutil/imgutil b/imgutil/imgutil index 9829450c..ae6773e9 100644 --- a/imgutil/imgutil +++ b/imgutil/imgutil @@ -727,7 +727,7 @@ def fingerprint_source_suse(files, sourcepath, args): if key == 'name': osname, ver, arch = val.split('-') if issuse: - return SuseHandler(osname, ver, arch) + return SuseHandler(osname, ver, arch, args) for filen in files: if '-release-8' in filen: @@ -800,7 +800,7 @@ def fingerprint_host_suse(args, hostpath='/'): if inf.name.startswith(b'SLE_'): osname = 'sle' if osname: - return SuseHandler(osname, inf.version, os.uname().machine) + return SuseHandler(osname, inf.version, os.uname().machine, args) def fingerprint_host(args, hostpath='/'): oshandler = None From ce98ff6b3de32c5c681b82fea87c5b80c09b9e13 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 27 Aug 2021 09:03:35 -0400 Subject: [PATCH 03/30] Fix suse imgutil build function --- imgutil/imgutil | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/imgutil/imgutil b/imgutil/imgutil index ae6773e9..e77f0b43 100644 --- a/imgutil/imgutil +++ b/imgutil/imgutil @@ -414,8 +414,10 @@ class OsHandler(object): class SuseHandler(OsHandler): def __init__(self, name, version, arch, args): - if not version.startswith(b'15.'): - raise Exception('Unsupported Suse version {}'.format(version.decode('utf8'))) + if not isinstance(version, str): + version = version.decade('utf8') + if not version.startswith('15.'): + raise Exception('Unsupported Suse version {}'.format(version)) self.oscategory = 'suse15' super().__init__(name, version, arch, args) self.zyppargs = [] @@ -443,11 +445,18 @@ class SuseHandler(OsHandler): mkdirp(targzypp) shutil.copytree( '/etc/zypp/repos.d/', os.path.join(targzypp, 'repos.d')) + idx = 1 for source in self.sources: - subprocess.check_call(['zypper', '-R', self.targpath, 'ar', source]) + if not source: + continue + subprocess.check_call(['zypper', '-R', self.targpath, 'ar', source, 'source-{}'.format(idx)]) + idx += 1 for source in self.addrepos.split(','): + if not source: + continue source = 'file://' + source - subprocess.check_call(['zypper', '-R', self.targpath, 'ar', source]) + subprocess.check_call(['zypper', '-R', self.targpath, 'ar', source, 'source-{}'.format(idx)]) + idx += 1 mydir = get_mydir(self.oscategory) mkdirp(os.path.join(self.targpath, 'usr/lib/dracut/modules.d')) mkdirp(os.path.join(self.targpath, 'etc/dracut.conf.d')) From c48f3a8b59af64a71478b8e2209cb399382c1373 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 27 Aug 2021 16:13:14 -0400 Subject: [PATCH 04/30] Sample script to navigate first password change --- misc/setinitalpwd.py | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 misc/setinitalpwd.py diff --git a/misc/setinitalpwd.py b/misc/setinitalpwd.py new file mode 100644 index 00000000..a640fdc7 --- /dev/null +++ b/misc/setinitalpwd.py @@ -0,0 +1,40 @@ +#!/usr/bin/python2 +import pyghmi.util.webclient as webclient +import json +import os +import sys + +tmppassword = 'to3BdS91ABrd' +missingargs = False +if 'XCCUSER' not in os.environ: + print('Must set XCCUSER environment variable') + missingargs = True +if 'XCCPASS' not in os.environ: + print('Must set XCCPASS environment variable') + missingargs = True +if missingargs: + sys.exit(1) + +w = webclient.SecureHTTPConnection(sys.argv[1], 443, verifycallback=lambda x: True) +w.connect() +adata = json.dumps({'username': 'USERID', 'password': 'PASSW0RD'}) +headers = {'Connection': 'keep-alive', 'Content-Type': 'application/json'} +w.request('POST', '/api/login', adata, headers) +rsp = w.getresponse() +if rsp.status == 200: + rspdata = json.loads(rsp.read()) + w.set_header('Content-Type', 'application/json') + w.set_header('Authorization', 'Bearer ' + rspdata['access_token']) + if '_csrf_token' in w.cookies: + w.set_header('X-XSRF-TOKEN', w.cookies['_csrf_token']) + if rspdata.get('pwchg_required', False): + print(repr(w.grab_json_response('/api/function', {'USER_UserPassChange': os.environ['XCCPASS']}))) + print(repr(w.grab_json_response('/api/dataset', { + 'USER_GlobalPassExpWarningPeriod': '0', + 'USER_GlobalPassExpPeriod': '0', + 'USER_GlobalMinPassReuseCycle': '0', + 'USER_GlobalMinPassReuseCycle': '0', + 'USER_GlobalMinPassChgInt': '0', + }))) + #print(repr(w.grab_json_response('/api/function', {'USER_UserPassChange': '1,' + os.environ['XCCPASS']}))) + From d69de620770de69388c23875fceeb554da02e825 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 27 Aug 2021 16:37:26 -0400 Subject: [PATCH 05/30] Amend setinitalpwd --- misc/setinitalpwd.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/misc/setinitalpwd.py b/misc/setinitalpwd.py index a640fdc7..32a97fd5 100644 --- a/misc/setinitalpwd.py +++ b/misc/setinitalpwd.py @@ -4,11 +4,7 @@ import json import os import sys -tmppassword = 'to3BdS91ABrd' missingargs = False -if 'XCCUSER' not in os.environ: - print('Must set XCCUSER environment variable') - missingargs = True if 'XCCPASS' not in os.environ: print('Must set XCCPASS environment variable') missingargs = True From 3d4462d7888f98d2ddebd521319dd31f53557d74 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 27 Aug 2021 18:29:38 -0400 Subject: [PATCH 06/30] Fix support for newer XCC firmware Newer XCC firmware requires more parameters for usermodify --- confluent_server/confluent/discovery/handlers/xcc.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/confluent_server/confluent/discovery/handlers/xcc.py b/confluent_server/confluent/discovery/handlers/xcc.py index 4f746a9a..f99df402 100644 --- a/confluent_server/confluent/discovery/handlers/xcc.py +++ b/confluent_server/confluent/discovery/handlers/xcc.py @@ -340,6 +340,10 @@ class NodeHandler(immhandler.NodeHandler): rsp, status = wc.grab_json_response_with_status( '/api/function', {'USER_UserModify': '{0},{1},,1,Administrator,0,0,0,0,,8,'.format(uid, username)}) + elif status == 200 and rsp.get('return', 0) == 13: + rsp, status = wc.grab_json_response_with_status( + '/api/function', + {'USER_UserModify': '{0},{1},,1,4,0,0,0,0,,8,,,'.format(uid, username)}) self.tmppasswd = None self._currcreds = (username, passwd) From aaea5bcaeb505f2d3292e05838fd906e26534185 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 30 Aug 2021 09:01:42 -0400 Subject: [PATCH 07/30] Make sure the netlink socket closes on the way out --- confluent_server/confluent/neighutil.py | 61 +++++++++++++------------ 1 file changed, 31 insertions(+), 30 deletions(-) diff --git a/confluent_server/confluent/neighutil.py b/confluent_server/confluent/neighutil.py index b2152443..cccbae32 100644 --- a/confluent_server/confluent/neighutil.py +++ b/confluent_server/confluent/neighutil.py @@ -15,8 +15,6 @@ # limitations under the License. # A consolidated manage of neighbor table information management. -# Ultimately, this should use AF_NETLINK, but in the interest of time, -# use ip neigh for the moment import confluent.util as util import os @@ -42,34 +40,37 @@ def _update_neigh(): ndmsg= b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' s.sendall(nlhdr + ndmsg) neightable = {} - while True: - pdata = s.recv(65536) - v = memoryview(pdata) - if struct.unpack('H', v[4:6])[0] == 3: - break - while len(v): - length, typ = struct.unpack('IH', v[:6]) - if typ == 28: - hlen = struct.calcsize('BIHBB') - _, idx, state, flags, typ = struct.unpack('BIHBB', v[16:16+hlen]) - if typ == 1: # only handle unicast entries - curraddr = None - currip = None - rta = v[16+hlen:length] - while len(rta): - rtalen, rtatyp = struct.unpack('HH', rta[:4]) - if rtatyp == 2: # hwaddr - curraddr = rta[4:rtalen].tobytes() - if len(curraddr) == 20: - curraddr = curraddr[12:] - elif rtatyp == 1: # ip address - currip = rta[4:rtalen].tobytes() - rta = rta[rtalen:] - if not rtalen: - break - if curraddr and currip: - neightable[currip] = curraddr - v = v[length:] + try: + while True: + pdata = s.recv(65536) + v = memoryview(pdata) + if struct.unpack('H', v[4:6])[0] == 3: + break + while len(v): + length, typ = struct.unpack('IH', v[:6]) + if typ == 28: + hlen = struct.calcsize('BIHBB') + _, idx, state, flags, typ = struct.unpack('BIHBB', v[16:16+hlen]) + if typ == 1: # only handle unicast entries + curraddr = None + currip = None + rta = v[16+hlen:length] + while len(rta): + rtalen, rtatyp = struct.unpack('HH', rta[:4]) + if rtatyp == 2: # hwaddr + curraddr = rta[4:rtalen].tobytes() + if len(curraddr) == 20: + curraddr = curraddr[12:] + elif rtatyp == 1: # ip address + currip = rta[4:rtalen].tobytes() + rta = rta[rtalen:] + if not rtalen: + break + if curraddr and currip: + neightable[currip] = curraddr + v = v[length:] + finally: + s.close() def get_hwaddr(ipaddr): From 6c32630bd944fc70e5079ad2d6bfeefc44b4ba77 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 30 Aug 2021 11:24:57 -0400 Subject: [PATCH 08/30] Fix multi-server reply processing The stateless code could get confused by multiple offers, fix the mistake. --- .../usr/lib/dracut/hooks/cmdline/10-confluentdiskless.sh | 2 +- .../initramfs/lib/dracut/hooks/cmdline/10-confluentdiskless.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/confluent_osdeploy/el8-diskless/initramfs/usr/lib/dracut/hooks/cmdline/10-confluentdiskless.sh b/confluent_osdeploy/el8-diskless/initramfs/usr/lib/dracut/hooks/cmdline/10-confluentdiskless.sh index 1b2ef09e..cb34f4f5 100644 --- a/confluent_osdeploy/el8-diskless/initramfs/usr/lib/dracut/hooks/cmdline/10-confluentdiskless.sh +++ b/confluent_osdeploy/el8-diskless/initramfs/usr/lib/dracut/hooks/cmdline/10-confluentdiskless.sh @@ -101,7 +101,7 @@ done cd / nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}') hostname $nodename -confluent_mgr=$(grep '^EXTMGRINFO:.*1$' /etc/confluent/confluent.info | awk -F': ' '{print $2}' | awk -F'|' '{print $1}') +confluent_mgr=$(grep '^EXTMGRINFO:.*1$' /etc/confluent/confluent.info | head -n 1 | awk -F': ' '{print $2}' | awk -F'|' '{print $1}') if [ -z "$confluent_mgr" ]; then confluent_mgr=$(grep ^MANAGER: /etc/confluent/confluent.info|head -n 1 | awk '{print $2}') fi diff --git a/confluent_osdeploy/suse15-diskless/initramfs/lib/dracut/hooks/cmdline/10-confluentdiskless.sh b/confluent_osdeploy/suse15-diskless/initramfs/lib/dracut/hooks/cmdline/10-confluentdiskless.sh index 7271b6e9..6fcb44c3 100644 --- a/confluent_osdeploy/suse15-diskless/initramfs/lib/dracut/hooks/cmdline/10-confluentdiskless.sh +++ b/confluent_osdeploy/suse15-diskless/initramfs/lib/dracut/hooks/cmdline/10-confluentdiskless.sh @@ -102,7 +102,7 @@ done cd / nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}') hostname $nodename -confluent_mgr=$(grep '^EXTMGRINFO:.*1$' /etc/confluent/confluent.info | awk -F': ' '{print $2}' | awk -F'|' '{print $1}') +confluent_mgr=$(grep '^EXTMGRINFO:.*1$' /etc/confluent/confluent.info | head -n 1|awk -F': ' '{print $2}' | awk -F'|' '{print $1}') if [ -z "$confluent_mgr" ]; then confluent_mgr=$(grep ^MANAGER: /etc/confluent/confluent.info|head -n 1 | awk '{print $2}') fi From 9afa33d0cc31c15a6785ecf47758180b976699b0 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 1 Sep 2021 10:23:43 -0400 Subject: [PATCH 09/30] Assisted redfish bmc setup This is a quick example script of using confluent switch scanning in conjunction with checking dhcp leases to handle some generic redfish bmc. --- misc/cfg-dhcp-redfish-bmcs-by-switch.py | 120 ++++++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100755 misc/cfg-dhcp-redfish-bmcs-by-switch.py diff --git a/misc/cfg-dhcp-redfish-bmcs-by-switch.py b/misc/cfg-dhcp-redfish-bmcs-by-switch.py new file mode 100755 index 00000000..797edc5d --- /dev/null +++ b/misc/cfg-dhcp-redfish-bmcs-by-switch.py @@ -0,0 +1,120 @@ +#!/usr/bin/python3 +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# Copyright 2017-2021 Lenovo +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +sys.path.append('/opt/confluent/lib/python') +import confluent.client as cli +import eventlet.greenpool +import gzip +import io +import json +import os +import struct +import subprocess +import time + +webclient = eventlet.import_patched('pyghmi.util.webclient') + + +bmcsbyuuid = {} +def checkfish(addr, mac): + wc = webclient.SecureHTTPConnection(addr, 443, verifycallback=lambda x: True) + wc.connect() + wc.request('GET', '/redfish/v1') + rsp = wc.getresponse() + body = rsp.read() + if body[:2] == b'\x1f\x8b': + body = gzip.GzipFile(fileobj=io.BytesIO(body)).read() + try: + body = json.loads(body) + except json.decoder.JSONDecodeError: + return + uuid = body.get('UUID', None) + if not uuid: + return + #This part is needed if a bmc sticks 'wire format' uuid in the json body + #Should be skipped for bmcs that present it sanely + uuidparts = uuid.split('-') + uuidparts[0] = '{:08x}'.format(struct.unpack('!I', struct.pack(' Date: Wed, 1 Sep 2021 10:28:49 -0400 Subject: [PATCH 10/30] Add some commentary --- misc/cfg-dhcp-redfish-bmcs-by-switch.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/misc/cfg-dhcp-redfish-bmcs-by-switch.py b/misc/cfg-dhcp-redfish-bmcs-by-switch.py index 797edc5d..fe304ae3 100755 --- a/misc/cfg-dhcp-redfish-bmcs-by-switch.py +++ b/misc/cfg-dhcp-redfish-bmcs-by-switch.py @@ -14,6 +14,22 @@ # See the License for the specific language governing permissions and # limitations under the License. +# This script demonstrates a strategy for redfish bmcs that +# dhcp to leverage the confluent switch scanning to help +# bootstrap such devices. Be aware of the uuid reformatting +# code, and determine if it is relevant for the target system +# in question. The normal thing would be to leave UUID as-is, +# but some implementations mangle it in a misunderstanding +# of 'wire format' UUID. Also, here xCAT is used as the +# 'dhcp helper', so that may need to be replaced with dnsmasq +# or direct isc dhcp code. + +# Unfortunately, this is particular about the dhcp server, +# the user must know if the bmc in question mangles the uuid +# or not, and other such limitation make this difficult to blindly +# recommend, but hopefully can be useful reference material + + import sys sys.path.append('/opt/confluent/lib/python') import confluent.client as cli From 96f50519d8a1fd55c53572cfbabb4e677df4cafc Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 1 Sep 2021 10:38:49 -0400 Subject: [PATCH 11/30] Fix typo in imgutil --- imgutil/imgutil | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/imgutil/imgutil b/imgutil/imgutil index e77f0b43..b9d63348 100644 --- a/imgutil/imgutil +++ b/imgutil/imgutil @@ -415,7 +415,7 @@ class OsHandler(object): class SuseHandler(OsHandler): def __init__(self, name, version, arch, args): if not isinstance(version, str): - version = version.decade('utf8') + version = version.decode('utf8') if not version.startswith('15.'): raise Exception('Unsupported Suse version {}'.format(version)) self.oscategory = 'suse15' @@ -831,9 +831,13 @@ def build_root(args): else: oshandler = fingerprint_host(args) if oshandler is None: - sys.stderr.write( - 'Unable to recognize source directory {0}\n'.format( - args.source)) + if args.source: + sys.stderr.write( + 'Unable to recognize source directory {0}\n'.format( + args.source)) + else: + sys.stderr.write( + 'Unable to recognize build system os\n') sys.exit(1) oshandler.set_target(args.scratchdir) oshandler.add_pkglists() From 27a1649d94f18048ab14f1de8748a3efe7df2c71 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 1 Sep 2021 10:44:25 -0400 Subject: [PATCH 12/30] Add more in-context stateless documentation Specify what default behavior is and reason to choose one over the other. --- imgutil/imgutil | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imgutil/imgutil b/imgutil/imgutil index b9d63348..6d65a149 100644 --- a/imgutil/imgutil +++ b/imgutil/imgutil @@ -971,7 +971,7 @@ def pack_image(args): 'PRETTY_NAME=', '').replace('"', '') label = '{0} ({1})'.format(prettyname, 'Diskless Boot') with open(os.path.join(outdir, 'profile.yaml'), 'w') as profiley: - profiley.write('label: {0}\nkernelargs: quiet # confluent_imagemethod=untethered|tethered\n'.format(label)) + profiley.write('label: {0}\nkernelargs: quiet # confluent_imagemethod=untethered|tethered # tethered is default when unspecified to save on memory, untethered will use more ram, but will not have any ongoing runtime root fs dependency on the http servers.\n'.format(label)) oscat = oshandler.oscategory confdir = '/opt/confluent/lib/osdeploy/{}-diskless'.format(oscat) os.symlink('{}/initramfs/addons.cpio'.format(confdir), From a7d894007f357a15f116c71a74759fd2bcf5fc6f Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 1 Sep 2021 11:00:53 -0400 Subject: [PATCH 13/30] Treat netgroup entries differently --- .../el8/profiles/default/scripts/syncfileclient | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/confluent_osdeploy/el8/profiles/default/scripts/syncfileclient b/confluent_osdeploy/el8/profiles/default/scripts/syncfileclient index dd22d6b3..e6884d50 100644 --- a/confluent_osdeploy/el8/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/el8/profiles/default/scripts/syncfileclient @@ -134,7 +134,9 @@ class CredMerger: else: self.discardnames[name] = 1 else: - if uid >= minid and uid <= maxid: + if name[0] in ('+', '#', '@'): + self.sourcedata.append(line) + elif uid >= minid and uid <= maxid: self.sourcedata.append(line) def read_shadow(self, source): @@ -167,6 +169,8 @@ class CredMerger: shadout.write(line + '\n') for line in self.sourcedata: name, _ = line.split(':', 1) + if name[0] in ('+', '#', '@'): + continue if name in self.shadowednames: continue shadout.write(name + ':!:::::::\n') @@ -216,4 +220,4 @@ def synchronize(): if __name__ == '__main__': - synchronize() \ No newline at end of file + synchronize() From 6264dc1ed1eee988196c2a39b48391069aaf6f95 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 1 Sep 2021 16:17:34 -0400 Subject: [PATCH 14/30] Add host pubkeys to build root --- imgutil/imgutil | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/imgutil/imgutil b/imgutil/imgutil index 6d65a149..a3c7d241 100644 --- a/imgutil/imgutil +++ b/imgutil/imgutil @@ -440,7 +440,10 @@ class SuseHandler(OsHandler): def prep_root(self, args): mkdirp(self.targpath) - if not self.sources: + if self.sources: + addkeycmd = ['rpm', '--root', self.targpath, '--import'] + glob.glob('/usr/lib/rpm/gnupg/keys/*.asc') + subprocess.check_call(addkeycmd) + else: targzypp = os.path.join(self.targpath, 'etc/zypp') mkdirp(targzypp) shutil.copytree( From f65056a0472276295b92916199d330a996b3e880 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 2 Sep 2021 08:05:11 -0400 Subject: [PATCH 15/30] Fix Suse image fingeprinting --- imgutil/imgutil | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/imgutil/imgutil b/imgutil/imgutil index a3c7d241..4025c18f 100644 --- a/imgutil/imgutil +++ b/imgutil/imgutil @@ -799,20 +799,27 @@ def fingerprint_host_el(args, hostpath='/'): def fingerprint_host_suse(args, hostpath='/'): - try: - import rpm - except ImportError: - return None - ts = rpm.TransactionSet(hostpath) - rpms = ts.dbMatch('provides', 'distribution-release') + vers = None osname = None - for inf in rpms: - if b'openSUSE' in inf.name and b'Leap' in inf.summary: - osname = 'opensuse_leap' - if inf.name.startswith(b'SLE_'): - osname = 'sle' + try: + with open(os.path.join(hostpath, 'etc/os-release')) as relfile: + relinfo = relfile.read().split('\n') + for inf in relinfo: + if '=' in inf: + key, val = inf.split('=', 1) + if key == 'ID': + if val.lower().replace('"', '') == 'opensuse-leap': + osname = 'opensuse_leap' + elif val.lower().replace( + '"', '') in ('sle_hpc', 'sles'): + osname = 'sle' + elif key == 'VERSION_ID': + vers = val.replace('"', '') + except IOError: + pass if osname: - return SuseHandler(osname, inf.version, os.uname().machine, args) + return SuseHandler(osname, vers, os.uname().machine, args) + def fingerprint_host(args, hostpath='/'): oshandler = None From 237670e9b1adf00c1a3d21ce7e5ca1d3d1f9ffc3 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 2 Sep 2021 08:39:45 -0400 Subject: [PATCH 16/30] Rework gpg key import for suse build Only pull in host keys when not using a source. When using source(s), pull in all normal gpg keys from local filesystem sources. --- imgutil/imgutil | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/imgutil/imgutil b/imgutil/imgutil index 4025c18f..40f04f51 100644 --- a/imgutil/imgutil +++ b/imgutil/imgutil @@ -439,11 +439,10 @@ class SuseHandler(OsHandler): self.sources.append('file://' + os.path.join(sourcepath, 'Product-HPC')) def prep_root(self, args): + gpgkeys = [] mkdirp(self.targpath) - if self.sources: - addkeycmd = ['rpm', '--root', self.targpath, '--import'] + glob.glob('/usr/lib/rpm/gnupg/keys/*.asc') - subprocess.check_call(addkeycmd) - else: + if not self.sources: + gpgkeys = glob.glob('/usr/lib/rpm/gnupg/keys/*.asc') targzypp = os.path.join(self.targpath, 'etc/zypp') mkdirp(targzypp) shutil.copytree( @@ -452,8 +451,14 @@ class SuseHandler(OsHandler): for source in self.sources: if not source: continue + if source.startswith('file://'): + gpgpath = source.replace('file://', '') + gpgkeys.extend(glob.glob(os.path.join(gpgpath, '*/gpg-pubkey*.asc'))) subprocess.check_call(['zypper', '-R', self.targpath, 'ar', source, 'source-{}'.format(idx)]) idx += 1 + if gpgkeys: + addkeycmd = ['rpm', '--root', self.targpath, '--import'] + gpgkeys + subprocess.check_call(addkeycmd) for source in self.addrepos.split(','): if not source: continue From ac45d6e5787754f6ee383d8c5a09b5c0a3b2c24d Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 2 Sep 2021 09:06:57 -0400 Subject: [PATCH 17/30] Support non-interactive request to exec command Make outside scripting easier by accepting an optional command to run in exec environment. --- imgutil/imgutil | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/imgutil/imgutil b/imgutil/imgutil index 40f04f51..9b7bc9a4 100644 --- a/imgutil/imgutil +++ b/imgutil/imgutil @@ -625,6 +625,7 @@ def main(): 'cause it to be mounted in image as /run/external/, -v /:/run/root ' 'will override the target to be /run/root', action='append') execp.add_argument('scratchdir', help='Directory of an unpacked diskless root') + execp.add_argument('cmd', nargs='*', help='Optional command to run (defaults to a shell)') unpackp = sps.add_parser('unpack', help='Unpack a diskless image to a scratch directory') unpackp.add_argument('profilename', help='The diskless OS profile to unpack') unpackp.add_argument('scratchdir', help='Directory to extract diskless root to') @@ -677,7 +678,12 @@ def exec_root_backend(args): os.chroot(installroot) os.chdir('/') os.environ['PS1'] = '[\x1b[1m\x1b[4mIMGUTIL EXEC {0}\x1b[0m \W]$ '.format(imgname) - os.execv('/bin/bash', ['/bin/bash', '--login', '--noprofile']) + if args.cmd: + if not args.cmd[0].startswith('/'): + args.cmd[0] = shutil.which(args.cmd[0]) + os.execv(args.cmd[0], args.cmd) + else: + os.execv('/bin/bash', ['/bin/bash', '--login', '--noprofile']) def _mount(src, dst, fstype=0, flags=0, options=0, mode=None): From 0d2ecd460d8891e86ce956543d94602e2db389d2 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 2 Sep 2021 10:17:11 -0400 Subject: [PATCH 18/30] Merge changes from EL8 imageboot --- .../profiles/default/scripts/imageboot.sh | 41 +++++++++++++++++-- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/confluent_osdeploy/suse15-diskless/profiles/default/scripts/imageboot.sh b/confluent_osdeploy/suse15-diskless/profiles/default/scripts/imageboot.sh index aa9212a3..244749aa 100644 --- a/confluent_osdeploy/suse15-diskless/profiles/default/scripts/imageboot.sh +++ b/confluent_osdeploy/suse15-diskless/profiles/default/scripts/imageboot.sh @@ -7,7 +7,29 @@ else confluent_urls="$confluent_urls https://$confluent_mgr/confluent-public/os/$confluent_profile/rootimg.sfs" /opt/confluent/bin/urlmount $confluent_urls /mnt/remoteimg fi -mount -o loop,ro /mnt/remoteimg/*.sfs /mnt/remote +/opt/confluent/bin/confluent_imginfo /mnt/remoteimg/rootimg.sfs > /tmp/rootimg.info +loopdev=$(losetup -f) +export mountsrc=$loopdev +losetup -r $loopdev /mnt/remoteimg/rootimg.sfs +if grep '^Format: confluent_crypted' /tmp/rootimg.info > /dev/null; then + curl -sf -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $(cat /etc/confluent/confluent.apikey)" https://$confluent_mgr/confluent-api/self/profileprivate/pending/rootimg.key > /tmp/rootimg.key + cipher=$(head -n 1 /tmp/rootimg.key) + key=$(tail -n 1 /tmp/rootimg.key) + len=$(wc -c /mnt/remoteimg/rootimg.sfs | awk '{print $1}') + len=$(((len-4096)/512)) + dmsetup create cryptimg --table "0 $len crypt $cipher $key 0 $loopdev 8" + /opt/confluent/bin/confluent_imginfo /dev/mapper/cryptimg > /tmp/rootimg.info + mountsrc=/dev/mapper/cryptimg +fi +if grep '^Format: squashfs' /tmp/rootimg.info > /dev/null; then + mount -o ro $mountsrc /mnt/remote +elif grep '^Format: confluent_multisquash' /tmp/rootimg.info; then + tail -n +3 /tmp/rootimg.info | awk '{gsub("/", "_"); print "echo 0 " $4 " linear '$mountsrc' " $3 " | dmsetup create mproot" $7}' > /tmp/setupmount.sh + . /tmp/setupmount.sh + cat /tmp/setupmount.sh |awk '{printf "mount /dev/mapper/"$NF" "; sub("mproot", ""); gsub("_", "/"); print "/mnt/remote"$NF}' > /tmp/mountparts.sh + . /tmp/mountparts.sh +fi + #mount -t tmpfs overlay /mnt/overlay modprobe zram memtot=$(grep ^MemTotal: /proc/meminfo|awk '{print $2}') @@ -15,8 +37,15 @@ memtot=$((memtot/2))$(grep ^MemTotal: /proc/meminfo | awk '{print $3'}) echo $memtot > /sys/block/zram0/disksize mkfs.xfs /dev/zram0 > /dev/null mount -o discard /dev/zram0 /mnt/overlay -mkdir -p /mnt/overlay/upper /mnt/overlay/work -mount -t overlay -o upperdir=/mnt/overlay/upper,workdir=/mnt/overlay/work,lowerdir=/mnt/remote disklessroot /sysroot +if [ ! -f /tmp/mountparts.sh ]; then + mkdir -p /mnt/overlay/upper /mnt/overlay/work + mount -t overlay -o upperdir=/mnt/overlay/upper,workdir=/mnt/overlay/work,lowerdir=/mnt/remote disklessroot /sysroot +else + for srcmount in $(cat /tmp/mountparts.sh | awk '{print $3}'); do + mkdir -p /mnt/overlay${srcmount}/upper /mnt/overlay${srcmount}/work + mount -t overlay -o upperdir=/mnt/overlay${srcmount}/upper,workdir=/mnt/overlay${srcmount}/work,lowerdir=${srcmount} disklesspart /sysroot${srcmount#/mnt/remote} + done +fi mkdir -p /sysroot/etc/ssh mkdir -p /sysroot/etc/confluent mkdir -p /sysroot/root/.ssh @@ -77,6 +106,7 @@ curl -sf https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/o mkdir -p /sysroot/opt/confluent/bin curl -sf https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/onboot.sh > /sysroot/opt/confluent/bin/onboot.sh chmod +x /sysroot/opt/confluent/bin/onboot.sh +cp /opt/confluent/bin/apiclient /sysroot/opt/confluent/bin ln -s /etc/systemd/system/onboot.service /sysroot/etc/systemd/system/multi-user.target.wants/onboot.service cp /etc/confluent/functions /sysroot/etc/confluent/functions @@ -101,4 +131,9 @@ dnsdomain=$(grep ^dnsdomain: /etc/confluent/confluent.deploycfg) dnsdomain=${dnsdomain#dnsdomain: } sed -i 's/^NETCONFIG_DNS_STATIC_SEARCHLIST="/NETCONFIG_DNS_STATIC_SEARCHLIST="'$dnsdomain/ /sysroot/etc/sysconfig/network/config cp /run/confluent/ifroute-* /run/confluent/ifcfg-* /sysroot/etc/sysconfig/network +if grep installtodisk /proc/cmdline > /dev/null; then + . /etc/confluent/functions + run_remote installimage + exec reboot -f +fi exec /opt/confluent/bin/start_root From bcc278e4d8e014a2b30589c8c8fcb8d571d23375 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 2 Sep 2021 10:30:30 -0400 Subject: [PATCH 19/30] Add losetup for new suse support --- imgutil/suse15/dracut/install | 1 + 1 file changed, 1 insertion(+) diff --git a/imgutil/suse15/dracut/install b/imgutil/suse15/dracut/install index 8c8f905e..6bf70376 100644 --- a/imgutil/suse15/dracut/install +++ b/imgutil/suse15/dracut/install @@ -20,6 +20,7 @@ dracut_install /lib64/libfuse.so.2 /lib64/libfuse.so.2.9.7 dracut_install chown chroot dd expr kill parted rsync sort blockdev findfs insmod lvm dracut_install /usr/lib/udev/rules.d/10-dm.rules /usr/sbin/dmsetup /usr/lib/udev/rules.d/95-dm-notify.rules dracut_install /usr/lib/systemd/network/99-default.link +dracut_install losetup # multipart support #this would be nfs with lock, but not needed, go nolock #dracut_install mount.nfs rpcbind rpc.statd /etc/netconfig sm-notify From bf017b55e1f1091fbc7d3f31ff9c25687fbbc2b8 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 2 Sep 2021 11:22:45 -0400 Subject: [PATCH 20/30] Have unpack more intuitively target the specific directory This makes it symmetric witht pack/unpack. --- imgutil/imgutil | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/imgutil/imgutil b/imgutil/imgutil index 9b7bc9a4..8c458ad3 100644 --- a/imgutil/imgutil +++ b/imgutil/imgutil @@ -875,7 +875,7 @@ def prep_decrypt(indir): currtabs = currtabs.decode('utf8').split('\n') usednames = set([]) for tab in currtabs: - if not tab: + if ':' not in tab: continue tabname, _ = tab.split(':', 1) usednames.add(tabname) @@ -924,9 +924,13 @@ def unpack_image(args): if hdr[:4] in (b'sqsh', b'hsqs'): break raise Exception('Unrecognized image format') - mkdirp(scratchdir) - os.chdir(scratchdir) - subprocess.check_call(['unsquashfs', '-d', 'rootfs', indir]) + while scratchdir.endswith('/'): + scratchdir = scratchdir[:-1] + parentdir = os.path.dirname(scratchdir) + targdir = os.path.basename(scratchdir) + mkdirp(parentdir) + os.chdir(parentdir) + subprocess.check_call(['unsquashfs', '-d', targdir, indir]) finally: if cleandmtable: mounted = True From fbef6442d5057f11d6be6f62dba4b1e368abd686 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 2 Sep 2021 11:29:40 -0400 Subject: [PATCH 21/30] Add a shorthand '-' to say 'mount to same path as outside' --- imgutil/imgutil | 2 ++ 1 file changed, 2 insertions(+) diff --git a/imgutil/imgutil b/imgutil/imgutil index 8c458ad3..679f1290 100644 --- a/imgutil/imgutil +++ b/imgutil/imgutil @@ -717,6 +717,8 @@ def _mount_constrained_fs(args, installroot): for v in args.volume: if ':' in v: src, dst = v.split(':') + if dst == '-': + dst = src while dst and dst[0] == '/': dst = dst[1:] dst = os.path.join(installroot, dst) From adcc2f0aa9b06d5ee95750499f9e1e400da9b522 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 2 Sep 2021 11:30:51 -0400 Subject: [PATCH 22/30] Add some usage to the volume flag for now '-' --- imgutil/imgutil | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imgutil/imgutil b/imgutil/imgutil index 679f1290..f9f20cf1 100644 --- a/imgutil/imgutil +++ b/imgutil/imgutil @@ -617,7 +617,7 @@ def main(): buildp.add_argument('-v', '--volume', help='Directory to make available in the build environment. -v / will ' 'cause it to be mounted in image as /run/external/, -v /:/run/root ' - 'will override the target to be /run/root', action='append') + 'will override the target to be /run/root, and something like /var/lib/repository:- will cause it to mount to the identical path inside the image', action='append') buildp.add_argument('scratchdir', help='Directory to build new diskless root in') execp = sps.add_parser('exec', help='Start specified scratch directory as container') execp.add_argument('-v', '--volume', From 19685b7a4e34abaa90a0f22316d16ef6c5e08cb0 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 2 Sep 2021 11:38:09 -0400 Subject: [PATCH 23/30] Accept relative path for unpack --- imgutil/imgutil | 1 + 1 file changed, 1 insertion(+) diff --git a/imgutil/imgutil b/imgutil/imgutil index f9f20cf1..241764e9 100644 --- a/imgutil/imgutil +++ b/imgutil/imgutil @@ -928,6 +928,7 @@ def unpack_image(args): raise Exception('Unrecognized image format') while scratchdir.endswith('/'): scratchdir = scratchdir[:-1] + scratchdir = os.path.abspath(scratchdir) parentdir = os.path.dirname(scratchdir) targdir = os.path.basename(scratchdir) mkdirp(parentdir) From 5d1d527e0919ffcb0a7c15480e331d8d0d79760e Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 2 Sep 2021 11:58:22 -0400 Subject: [PATCH 24/30] If reasonable, provide a 'distribution' link for diskless If the user uses '-s', then preserve that in distribution like a diskful one would. --- imgutil/imgutil | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/imgutil/imgutil b/imgutil/imgutil index 241764e9..67e12c39 100644 --- a/imgutil/imgutil +++ b/imgutil/imgutil @@ -705,6 +705,11 @@ def build_root_backend(optargs): installroot = args.scratchdir _mount_constrained_fs(args, installroot) oshandler.prep_root(optargs) + mkdirp(os.path.join(installroot, 'etc/')) + with open(os.path.join(installroot, 'etc/img.buildinfo'), 'w') as imginfo: + imginfo.write('BUILDDATE={}\n'.format(datetime.datetime.now().strftime('%Y-%m-%dT%H:%M'))) + if args.source: + imginfo.write('BUILDSRC={}\n'.format(args.source)) def _mount_constrained_fs(args, installroot): @@ -954,6 +959,16 @@ def pack_image(args): raise Exception('Full path not supported, supply only the profile name') privdir = os.path.join('/var/lib/confluent/private/os/', outdir) outdir = os.path.join('/var/lib/confluent/public/os/', outdir) + imginfofile = os.path.join(args.scratchdir, 'etc/img.buildinfo') + distpath = None + try: + with open(imginfofile) as imginfoin: + imginfo = imginfoin.read.split('\n') + for lineinfo in imginfo: + if lineinfo.startswith('BUILDSRC='): + distpath = lineinfo.replace('BUILDSRC=', '') + except IOError: + pass kerns = glob.glob(os.path.join(args.scratchdir, 'boot/vmlinuz-*')) kvermap = {} for kern in kerns: @@ -987,6 +1002,8 @@ def pack_image(args): os.remove(tmploc) with open(os.path.join(outdir, 'build-info'), 'w') as buildinfo: buildinfo.write('Packed from {} on {}\n'.format(args.scratchdir, datetime.datetime.now().strftime('%Y-%m-%dT%H:%M'))) + if distpath: + os.symlink(distpath, os.path.join(outdir, 'distribution')) oshandler = fingerprint_host(args, args.scratchdir) tryupdate = False if oshandler: From 63d9aea21870289e0b7686cac96b3651a44afdc3 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 2 Sep 2021 12:10:07 -0400 Subject: [PATCH 25/30] Support relative paths for build scratchdir and pkglist --- imgutil/imgutil | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/imgutil/imgutil b/imgutil/imgutil index 67e12c39..d8c31c23 100644 --- a/imgutil/imgutil +++ b/imgutil/imgutil @@ -387,6 +387,8 @@ class OsHandler(object): except AttributeError: pkglist = '' if pkglist: + if os.path.exists(os.path.abspath(pkglist)): + pkglist = os.path.abspath(pkglist) self.pkglist = pkglist if '/' not in self.pkglist: self.pkglist = os.path.join(get_mydir(self.oscategory), self.pkglist) @@ -848,6 +850,8 @@ def fingerprint_host(args, hostpath='/'): return oshandler def build_root(args): + if args.scratchdir: + args.scratchdir = os.path.abspath(args.scratchdir) check_root(args.scratchdir) yumargs = ['yum', '--installroot={0}'.format(args.scratchdir)] if args.source: From 558139d65461051c67ff7ea0ddf07a6dc1c9b31e Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 2 Sep 2021 12:11:48 -0400 Subject: [PATCH 26/30] Fix bug around pulling in img build info --- imgutil/imgutil | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imgutil/imgutil b/imgutil/imgutil index d8c31c23..e2fb93e0 100644 --- a/imgutil/imgutil +++ b/imgutil/imgutil @@ -967,7 +967,7 @@ def pack_image(args): distpath = None try: with open(imginfofile) as imginfoin: - imginfo = imginfoin.read.split('\n') + imginfo = imginfoin.read().split('\n') for lineinfo in imginfo: if lineinfo.startswith('BUILDSRC='): distpath = lineinfo.replace('BUILDSRC=', '') From 57a8149844b0cfc42f13a2589c2d57f9e068ea3c Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 3 Sep 2021 09:16:22 -0400 Subject: [PATCH 27/30] Fix adding repositories to rhel-style diskless --- imgutil/imgutil | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/imgutil/imgutil b/imgutil/imgutil index e2fb93e0..619a2307 100644 --- a/imgutil/imgutil +++ b/imgutil/imgutil @@ -363,7 +363,7 @@ def create_yumconf(sourcedir, addrepos): yumconf.write('[addrepo-{0}]\n'.format(addrepoidx)) yumconf.write('name=Add-on repository {0}\n'.format(addrepoidx)) yumconf.write('baseurl={0}\n'.format(repo)) - yumconf.write('enabled=1\ngpgcheck\0\n\n') + yumconf.write('enabled=1\ngpgcheck=0\n\n') addrepoidx += 1 return repodir @@ -496,7 +496,7 @@ class ElHandler(OsHandler): yumconfig = create_yumconf(sourcepath, self.addrepos) self.yumargs.extend( ['--setopt=reposdir={0}'.format(yumconfig), '--disablerepo=*', - '--enablerepo=genimage-*']) + '--enablerepo=genimage-*', '--enablerepo=addrepo-*']) self.sourcepath = sourcepath def set_target(self, targpath): From 5422f904f95ce2fbfd873fa114bd207f12529ea8 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 3 Sep 2021 12:11:10 -0400 Subject: [PATCH 28/30] Add a breadcrumb for scripts to detect exec context --- imgutil/imgutil | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/imgutil/imgutil b/imgutil/imgutil index 619a2307..46d19e4d 100644 --- a/imgutil/imgutil +++ b/imgutil/imgutil @@ -680,6 +680,7 @@ def exec_root_backend(args): os.chroot(installroot) os.chdir('/') os.environ['PS1'] = '[\x1b[1m\x1b[4mIMGUTIL EXEC {0}\x1b[0m \W]$ '.format(imgname) + os.environ['CONFLUENT_IMGUTIL_MODE'] = 'exec' if args.cmd: if not args.cmd[0].startswith('/'): args.cmd[0] = shutil.which(args.cmd[0]) @@ -708,7 +709,7 @@ def build_root_backend(optargs): _mount_constrained_fs(args, installroot) oshandler.prep_root(optargs) mkdirp(os.path.join(installroot, 'etc/')) - with open(os.path.join(installroot, 'etc/img.buildinfo'), 'w') as imginfo: + with open(os.path.join(installroot, 'etc/confluentimg.buildinfo'), 'w') as imginfo: imginfo.write('BUILDDATE={}\n'.format(datetime.datetime.now().strftime('%Y-%m-%dT%H:%M'))) if args.source: imginfo.write('BUILDSRC={}\n'.format(args.source)) @@ -963,7 +964,7 @@ def pack_image(args): raise Exception('Full path not supported, supply only the profile name') privdir = os.path.join('/var/lib/confluent/private/os/', outdir) outdir = os.path.join('/var/lib/confluent/public/os/', outdir) - imginfofile = os.path.join(args.scratchdir, 'etc/img.buildinfo') + imginfofile = os.path.join(args.scratchdir, 'etc/confluentimg.buildinfo') distpath = None try: with open(imginfofile) as imginfoin: From de5350ca6e04849b36f63f1555661eadcee13290 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 3 Sep 2021 15:12:06 -0400 Subject: [PATCH 29/30] Fix additional repository argument for suse build --- imgutil/imgutil | 2 ++ 1 file changed, 2 insertions(+) diff --git a/imgutil/imgutil b/imgutil/imgutil index 46d19e4d..6ba2d3dc 100644 --- a/imgutil/imgutil +++ b/imgutil/imgutil @@ -464,6 +464,8 @@ class SuseHandler(OsHandler): for source in self.addrepos.split(','): if not source: continue + if not source.startswith('/') and os.path.exists(os.path.abspath(source)): + source = os.path.abspath(source) source = 'file://' + source subprocess.check_call(['zypper', '-R', self.targpath, 'ar', source, 'source-{}'.format(idx)]) idx += 1 From 6465203918fe2de6c90d493e094fae8cfcaceeeb Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 3 Sep 2021 16:14:44 -0400 Subject: [PATCH 30/30] Add ability to pack and combine with a previous baseprofile For example, this is a good complement to unpack in a tethered environment: unpack profile-v1 /root/scratch #modify /root/scratch pack /root/scratch profile-v2 -b profile-v1 --- imgutil/imgutil | 72 ++++++++++++++++++++++++++++++++++--------------- 1 file changed, 50 insertions(+), 22 deletions(-) diff --git a/imgutil/imgutil b/imgutil/imgutil index 6ba2d3dc..fd237a44 100644 --- a/imgutil/imgutil +++ b/imgutil/imgutil @@ -636,6 +636,7 @@ def main(): packp = sps.add_parser('pack', help='Pack a scratch directory to a diskless profile') packp.add_argument('scratchdir', help='Directory containing diskless root') packp.add_argument('profilename', help='The desired diskless OS profile name to pack the root into') + packp.add_argument('-b', '--baseprofile', help='Profile to copy extra info from, for example to make a new version of an existing profile, reference the previous one as baseprofile', default=None) capturep = sps.add_parser('capture', help='Capture an image for cloning from a running system') capturep.add_argument('node', help='Node to capture image from') capturep.add_argument('profilename', help='Profile name for captured image') @@ -960,6 +961,25 @@ def unpack_image(args): time.sleep(0.1) +def recursecp(source, targ): + if os.path.islink(source): + if os.path.exists(targ): + return + linktarg = os.readlink(source) + os.symlink(linktarg, targ) + if os.path.isdir(source): + if not os.path.exists(targ): + os.mkdir(targ) + for entry in os.listdir(source): + srcentry = os.path.join(source, entry) + targentry = os.path.join(targ, entry) + recursecp(srcentry, targentry) + elif os.path.exists(targ): + return + else: + shutil.copy2(source, targ) + + def pack_image(args): outdir = args.profilename if '/' in outdir: @@ -1008,28 +1028,36 @@ def pack_image(args): encrypt_image(tmploc, os.path.join(outdir, 'rootimg.sfs'), '{}/pending/rootimg.key'.format(privdir)) os.remove(tmploc) with open(os.path.join(outdir, 'build-info'), 'w') as buildinfo: - buildinfo.write('Packed from {} on {}\n'.format(args.scratchdir, datetime.datetime.now().strftime('%Y-%m-%dT%H:%M'))) - if distpath: - os.symlink(distpath, os.path.join(outdir, 'distribution')) - oshandler = fingerprint_host(args, args.scratchdir) - tryupdate = False - if oshandler: - prettyname = oshandler.osname - with open(os.path.join(args.scratchdir, 'etc/os-release')) as osr: - osrdata = osr.read().split('\n') - for line in osrdata: - if line.startswith('PRETTY_NAME="'): - prettyname = line.replace( - 'PRETTY_NAME=', '').replace('"', '') - label = '{0} ({1})'.format(prettyname, 'Diskless Boot') - with open(os.path.join(outdir, 'profile.yaml'), 'w') as profiley: - profiley.write('label: {0}\nkernelargs: quiet # confluent_imagemethod=untethered|tethered # tethered is default when unspecified to save on memory, untethered will use more ram, but will not have any ongoing runtime root fs dependency on the http servers.\n'.format(label)) - oscat = oshandler.oscategory - confdir = '/opt/confluent/lib/osdeploy/{}-diskless'.format(oscat) - os.symlink('{}/initramfs/addons.cpio'.format(confdir), - os.path.join(outdir, 'boot/initramfs/addons.cpio')) - if os.path.exists('{}/profiles/default'.format(confdir)): - copy_tree('{}/profiles/default'.format(confdir), outdir) + buildinfo.write('PACKEDFROM={}\nPACKDATE={}\n'.format(args.scratchdir, datetime.datetime.now().strftime('%Y-%m-%dT%H:%M'))) + if args.baseprofile: + buildinfo.write('BASEPROFILE={}\n'.format(args.baseprofile)) + if args.baseprofile: + if '/' not in args.baseprofile: + args.baseprofile = os.path.join('/var/lib/confluent/public/os', args.baseprofile) + recursecp(args.baseprofile, outdir) + tryupdate = True + else: + if distpath: + os.symlink(distpath, os.path.join(outdir, 'distribution')) + oshandler = fingerprint_host(args, args.scratchdir) + tryupdate = False + if oshandler: + prettyname = oshandler.osname + with open(os.path.join(args.scratchdir, 'etc/os-release')) as osr: + osrdata = osr.read().split('\n') + for line in osrdata: + if line.startswith('PRETTY_NAME="'): + prettyname = line.replace( + 'PRETTY_NAME=', '').replace('"', '') + label = '{0} ({1})'.format(prettyname, 'Diskless Boot') + with open(os.path.join(outdir, 'profile.yaml'), 'w') as profiley: + profiley.write('label: {0}\nkernelargs: quiet # confluent_imagemethod=untethered|tethered # tethered is default when unspecified to save on memory, untethered will use more ram, but will not have any ongoing runtime root fs dependency on the http servers.\n'.format(label)) + oscat = oshandler.oscategory + confdir = '/opt/confluent/lib/osdeploy/{}-diskless'.format(oscat) + os.symlink('{}/initramfs/addons.cpio'.format(confdir), + os.path.join(outdir, 'boot/initramfs/addons.cpio')) + if os.path.exists('{}/profiles/default'.format(confdir)): + copy_tree('{}/profiles/default'.format(confdir), outdir) tryupdate = True try: pwd.getpwnam('confluent')