From 22008f9dc9ca9dcae3def2e25ce767ffaabd0b10 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 19 Jul 2021 17:30:26 -0400 Subject: [PATCH] Image cloning changes Refactor and try to mask ssh keys for root user. Try to preserve selinux context for masked files. Add progress indicator for writing to disk. --- .../profiles/default/scripts/image2disk.py | 68 ++++++++++++++++++- imgutil/imgutil | 44 ++++++++---- 2 files changed, 97 insertions(+), 15 deletions(-) diff --git a/confluent_osdeploy/el8-diskless/profiles/default/scripts/image2disk.py b/confluent_osdeploy/el8-diskless/profiles/default/scripts/image2disk.py index afe05b6a..0c484c71 100644 --- a/confluent_osdeploy/el8-diskless/profiles/default/scripts/image2disk.py +++ b/confluent_osdeploy/el8-diskless/profiles/default/scripts/image2disk.py @@ -2,7 +2,10 @@ import json import os import re +import time +import socket import struct +import sys import subprocess def get_next_part_meta(img, imgsize): @@ -78,6 +81,20 @@ def fixup(rootdir, vols): entry[3] = entry[3].ljust(28) tab = '\t'.join(entry) tfile.write(tab + '\n') + with open(os.path.join(rootdir, 'etc/hostname'), 'w') as nameout: + nameout.write(socket.gethostname()) + #NEED: grub config, ssh (maybe in script that calls image2disk), hostname. + #network interfaces, /etc/shadow of root's password, efibootmgr, various failed services + # grub error: +# error: ../../grub-core/commands/search.c:296:no such device: +#^M7c1840f3-64e3-4fca-ae3d-aa5ae9333e32. +#^Merror: ../../grub-core/commands/search.c:296:no such device: +#^M7c1840f3-64e3-4fca-ae3d-aa5ae9333e32. +#^Merror: ../../grub-core/commands/search.c:296:no such device: A278-1D2E. +#^Merror: ../../grub-core/commands/search.c:296:no such device: A278-1D2E. + + + def had_swap(): @@ -224,7 +241,56 @@ def install_to_disk(imgpath): subprocess.check_call(['mount', vol['targetdisk'], '/run/imginst/targ']) source = vol['mount'].replace('/', '_') source = '/run/imginst/sources/' + source - subprocess.check_call(['cp', '-ax', source + '/.', '/run/imginst/targ']) + blankfsstat = os.statvfs('/run/imginst/targ') + blankused = (blankfsstat.f_blocks - blankfsstat.f_bfree) * blankfsstat.f_bsize + sys.stdout.write('\nWriting {0}: '.format(vol['mount'])) + with subprocess.Popen(['cp', '-ax', source + '/.', '/run/imginst/targ']) as copier: + stillrunning = copier.poll() + lastprogress = 0.0 + while stillrunning is None: + currfsstat = os.statvfs('/run/imginst/targ') + currused = (currfsstat.f_blocks - currfsstat.f_bfree) * currfsstat.f_bsize + currused -= blankused + with open('/proc/meminfo') as meminf: + for line in meminf.read().split('\n'): + if line.startswith('Dirty:'): + _, dirty, _ = line.split() + dirty = int(dirty) * 1024 + progress = (currused - dirty) / vol['minsize'] + if progress < lastprogress: + progress = lastprogress + if progress > 0.99: + progress = 0.99 + lastprogress = progress + progress = progress * 100 + sys.stdout.write('\rWriting {0}: {1:3.2f}%'.format(vol['mount'], progress).ljust(70)) + sys.stdout.flush() + time.sleep(0.5) + stillrunning = copier.poll() + if stillrunning != 0: + raise Exception("Error copying volume") + with subprocess.Popen(['sync']) as syncrun: + stillrunning = syncrun.poll() + while stillrunning is None: + with open('/proc/meminfo') as meminf: + for line in meminf.read().split('\n'): + if line.startswith('Dirty:'): + _, dirty, _ = line.split() + dirty = int(dirty) * 1024 + progress = (vol['minsize'] - dirty) / vol['minsize'] + if progress < lastprogress: + progress = lastprogress + if progress > 0.99: + progress = 0.99 + lastprogress = progress + progress = progress * 100 + sys.stdout.write('\rWriting {0}: {1:3.2f}%'.format(vol['mount'], progress).ljust(70)) + sys.stdout.flush() + time.sleep(0.5) + stillrunning = syncrun.poll() + sys.stdout.write('\rDone writing {0}'.format(vol['mount']).ljust(70)) + sys.stdout.write('\n') + sys.stdout.flush() subprocess.check_call(['umount', '/run/imginst/targ']) for vol in allvols: subprocess.check_call(['mount', vol['targetdisk'], '/run/imginst/targ/' + vol['mount']]) diff --git a/imgutil/imgutil b/imgutil/imgutil index f0d839ee..38a15818 100644 --- a/imgutil/imgutil +++ b/imgutil/imgutil @@ -66,31 +66,47 @@ def sanitize_shadow(shadowfile): newshadow += ':'.join(passent) + '\n' return newshadow -def mask_file(filename, maskwith='/run/imgutil/captmp/empty'): - if os.path.exists(filename): - _mount_file(maskwith, filename) +def mask_file(mdir, filename, maskwith='/run/imgutil/captmp/empty'): + if filename.startswith(mdir): + filename = filename.replace(mdir, '', 1) + if filename[0] == '/': + filename = filename[1:] + filename = os.path.join(mdir, filename) + if filename[0] == '/': + filename = filename[1:] + filename = os.path.join('/run/imgutil/capin/', filename) + if os.path.exists(filename): + secontext = os.getxattr(filename, 'security.selinux') + _mount_file(maskwith, filename) + if secontext: + secontext = secontext.split(b'\x00', 1)[0].decode('utf8') + subprocess.check_call(['chcon', secontext, maskwith]) def capture_fs(args): fsinfo, fname = args _mount(fsinfo['mount'], '/run/imgutil/capin', flags=MS_BIND|MS_RDONLY) targdir = None - if fsinfo['mount'] == '/etc': - targdir = '/run/imgutil/capin' - elif fsinfo['mount'] == '/': - targdir = '/run/imgutil/capin/etc' - if targdir is not None: - mask_file(os.path.join(targdir, 'shadow'), '/run/imgutil/captmp/shadow') - mask_file(os.path.join(targdir, 'gshadow'), '/run/imgutil/captmp/gshadow') - mask_file(os.path.join(targdir, 'fstab'), '/run/imgutil/captmp/fstab') - mask_file(os.path.join(targdir, 'shadow-')) - mask_file(os.path.join(targdir, 'gshadow-')) - mask_file(os.path.join(targdir, 'hostname')) + mdir = fsinfo['mount'] + mask_file(mdir, '/etc/shadow', '/run/imgutil/captmp/shadow') + mask_file(mdir, '/etc/gshadow', '/run/imgutil/captmp/gshadow') + mask_file(mdir, '/etc/fstab', '/run/imgutil/captmp/fstab') + mask_file(mdir, '/etc/confluent/confluent.apikey') + mask_file(mdir, '/etc/shadow-') + mask_file(mdir, '/etc/gshadow-') + mask_file(mdir, '/etc/hostname') + if '/etc'.startswith(mdir): + targdir = '/etc'.replace(mdir, '', 1) + targdir = os.path.join('/run/imgutil/capin', targdir) for tname in glob.glob(os.path.join(targdir, 'ssh/*key')): _mount_file('/run/imgutil/captmp/empty', tname) for tname in glob.glob(os.path.join(targdir, 'pki/tls/private/*')): _mount_file('/run/imgutil/captmp/empty', tname) if os.path.exists(os.path.join(targdir, 'sysconfig/network-scripts')): _mount('none', os.path.join(targdir, 'sysconfig/network-scripts'), 'tmpfs') + if '/root'.startswith(mdir): + targdir = '/root'.replace(mdir, '', 1) + for tname in glob.glob(os.path.join(targdir, '.ssh/id_*')): + _mount_file('/run/imgutil/captmp/empty, tname') subprocess.check_call(['mksquashfs', '/run/imgutil/capin', fname + '.sfs', '-comp', 'xz']) def capture_system():