From 352da940055d60356ecfae05d4134c5443667339 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 25 Aug 2022 15:21:49 -0400 Subject: [PATCH] Implement rebase feature ofr osdeploy Permit user to opt into a rebase of a profile, to pick up potential updates from the confluent packaged stock profiles for files the user has not yet customized. --- confluent_client/confluent_env.sh | 4 +- .../suse15/profiles/hpc/autoyast.leap | 132 ++++++++++++++++++ .../profiles/hpc/{autoyast => autoyast.sle} | 4 - .../suse15/profiles/hpc/initprofile.sh | 4 +- .../suse15/profiles/server/autoyast.leap | 132 ++++++++++++++++++ .../server/{autoyast => autoyast.sle} | 4 - .../suse15/profiles/server/initprofile.sh | 4 +- confluent_server/bin/osdeploy | 17 +++ confluent_server/confluent/core.py | 19 ++- confluent_server/confluent/osimage.py | 83 +++++++++-- 10 files changed, 377 insertions(+), 26 deletions(-) create mode 100644 confluent_osdeploy/suse15/profiles/hpc/autoyast.leap rename confluent_osdeploy/suse15/profiles/hpc/{autoyast => autoyast.sle} (98%) create mode 100644 confluent_osdeploy/suse15/profiles/server/autoyast.leap rename confluent_osdeploy/suse15/profiles/server/{autoyast => autoyast.sle} (98%) diff --git a/confluent_client/confluent_env.sh b/confluent_client/confluent_env.sh index 0b13574b..4d6cba5f 100644 --- a/confluent_client/confluent_env.sh +++ b/confluent_client/confluent_env.sh @@ -142,7 +142,7 @@ _confluent_osimage_completion() { _confluent_get_args if [ $NUMARGS == 2 ]; then - COMPREPLY=($(compgen -W "initialize import updateboot" -- ${COMP_WORDS[COMP_CWORD]})) + COMPREPLY=($(compgen -W "initialize import updateboot rebase" -- ${COMP_WORDS[COMP_CWORD]})) return elif [ ${CMPARGS[1]} == 'initialize' ]; then COMPREPLY=($(compgen -W "-h -u -s -t -i" -- ${COMP_WORDS[COMP_CWORD]})) @@ -150,7 +150,7 @@ _confluent_osimage_completion() compopt -o default COMPREPLY=() return - elif [ ${CMPARGS[1]} == 'updateboot' ]; then + elif [ ${CMPARGS[1]} == 'updateboot' -o ${CMPARGS[1]} == 'rebase' ]; then COMPREPLY=($(compgen -W "-n $(confetty show /deployment/profiles|sed -e 's/\///')" -- "${COMP_WORDS[COMP_CWORD]}")) return fi diff --git a/confluent_osdeploy/suse15/profiles/hpc/autoyast.leap b/confluent_osdeploy/suse15/profiles/hpc/autoyast.leap new file mode 100644 index 00000000..d3c6971a --- /dev/null +++ b/confluent_osdeploy/suse15/profiles/hpc/autoyast.leap @@ -0,0 +1,132 @@ + + + + + + UTC + + + + false + + false + + + + + + base + + + openssh + iputils + python3 + openssl + chrony + rsync + screen + vim + binutils + pciutils + usbutils + nfs-client + ethtool + + + + + %%INSTDISK%% + true + all + + + xfs + / + max + + + swap + auto + + + /boot + 500M + + + + + + + root + %%ROOTPASSWORD%% + true + + + + + + %%NODENAME%% + + true + + + + + sshd + + + + + + + + + + + + + + + + + diff --git a/confluent_osdeploy/suse15/profiles/hpc/autoyast b/confluent_osdeploy/suse15/profiles/hpc/autoyast.sle similarity index 98% rename from confluent_osdeploy/suse15/profiles/hpc/autoyast rename to confluent_osdeploy/suse15/profiles/hpc/autoyast.sle index 3db89f76..cf3f5569 100644 --- a/confluent_osdeploy/suse15/profiles/hpc/autoyast +++ b/confluent_osdeploy/suse15/profiles/hpc/autoyast.sle @@ -16,7 +16,6 @@ dynamic behavior and replace with static configuration. false - %%IFSLE%% sle-module-basesystem/Module-Basesystem @@ -27,14 +26,11 @@ dynamic behavior and replace with static configuration. Legacy-Module/Module-Legacy - %%ENDIFSLE%% - %%IFSLE%% SLE_HPC - %%ENDIFSLE%% base diff --git a/confluent_osdeploy/suse15/profiles/hpc/initprofile.sh b/confluent_osdeploy/suse15/profiles/hpc/initprofile.sh index 8f338896..9c6c295e 100644 --- a/confluent_osdeploy/suse15/profiles/hpc/initprofile.sh +++ b/confluent_osdeploy/suse15/profiles/hpc/initprofile.sh @@ -17,7 +17,7 @@ ln -s $1/boot/x86_64/loader/initrd $2/boot/initramfs/distribution && \ mkdir -p $2/boot/efi/boot && \ ln -s $1/EFI/BOOT/bootx64.efi $1/EFI/BOOT/grub.efi $2/boot/efi/boot/ if [[ $profile =~ ^sle.* ]]; then - sed -i 's/%%IFSLE%%//;s/%%ENDIFSLE%%//' $2/autoyast + ln -s autoyast.sle $2/autoyast else - sed -i '/%%IFSLE%%/,/%%ENDIFSLE%%/d' $2/autoyast + ln -s autoyast.leap $2/autoyast fi diff --git a/confluent_osdeploy/suse15/profiles/server/autoyast.leap b/confluent_osdeploy/suse15/profiles/server/autoyast.leap new file mode 100644 index 00000000..d3c6971a --- /dev/null +++ b/confluent_osdeploy/suse15/profiles/server/autoyast.leap @@ -0,0 +1,132 @@ + + + + + + UTC + + + + false + + false + + + + + + base + + + openssh + iputils + python3 + openssl + chrony + rsync + screen + vim + binutils + pciutils + usbutils + nfs-client + ethtool + + + + + %%INSTDISK%% + true + all + + + xfs + / + max + + + swap + auto + + + /boot + 500M + + + + + + + root + %%ROOTPASSWORD%% + true + + + + + + %%NODENAME%% + + true + + + + + sshd + + + + + + + + + + + + + + + + + diff --git a/confluent_osdeploy/suse15/profiles/server/autoyast b/confluent_osdeploy/suse15/profiles/server/autoyast.sle similarity index 98% rename from confluent_osdeploy/suse15/profiles/server/autoyast rename to confluent_osdeploy/suse15/profiles/server/autoyast.sle index 59e4c7c9..b8e271f1 100644 --- a/confluent_osdeploy/suse15/profiles/server/autoyast +++ b/confluent_osdeploy/suse15/profiles/server/autoyast.sle @@ -16,7 +16,6 @@ dynamic behavior and replace with static configuration. false - %%IFSLE%% sle-module-basesystem/Module-Basesystem @@ -26,14 +25,11 @@ dynamic behavior and replace with static configuration. Legacy-Module/Module-Legacy - %%ENDIFSLE%% - %%IFSLE%% SLES - %%ENDIFSLE%% base diff --git a/confluent_osdeploy/suse15/profiles/server/initprofile.sh b/confluent_osdeploy/suse15/profiles/server/initprofile.sh index 8f338896..9c6c295e 100644 --- a/confluent_osdeploy/suse15/profiles/server/initprofile.sh +++ b/confluent_osdeploy/suse15/profiles/server/initprofile.sh @@ -17,7 +17,7 @@ ln -s $1/boot/x86_64/loader/initrd $2/boot/initramfs/distribution && \ mkdir -p $2/boot/efi/boot && \ ln -s $1/EFI/BOOT/bootx64.efi $1/EFI/BOOT/grub.efi $2/boot/efi/boot/ if [[ $profile =~ ^sle.* ]]; then - sed -i 's/%%IFSLE%%//;s/%%ENDIFSLE%%//' $2/autoyast + ln -s autoyast.sle $2/autoyast else - sed -i '/%%IFSLE%%/,/%%ENDIFSLE%%/d' $2/autoyast + ln -s autoyast.leap $2/autoyast fi diff --git a/confluent_server/bin/osdeploy b/confluent_server/bin/osdeploy index 378d00d2..a9bb2d2d 100644 --- a/confluent_server/bin/osdeploy +++ b/confluent_server/bin/osdeploy @@ -56,6 +56,8 @@ def main(args): help='Push profile.yaml of the named profile data into boot assets as appropriate') upb.add_argument('profile', help='Profile to update boot assets') osls = sp.add_parser('list', help='List OS images available for deployment') + ubp = sp.add_parser('rebase', help='Update stock profile content from packaged updates') + ubp.add_argument('profile', help='Profile to rebase from packaged content') cmdset = ap.parse_args() if cmdset.command == 'list': @@ -66,6 +68,8 @@ def main(args): return initialize(cmdset) if cmdset.command == 'updateboot': return updateboot(cmdset.profile) + if cmdset.command == 'rebase': + return rebase(cmdset.profile) ap.print_help() @@ -404,6 +408,19 @@ def updateboot(profilename): print(repr(rsp)) +def rebase(profilename): + c = client.Command() + for rsp in c.update('/deployment/profiles/{0}'.format(profilename), {'rebase': 1}): + if 'updated' in rsp: + print('Updated: {0}'.format(rsp['updated'])) + elif 'customized' in rsp: + print('Skipping update of {0} as current copy was customized or no manifest data was available'.format(rsp['customized'])) + elif 'error' in rsp: + sys.stderr.write(rsp['error'] + '\n') + sys.exit(rsp['errorcode']) + else: + print(repr(rsp)) + def oslist(): c = client.Command() print("Distributions:") diff --git a/confluent_server/confluent/core.py b/confluent_server/confluent/core.py index 61b23c69..718dcc52 100644 --- a/confluent_server/confluent/core.py +++ b/confluent_server/confluent/core.py @@ -192,10 +192,21 @@ def handle_deployment(configmanager, inputdata, pathcomponents, return if len(pathcomponents) == 3: profname = pathcomponents[-1] - if operation == 'update' and 'updateboot' in inputdata: - osimage.update_boot(profname) - yield msg.KeyValueData({'updated': profname}) - return + if operation == 'update': + if 'updateboot' in inputdata: + osimage.update_boot(profname) + yield msg.KeyValueData({'updated': profname}) + return + elif 'rebase' in inputdata: + try: + updated, customized = osimage.rebase_profile(profname) + except osimage.ManifestMissing: + raise exc.InvalidArgumentException('Specified profile {0} does not have a manifest.yaml for rebase'.format(profname)) + for upd in updated: + yield msg.KeyValueData({'updated': upd}) + for cust in customized: + yield msg.KeyValueData({'customized': cust}) + return if pathcomponents[1] == 'importing': if len(pathcomponents) == 2 or not pathcomponents[-1]: if operation == 'retrieve': diff --git a/confluent_server/confluent/osimage.py b/confluent_server/confluent/osimage.py index 7c1bd17e..68fc22b7 100644 --- a/confluent_server/confluent/osimage.py +++ b/confluent_server/confluent/osimage.py @@ -44,7 +44,7 @@ def relax_umask(): def makedirs(path, mode): try: - os.makedirs(path, 0o755) + os.makedirs(path, mode) except OSError as e: if e.errno != 17: raise @@ -645,6 +645,78 @@ def get_profile_label(profile): importing = {} +class ManifestMissing(Exception): + pass + +def copy_file(src, dst): + newdir = os.path.dirname(dst) + makedirs(newdir, 0o755) + shutil.copy2(src, dst) + +def get_hash(fname): + currhash = hashlib.sha512() + with open(fname, 'rb') as currf: + currd = currf.read(2048) + while currd: + currhash.update(currd) + currd = currf.read(2048) + return currhash.hexdigest() + + +def rebase_profile(dirname): + if dirname.startswith('/var/lib/confluent/public'): + profiledir = dirname + else: + profiledir = '/var/lib/confluent/public/os/{0}'.format(dirname) + currhashes = get_hashes(profiledir) + festfile = os.path.join(profiledir, 'manifest.yaml') + try: + with open(festfile, 'r') as festfile: + manifest = yaml.safe_load(festfile) + except IOError: + raise ManifestMissing() + distdir = manifest['distdir'] + newdisthashes = get_hashes(distdir) + olddisthashes = manifest['disthashes'] + customized = [] + newmanifest = [] + updated = [] + for updatecandidate in newdisthashes: + newfilename = os.path.join(profiledir, updatecandidate) + distfilename = os.path.join(distdir, updatecandidate) + newdisthash = newdisthashes[updatecandidate] + currhash = currhashes.get(updatecandidate, None) + olddisthash = olddisthashes.get(updatecandidate, None) + if not currhash: # file does not exist yet + copy_file(distfilename, newfilename) + newmanifest.append(updatecandidate) + updated.append(updatecandidate) + elif currhash == newdisthash: + newmanifest.append(updatecandidate) + elif currhash != olddisthash: + customized.append(updatecandidate) + else: + copy_file(distfilename, newfilename) + updated.append(updatecandidate) + newmanifest.append(updatecandidate) + for nf in newmanifest: + nfname = os.path.join(profiledir, nf) + currhash = get_hash(nfname) + manifest['disthashes'][nf] = currhash + with open('{0}/manifest.yaml'.format(profiledir), 'w') as yout: + yout.write('# This manifest enables rebase to know original source of profile data and if any customizations have been done\n') + yout.write(yaml.dump(manifest, default_flow_style=False)) + return updated, customized + + # if currhash == disthash: + # no update required, update manifest + # elif currhash != olddisthash: + # customization detected, skip + # else + # update required, manifest update + + + def get_hashes(dirname): hashmap = {} for dname, _, fnames in os.walk(dirname): @@ -653,13 +725,8 @@ def get_hashes(dirname): continue fullname = os.path.join(dname, fname) currhash = hashlib.sha512() - with open(fullname, 'rb') as currf: - currd = currf.read(2048) - while currd: - currhash.update(currd) - currd = currf.read(2048) - subname = fullname.replace(dirname + '/', '') - hashmap[subname] = currhash.hexdigest() + subname = fullname.replace(dirname + '/', '') + hashmap[subname] = get_hash(fullname) return hashmap