Implement rebase feature ofr osdeploy

Permit user to opt into a rebase of a
profile, to pick up potential updates
from the confluent packaged stock
profiles for files the user has not yet
customized.

confluent_client/confluent_env.sh

@@ -142,7 +142,7 @@ _confluent_osimage_completion()
 {
     _confluent_get_args
     if [ $NUMARGS == 2 ]; then
-        COMPREPLY=($(compgen -W "initialize import updateboot" -- ${COMP_WORDS[COMP_CWORD]}))
+        COMPREPLY=($(compgen -W "initialize import updateboot rebase" -- ${COMP_WORDS[COMP_CWORD]}))
         return
     elif [ ${CMPARGS[1]} == 'initialize' ]; then
         COMPREPLY=($(compgen -W "-h -u -s -t -i" -- ${COMP_WORDS[COMP_CWORD]}))
@@ -150,7 +150,7 @@ _confluent_osimage_completion()
         compopt -o default
         COMPREPLY=()
         return
-    elif [ ${CMPARGS[1]} == 'updateboot' ]; then
+    elif [ ${CMPARGS[1]} == 'updateboot' -o ${CMPARGS[1]} == 'rebase' ]; then
         COMPREPLY=($(compgen -W "-n $(confetty show /deployment/profiles|sed -e 's/\///')" -- "${COMP_WORDS[COMP_CWORD]}"))
         return
     fi

confluent_osdeploy/suse15/profiles/hpc/autoyast.leap

@@ -0,0 +1,132 @@
+<?xml version="1.0"?>
+<!DOCTYPE profile>
+<!--
+This autoyast file will be processed by pre.sh before applying. See pre.sh for
+info on modifying its behavior, and also search and replace '%%' to remove
+dynamic behavior and replace with static configuration.
+--> 
+<profile xmlns="http://www.suse.com/1.0/yast2ns" xmlns:config="http://www.suse.com/1.0/configns">
+    <timezone>
+      <hwclock>UTC</hwclock>
+      <!--INSERT:/tmp/timezone-->
+    </timezone>
+    <general>
+      <self_update config:type="boolean">false</self_update>
+      <mode>
+	      <confirm config:type="boolean">false</confirm>
+      </mode>
+    </general>
+    <!--INSERT:/tmp/bootloader.xml-->
+    <software>
+      <patterns config:type="list">
+        <pattern>base</pattern>
+      </patterns>
+      <packages config:type="list">
+	<package>openssh</package>
+	<package>iputils</package>
+        <package>python3</package>
+        <package>openssl</package>
+        <package>chrony</package>
+        <package>rsync</package>
+        <package>screen</package>
+        <package>vim</package>
+        <package>binutils</package>
+        <package>pciutils</package>
+        <package>usbutils</package>
+        <package>nfs-client</package>
+        <package>ethtool</package>
+      </packages>
+    </software>
+  <partitioning config:type="list">
+	  <drive>
+		  <device>%%INSTDISK%%</device>
+		  <initialize config:type="boolean">true</initialize>
+		  <use>all</use>
+                  <partitions config:type="list">
+                         <partition>
+                                 <filesystem config:type="symbol">xfs</filesystem>
+                                 <mount>/</mount>
+                                 <size>max</size>
+                         </partition>
+                         <partition>
+                                 <mount>swap</mount>
+                                 <size>auto</size>
+                         </partition>
+                         <partition>
+                                 <mount>/boot</mount>
+                                 <size>500M</size>
+                         </partition>
+                 </partitions>
+          </drive>
+ </partitioning>
+    <users config:type="list">
+      <user>
+        <username>root</username>
+        <user_password>%%ROOTPASSWORD%%</user_password>
+        <encrypted config:type="boolean">true</encrypted>
+	<!--INSERT:/tmp/rootkeys.xml-->
+      </user>
+    </users>
+    <networking>
+	    <dns>
+		    <hostname>%%NODENAME%%</hostname>
+            </dns>
+	    <keep_install_network config:type="boolean">true</keep_install_network>
+    </networking>
+    <services-manager>
+	    <services>
+		    <enable config:type="list">
+			    <service>sshd</service>
+			</enable>
+		</services>
+	</services-manager>
+    <scripts>
+         <pre-scripts config:type="list">
+        <script>
+          <filename>preinstall.sh</filename>
+          <source>
+<![CDATA[
+#!/bin/sh
+confluent_mgr=$(grep ^deploy_server /etc/confluent/confluent.deploycfg|awk '{print $2}')
+confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|sed -e 's/^profile: //')
+proto=$(grep ^protocol: /etc/confluent/confluent.deploycfg |awk '{print $2}')
+curl $proto://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/pre.sh > /tmp/pre.sh
+. /tmp/pre.sh
+]]>
+          </source>
+        </script>
+      </pre-scripts>
+         <chroot-scripts config:type="list">
+        <script>
+          <filename>chroot.sh</filename>
+          <source>
+<![CDATA[
+#!/bin/sh
+confluent_mgr=$(grep ^deploy_server /etc/confluent/confluent.deploycfg|awk '{print $2}')
+confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|sed -e 's/^profile: //')
+proto=$(grep ^protocol: /etc/confluent/confluent.deploycfg |awk '{print $2}')
+curl $proto://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/prechroot.sh > /tmp/prechroot.sh
+. /tmp/prechroot.sh
+curl -f $proto://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/firstboot.sh > /mnt/etc/confluent/firstboot.sh
+curl -f $proto://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/post.sh > /mnt/etc/confluent/post.sh
+chmod +x /mnt/etc/confluent/firstboot.sh
+chmod +x /mnt/etc/confluent/post.sh
+cp /mnt/etc/confluent/post.sh /mnt/var/adm/autoinstall/scripts/
+]]>
+          </source>
+        </script>
+        <script>
+          <chrooted config:type="boolean">true</chrooted>
+          <filename>post.sh</filename>
+          <location>file:///etc/confluent/post.sh</location>
+        </script>
+      </chroot-scripts>
+      <init-scripts config:type="list">
+        <script>
+          <filename>firstboot.sh</filename>
+          <location>file:///etc/confluent/firstboot.sh</location>
+        </script>
+     </init-scripts> 
+
+    </scripts>
+</profile>

confluent_osdeploy/suse15/profiles/hpc/autoyast.sle

@@ -16,7 +16,6 @@ dynamic behavior and replace with static configuration.
 	      <confirm config:type="boolean">false</confirm>
       </mode>
     </general>
-    %%IFSLE%%
     <add-on>
       <add_on_products config:type="list">
         <listentry><media_url/><!--INSERT:/tmp/pkgurl--><product>sle-module-basesystem</product><product_dir>/Module-Basesystem</product_dir></listentry>
@@ -27,14 +26,11 @@ dynamic behavior and replace with static configuration.
 	      <listentry><media_url/><!--INSERT:/tmp/pkgurl--><product>Legacy-Module</product><product_dir>/Module-Legacy</product_dir></listentry>
       </add_on_products>
     </add-on>
-    %%ENDIFSLE%%
     <!--INSERT:/tmp/bootloader.xml-->
     <software>
-       %%IFSLE%%
        <products config:type="list">
           <product>SLE_HPC</product>
       </products>
-       %%ENDIFSLE%%
       <patterns config:type="list">
         <pattern>base</pattern>
       </patterns>

confluent_osdeploy/suse15/profiles/hpc/initprofile.sh

@@ -17,7 +17,7 @@ ln -s $1/boot/x86_64/loader/initrd $2/boot/initramfs/distribution && \
 mkdir -p $2/boot/efi/boot && \
 ln -s $1/EFI/BOOT/bootx64.efi $1/EFI/BOOT/grub.efi $2/boot/efi/boot/
 if [[ $profile =~ ^sle.* ]]; then
-	sed -i 's/%%IFSLE%%//;s/%%ENDIFSLE%%//' $2/autoyast
+	ln -s autoyast.sle $2/autoyast
 else
-	sed -i '/%%IFSLE%%/,/%%ENDIFSLE%%/d' $2/autoyast
+	ln -s autoyast.leap $2/autoyast
 fi

confluent_osdeploy/suse15/profiles/server/autoyast.leap

@@ -0,0 +1,132 @@
+<?xml version="1.0"?>
+<!DOCTYPE profile>
+<!--
+This autoyast file will be processed by pre.sh before applying. See pre.sh for
+info on modifying its behavior, and also search and replace '%%' to remove
+dynamic behavior and replace with static configuration.
+--> 
+<profile xmlns="http://www.suse.com/1.0/yast2ns" xmlns:config="http://www.suse.com/1.0/configns">
+    <timezone>
+      <hwclock>UTC</hwclock>
+      <!--INSERT:/tmp/timezone-->
+    </timezone>
+    <general>
+      <self_update config:type="boolean">false</self_update>
+      <mode>
+	      <confirm config:type="boolean">false</confirm>
+      </mode>
+    </general>
+    <!--INSERT:/tmp/bootloader.xml-->
+    <software>
+      <patterns config:type="list">
+        <pattern>base</pattern>
+      </patterns>
+      <packages config:type="list">
+	<package>openssh</package>
+	<package>iputils</package>
+        <package>python3</package>
+        <package>openssl</package>
+        <package>chrony</package>
+        <package>rsync</package>
+        <package>screen</package>
+        <package>vim</package>
+        <package>binutils</package>
+        <package>pciutils</package>
+        <package>usbutils</package>
+        <package>nfs-client</package>
+        <package>ethtool</package>
+      </packages>
+    </software>
+  <partitioning config:type="list">
+	  <drive>
+		  <device>%%INSTDISK%%</device>
+		  <initialize config:type="boolean">true</initialize>
+		  <use>all</use>
+                  <partitions config:type="list">
+                         <partition>
+                                 <filesystem config:type="symbol">xfs</filesystem>
+                                 <mount>/</mount>
+                                 <size>max</size>
+                         </partition>
+                         <partition>
+                                 <mount>swap</mount>
+                                 <size>auto</size>
+                         </partition>
+                         <partition>
+                                 <mount>/boot</mount>
+                                 <size>500M</size>
+                         </partition>
+                 </partitions>
+          </drive>
+ </partitioning>
+    <users config:type="list">
+      <user>
+        <username>root</username>
+        <user_password>%%ROOTPASSWORD%%</user_password>
+        <encrypted config:type="boolean">true</encrypted>
+	<!--INSERT:/tmp/rootkeys.xml-->
+      </user>
+    </users>
+    <networking>
+	    <dns>
+		    <hostname>%%NODENAME%%</hostname>
+            </dns>
+	    <keep_install_network config:type="boolean">true</keep_install_network>
+    </networking>
+    <services-manager>
+	    <services>
+		    <enable config:type="list">
+			    <service>sshd</service>
+			</enable>
+		</services>
+	</services-manager>
+    <scripts>
+         <pre-scripts config:type="list">
+        <script>
+          <filename>preinstall.sh</filename>
+          <source>
+<![CDATA[
+#!/bin/sh
+confluent_mgr=$(grep ^deploy_server /etc/confluent/confluent.deploycfg|awk '{print $2}')
+confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|sed -e 's/^profile: //')
+proto=$(grep ^protocol: /etc/confluent/confluent.deploycfg |awk '{print $2}')
+curl $proto://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/pre.sh > /tmp/pre.sh
+. /tmp/pre.sh
+]]>
+          </source>
+        </script>
+      </pre-scripts>
+         <chroot-scripts config:type="list">
+        <script>
+          <filename>chroot.sh</filename>
+          <source>
+<![CDATA[
+#!/bin/sh
+confluent_mgr=$(grep ^deploy_server /etc/confluent/confluent.deploycfg|awk '{print $2}')
+confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|sed -e 's/^profile: //')
+proto=$(grep ^protocol: /etc/confluent/confluent.deploycfg |awk '{print $2}')
+curl $proto://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/prechroot.sh > /tmp/prechroot.sh
+. /tmp/prechroot.sh
+curl -f $proto://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/firstboot.sh > /mnt/etc/confluent/firstboot.sh
+curl -f $proto://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/post.sh > /mnt/etc/confluent/post.sh
+chmod +x /mnt/etc/confluent/firstboot.sh
+chmod +x /mnt/etc/confluent/post.sh
+cp /mnt/etc/confluent/post.sh /mnt/var/adm/autoinstall/scripts/
+]]>
+          </source>
+        </script>
+        <script>
+          <chrooted config:type="boolean">true</chrooted>
+          <filename>post.sh</filename>
+          <location>file:///etc/confluent/post.sh</location>
+        </script>
+      </chroot-scripts>
+      <init-scripts config:type="list">
+        <script>
+          <filename>firstboot.sh</filename>
+          <location>file:///etc/confluent/firstboot.sh</location>
+        </script>
+     </init-scripts> 
+
+    </scripts>
+</profile>

confluent_osdeploy/suse15/profiles/server/autoyast.sle

@@ -16,7 +16,6 @@ dynamic behavior and replace with static configuration.
 	      <confirm config:type="boolean">false</confirm>
       </mode>
     </general>
-    %%IFSLE%%
     <add-on>
       <add_on_products config:type="list">
         <listentry><media_url/><!--INSERT:/tmp/pkgurl--><product>sle-module-basesystem</product><product_dir>/Module-Basesystem</product_dir></listentry>
@@ -26,14 +25,11 @@ dynamic behavior and replace with static configuration.
 	      <listentry><media_url/><!--INSERT:/tmp/pkgurl--><product>Legacy-Module</product><product_dir>/Module-Legacy</product_dir></listentry>
       </add_on_products>
     </add-on>
-    %%ENDIFSLE%%
     <!--INSERT:/tmp/bootloader.xml-->
     <software>
-       %%IFSLE%%
        <products config:type="list">
           <product>SLES</product>
       </products>
-       %%ENDIFSLE%%
       <patterns config:type="list">
         <pattern>base</pattern>
       </patterns>

confluent_osdeploy/suse15/profiles/server/initprofile.sh

@@ -17,7 +17,7 @@ ln -s $1/boot/x86_64/loader/initrd $2/boot/initramfs/distribution && \
 mkdir -p $2/boot/efi/boot && \
 ln -s $1/EFI/BOOT/bootx64.efi $1/EFI/BOOT/grub.efi $2/boot/efi/boot/
 if [[ $profile =~ ^sle.* ]]; then
-	sed -i 's/%%IFSLE%%//;s/%%ENDIFSLE%%//' $2/autoyast
+	ln -s autoyast.sle $2/autoyast
 else
-	sed -i '/%%IFSLE%%/,/%%ENDIFSLE%%/d' $2/autoyast
+	ln -s autoyast.leap $2/autoyast
 fi

confluent_server/bin/osdeploy

@@ -56,6 +56,8 @@ def main(args):
         help='Push profile.yaml of the named profile data into boot assets as appropriate')
     upb.add_argument('profile', help='Profile to update boot assets')
     osls = sp.add_parser('list', help='List OS images available for deployment')
+    ubp = sp.add_parser('rebase', help='Update stock profile content from packaged updates')
+    ubp.add_argument('profile', help='Profile to rebase from packaged content')
     cmdset = ap.parse_args()
 
     if cmdset.command == 'list':
@@ -66,6 +68,8 @@ def main(args):
         return initialize(cmdset)
     if cmdset.command == 'updateboot':
         return updateboot(cmdset.profile)
+    if cmdset.command == 'rebase':
+        return rebase(cmdset.profile)
     ap.print_help()
 
 
@@ -404,6 +408,19 @@ def updateboot(profilename):
             print(repr(rsp))
 
 
+def rebase(profilename):
+    c = client.Command()
+    for rsp in c.update('/deployment/profiles/{0}'.format(profilename), {'rebase': 1}):
+        if 'updated' in rsp:
+            print('Updated: {0}'.format(rsp['updated']))
+        elif 'customized' in rsp:
+            print('Skipping update of {0} as current copy was customized or no manifest data was available'.format(rsp['customized']))
+        elif 'error' in rsp:
+            sys.stderr.write(rsp['error'] + '\n')
+            sys.exit(rsp['errorcode'])
+        else:
+            print(repr(rsp))
+
 def oslist():
     c = client.Command()
     print("Distributions:")

confluent_server/confluent/core.py

@@ -192,10 +192,21 @@ def handle_deployment(configmanager, inputdata, pathcomponents,
             return
         if len(pathcomponents) == 3:
             profname = pathcomponents[-1]
-            if operation == 'update' and 'updateboot' in inputdata:
-                osimage.update_boot(profname)
-                yield msg.KeyValueData({'updated': profname})
-                return
+            if operation == 'update':
+                if 'updateboot' in inputdata:
+                    osimage.update_boot(profname)
+                    yield msg.KeyValueData({'updated': profname})
+                    return
+                elif 'rebase' in inputdata:
+                    try:
+                        updated, customized = osimage.rebase_profile(profname)
+                    except osimage.ManifestMissing:
+                        raise exc.InvalidArgumentException('Specified profile {0} does not have a manifest.yaml for rebase'.format(profname))
+                    for upd in updated:
+                        yield msg.KeyValueData({'updated': upd})                        
+                    for cust in customized:
+                        yield msg.KeyValueData({'customized': cust})
+                    return
     if pathcomponents[1] == 'importing':
         if len(pathcomponents) == 2 or not pathcomponents[-1]:
             if operation == 'retrieve':

confluent_server/confluent/osimage.py

@@ -44,7 +44,7 @@ def relax_umask():
 
 def makedirs(path, mode):
     try:
-        os.makedirs(path, 0o755)
+        os.makedirs(path, mode)
     except OSError as e:
         if e.errno != 17:
             raise
@@ -645,6 +645,78 @@ def get_profile_label(profile):
 importing = {}
 
 
+class ManifestMissing(Exception):
+    pass
+
+def copy_file(src, dst):
+    newdir = os.path.dirname(dst)
+    makedirs(newdir, 0o755)
+    shutil.copy2(src, dst)
+
+def get_hash(fname):
+    currhash = hashlib.sha512()
+    with open(fname, 'rb') as currf:
+        currd = currf.read(2048)
+        while currd:
+            currhash.update(currd)
+            currd = currf.read(2048)
+    return currhash.hexdigest()
+
+
+def rebase_profile(dirname):
+    if dirname.startswith('/var/lib/confluent/public'):
+        profiledir = dirname
+    else:
+        profiledir = '/var/lib/confluent/public/os/{0}'.format(dirname)
+    currhashes = get_hashes(profiledir)
+    festfile = os.path.join(profiledir, 'manifest.yaml')
+    try:
+        with open(festfile, 'r') as festfile:
+            manifest = yaml.safe_load(festfile)
+    except IOError:
+        raise ManifestMissing()
+    distdir = manifest['distdir']
+    newdisthashes = get_hashes(distdir)
+    olddisthashes = manifest['disthashes']
+    customized = []
+    newmanifest = []
+    updated = []
+    for updatecandidate in newdisthashes:
+        newfilename = os.path.join(profiledir, updatecandidate)
+        distfilename = os.path.join(distdir, updatecandidate)
+        newdisthash = newdisthashes[updatecandidate]
+        currhash = currhashes.get(updatecandidate, None)
+        olddisthash = olddisthashes.get(updatecandidate, None)
+        if not currhash:  # file does not exist yet
+            copy_file(distfilename, newfilename)
+            newmanifest.append(updatecandidate)
+            updated.append(updatecandidate)
+        elif currhash == newdisthash:
+            newmanifest.append(updatecandidate)
+        elif currhash != olddisthash:
+            customized.append(updatecandidate)
+        else:
+            copy_file(distfilename, newfilename)
+            updated.append(updatecandidate)
+            newmanifest.append(updatecandidate)
+    for nf in newmanifest:
+        nfname = os.path.join(profiledir, nf)
+        currhash = get_hash(nfname)
+        manifest['disthashes'][nf] = currhash
+    with open('{0}/manifest.yaml'.format(profiledir), 'w') as yout:
+            yout.write('# This manifest enables rebase to know original source of profile data and if any customizations have been done\n')
+            yout.write(yaml.dump(manifest, default_flow_style=False))
+    return updated, customized
+
+    # if currhash == disthash:
+    #      no update required, update manifest
+    # elif currhash != olddisthash:
+    #      customization detected, skip
+    # else
+    #      update required, manifest update
+    
+    
+        
 def get_hashes(dirname):
     hashmap = {}
     for dname, _, fnames in os.walk(dirname):
@@ -653,13 +725,8 @@ def get_hashes(dirname):
                 continue
             fullname = os.path.join(dname, fname)
             currhash = hashlib.sha512()
-            with open(fullname, 'rb') as currf:
-                currd = currf.read(2048)
-                while currd:
-                    currhash.update(currd)
-                    currd = currf.read(2048)
-                subname = fullname.replace(dirname + '/', '')
-                hashmap[subname] = currhash.hexdigest()
+            subname = fullname.replace(dirname + '/', '')
+            hashmap[subname] = get_hash(fullname)
     return hashmap