diff --git a/confluent_osdeploy/suse15/profiles/server/ansible/firstboot.d/README.txt b/confluent_osdeploy/suse15/profiles/server/ansible/firstboot.d/README.txt new file mode 100644 index 00000000..97e5f506 --- /dev/null +++ b/confluent_osdeploy/suse15/profiles/server/ansible/firstboot.d/README.txt @@ -0,0 +1,25 @@ +Ansible playbooks ending in .yml or .yaml that are placed into this directory will be executed at the +appropriate phase of the install process. + +The 'hosts' may be omitted, and if included will be ignored, replaced with the host that is specifically +requesting the playbooks be executed. + +Also, the playbooks will be executed on the deployment server. Hence it may be slower in aggregate than +running content under scripts/ which ask much less of the deployment server + +Here is an example of what a playbook would look like broadly: + +- name: Example + gather_facts: no + tasks: + - name: Example1 + lineinfile: + path: /etc/hosts + line: 1.2.3.4 test1 + create: yes + - name: Example2 + lineinfile: + path: /etc/hosts + line: 1.2.3.5 test2 + create: yes + diff --git a/confluent_osdeploy/suse15/profiles/server/ansible/post.d/README.txt b/confluent_osdeploy/suse15/profiles/server/ansible/post.d/README.txt new file mode 100644 index 00000000..97e5f506 --- /dev/null +++ b/confluent_osdeploy/suse15/profiles/server/ansible/post.d/README.txt @@ -0,0 +1,25 @@ +Ansible playbooks ending in .yml or .yaml that are placed into this directory will be executed at the +appropriate phase of the install process. + +The 'hosts' may be omitted, and if included will be ignored, replaced with the host that is specifically +requesting the playbooks be executed. + +Also, the playbooks will be executed on the deployment server. Hence it may be slower in aggregate than +running content under scripts/ which ask much less of the deployment server + +Here is an example of what a playbook would look like broadly: + +- name: Example + gather_facts: no + tasks: + - name: Example1 + lineinfile: + path: /etc/hosts + line: 1.2.3.4 test1 + create: yes + - name: Example2 + lineinfile: + path: /etc/hosts + line: 1.2.3.5 test2 + create: yes + diff --git a/confluent_osdeploy/suse15/profiles/server/autoyast b/confluent_osdeploy/suse15/profiles/server/autoyast new file mode 100644 index 00000000..59e4c7c9 --- /dev/null +++ b/confluent_osdeploy/suse15/profiles/server/autoyast @@ -0,0 +1,148 @@ + + + + + + UTC + + + + false + + false + + + %%IFSLE%% + + + sle-module-basesystem/Module-Basesystem + sle-module-server-applications/Module-Server-Applications + sle-module-containers/Module-Containers + SLES/Product-SLES + Legacy-Module/Module-Legacy + + + %%ENDIFSLE%% + + + %%IFSLE%% + + SLES + + %%ENDIFSLE%% + + base + + + openssh + iputils + python3 + openssl + chrony + rsync + screen + vim + binutils + pciutils + usbutils + nfs-client + ethtool + + + + + %%INSTDISK%% + true + all + + + xfs + / + max + + + swap + auto + + + /boot + 500M + + + + + + + root + %%ROOTPASSWORD%% + true + + + + + + %%NODENAME%% + + true + + + + + sshd + + + + + + + + + + + + + + + + + diff --git a/confluent_osdeploy/suse15/profiles/server/initprofile.sh b/confluent_osdeploy/suse15/profiles/server/initprofile.sh new file mode 100644 index 00000000..8f338896 --- /dev/null +++ b/confluent_osdeploy/suse15/profiles/server/initprofile.sh @@ -0,0 +1,23 @@ +#!/bin/sh +discnum=$(basename $1) +if [ "$discnum" != 1 ]; then exit 0; fi +if [ -e $2/boot/kernel ]; then exit 0; fi +profile=$(basename $2) + +if [[ $profile =~ ^sle.* ]]; then + if ls $1/Product-* >& /dev/null; then + ln -s $1 $2/product + else + ln -s ${1%1}2 $2/product + fi +fi +sed -i 's/sle 15/SUSE Linux Enterprise 15/; s/opensuse_leap/openSUSE Leap/' $2/profile.yaml +ln -s $1/boot/x86_64/loader/linux $2/boot/kernel && \ +ln -s $1/boot/x86_64/loader/initrd $2/boot/initramfs/distribution && \ +mkdir -p $2/boot/efi/boot && \ +ln -s $1/EFI/BOOT/bootx64.efi $1/EFI/BOOT/grub.efi $2/boot/efi/boot/ +if [[ $profile =~ ^sle.* ]]; then + sed -i 's/%%IFSLE%%//;s/%%ENDIFSLE%%//' $2/autoyast +else + sed -i '/%%IFSLE%%/,/%%ENDIFSLE%%/d' $2/autoyast +fi diff --git a/confluent_osdeploy/suse15/profiles/server/profile.yaml b/confluent_osdeploy/suse15/profiles/server/profile.yaml new file mode 100644 index 00000000..99e1a44d --- /dev/null +++ b/confluent_osdeploy/suse15/profiles/server/profile.yaml @@ -0,0 +1,3 @@ +label: %%DISTRO%% %%VERSION%% %%ARCH%% (Default Profile) +kernelargs: quiet # These arguments are passed to the installer +#installedargs: example # These arguments would be added to the installed system diff --git a/confluent_osdeploy/suse15/profiles/server/scripts/firstboot.custom b/confluent_osdeploy/suse15/profiles/server/scripts/firstboot.custom new file mode 100644 index 00000000..e2b7f793 --- /dev/null +++ b/confluent_osdeploy/suse15/profiles/server/scripts/firstboot.custom @@ -0,0 +1,9 @@ +#!/bin/sh + +# This script runs at the end of the final boot + +. /etc/confluent/functions + +# Custom scripts may go here +# run_remote example.sh +# run_remote_python example.py diff --git a/confluent_osdeploy/suse15/profiles/server/scripts/firstboot.sh b/confluent_osdeploy/suse15/profiles/server/scripts/firstboot.sh new file mode 100644 index 00000000..8cd7ac9b --- /dev/null +++ b/confluent_osdeploy/suse15/profiles/server/scripts/firstboot.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +# This script runs at the end of the final boot, updating status +exec >> /var/log/confluent/confluent-firstboot.log +exec 2>> /var/log/confluent/confluent-firstboot.log +chmod 600 /var/log/confluent/confluent-firstboot.log + +nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}') +v6cfg=$(grep ^ipv6_method: /etc/confluent/confluent.deploycfg) +v6cfg=${v6cfg#ipv6_method: } +if [ "$v6cfg" = "static" ]; then + confluent_mgr=$(grep ^deploy_server_v6: /etc/confluent/confluent.deploycfg) + confluent_mgr=${confluent_mgr#deploy_server_v6: } + confluent_mgr="[$confluent_mgr]" +else + confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg) + confluent_mgr=${confluent_mgr#deploy_server: } +fi +confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|sed -e 's/^profile: //') +proto=$(grep ^protocol: /etc/confluent/confluent.deploycfg |awk '{print $2}') +confluent_apikey=$(cat /etc/confluent/confluent.apikey) +. /etc/confluent/functions +while ! ping -c 1 $confluent_mgr >& /dev/null; do + sleep 1 +done + +for i in /etc/ssh/ssh_host*key.pub; do + certname=${i/.pub/-cert.pub} + curl -f -X POST -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $(cat /etc/confluent/confluent.apikey)" -d @$i https://$confluent_mgr/confluent-api/self/sshcert > $certname +done +systemctl restart sshd +run_remote_python confignet +run_remote firstboot.custom + +# Firstboot scripts may be placed into firstboot.d, e.g. firstboot.d/01-firstaction.sh, firstboot.d/02-secondaction.sh +run_remote_parts firstboot.d + +# Induce execution of remote configuration, e.g. ansible plays in ansible/firstboot.d/ +run_remote_config firstboot.d + +curl --capath /etc/confluent/tls -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" -f -X POST -d "status: complete" https://$confluent_mgr/confluent-api/self/updatestatus diff --git a/confluent_osdeploy/suse15/profiles/server/scripts/functions b/confluent_osdeploy/suse15/profiles/server/scripts/functions new file mode 100644 index 00000000..d70c63db --- /dev/null +++ b/confluent_osdeploy/suse15/profiles/server/scripts/functions @@ -0,0 +1,196 @@ +#!/bin/bash +function test_mgr() { + if curl -s https://${1}/confluent-api/ > /dev/null; then + return 0 + fi + return 1 +} + +function confluentpython() { + if [ -x /usr/libexec/platform-python ]; then + /usr/libexec/platform-python $* + elif [ -x /usr/bin/python3 ]; then + /usr/bin/python3 $* + elif [ -x /usr/bin/python ]; then + /usr/bin/python $* + elif [ -x /usr/bin/python2 ]; then + /usr/bin/python2 $* + fi +} + +function set_confluent_vars() { + if [ -z "$nodename" ]; then + nodename=$(grep ^NODENAME: /etc/confluent/confluent.info | awk '{print $2}') + fi + if [[ "$confluent_mgr" == *"%"* ]]; then + confluent_mgr="" + fi + if [ -z "$confluent_mgr" ]; then + confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //') + if ! test_mgr $confluent_mgr; then + confluent_mgr=$(grep ^deploy_server_v6: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //') + if [[ "$confluent_mgr" = *":"* ]]; then + confluent_mgr="[$confluent_mgr]" + fi + fi + if ! test_mgr $confluent_mgr; then + BESTMGRS=$(grep ^EXTMGRINFO: /etc/confluent/confluent.info | grep '|1$' | sed -e 's/EXTMGRINFO: //' -e 's/|.*//') + OKMGRS=$(grep ^EXTMGRINFO: /etc/confluent/confluent.info | grep '|0$' | sed -e 's/EXTMGRINFO: //' -e 's/|.*//') + for confluent_mgr in $BESTMGRS $OKMGRS; do + if [[ $confluent_mgr == *":"* ]]; then + confluent_mgr="[$confluent_mgr]" + fi + if test_mgr $confluent_mgr; then + break + fi + done + fi + fi + if [ -z "$confluent_profile" ]; then + confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //') + fi +} + +fetch_remote() { + curlargs="" + if [ -f /etc/confluent/ca.pem ]; then + curlargs=" --cacert /etc/confluent/ca.pem" + fi + set_confluent_vars + mkdir -p $(dirname $1) + curl -f -sS $curlargs https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/$1 > $1 + if [ $? != 0 ]; then echo $1 failed to download; return 1; fi +} + +source_remote_parts() { + confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX) + apiclient=/opt/confluent/bin/apiclient + if [ -f /etc/confluent/apiclient ]; then + apiclient=/etc/confluent/apiclient + fi + scriptlist=$(confluentpython $apiclient /confluent-api/self/scriptlist/$1|sed -e 's/^- //') + for script in $scriptlist; do + source_remote $1/$script + done + rm -rf $confluentscripttmpdir + unset confluentscripttmpdir +} + +run_remote_parts() { + confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX) + apiclient=/opt/confluent/bin/apiclient + if [ -f /etc/confluent/apiclient ]; then + apiclient=/etc/confluent/apiclient + fi + scriptlist=$(confluentpython $apiclient /confluent-api/self/scriptlist/$1|sed -e 's/^- //') + for script in $scriptlist; do + run_remote $1/$script + done + rm -rf $confluentscripttmpdir + unset confluentscripttmpdir +} + +source_remote() { + set_confluent_vars + unsettmpdir=0 + echo + echo '---------------------------------------------------------------------------' + echo Sourcing $1 from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/ + if [ -z "$confluentscripttmpdir" ]; then + confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX) + unsettmpdir=1 + fi + echo Sourcing from $confluentscripttmpdir + cd $confluentscripttmpdir + fetch_remote $1 + if [ $? != 0 ]; then echo $1 failed to download; return 1; fi + chmod +x $1 + cmd=$1 + shift + source ./$cmd + cd - > /dev/null + if [ "$unsettmpdir" = 1 ]; then + rm -rf $confluentscripttmpdir + unset confluentscripttmpdir + unsettmpdir=0 + fi + rm -rf $confluentscripttmpdir + return $retcode +} + +run_remote() { + requestedcmd="'$*'" + unsettmpdir=0 + set_confluent_vars + echo + echo '---------------------------------------------------------------------------' + echo Running $requestedcmd from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/ + if [ -z "$confluentscripttmpdir" ]; then + confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX) + unsettmpdir=1 + fi + echo Executing in $confluentscripttmpdir + cd $confluentscripttmpdir + fetch_remote $1 + if [ $? != 0 ]; then echo $requestedcmd failed to download; return 1; fi + chmod +x $1 + cmd=$1 + if [ -x /usr/bin/chcon ]; then + chcon system_u:object_r:bin_t:s0 $cmd + fi + shift + ./$cmd $* + retcode=$? + if [ $retcode -ne 0 ]; then + echo "$requestedcmd exited with code $retcode" + fi + cd - > /dev/null + if [ "$unsettmpdir" = 1 ]; then + rm -rf $confluentscripttmpdir + unset confluentscripttmpdir + unsettmpdir=0 + fi + return $retcode +} + +run_remote_python() { + echo + set_confluent_vars + if [ -f /etc/confluent/ca.pem ]; then + curlargs=" --cacert /etc/confluent/ca.pem" + fi + echo '---------------------------------------------------------------------------' + echo Running python script "'$*'" from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/ + confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX) + echo Executing in $confluentscripttmpdir + cd $confluentscripttmpdir + mkdir -p $(dirname $1) + curl -f -sS $curlargs https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/$1 > $1 + if [ $? != 0 ]; then echo "'$*'" failed to download; return 1; fi + confluentpython $* + retcode=$? + echo "'$*' exited with code $retcode" + cd - > /dev/null + rm -rf $confluentscripttmpdir + unset confluentscripttmpdir + return $retcode +} + +run_remote_config() { + echo + set_confluent_vars + apiclient=/opt/confluent/bin/apiclient + if [ -f /etc/confluent/apiclient ]; then + apiclient=/etc/confluent/apiclient + fi + echo '---------------------------------------------------------------------------' + echo Requesting to run remote configuration for "'$*'" from $confluent_mgr under profile $confluent_profile + confluentpython $apiclient /confluent-api/self/remoteconfig/"$*" -d {} + confluentpython $apiclient /confluent-api/self/remoteconfig/status -w 204 + echo + echo 'Completed remote configuration' + echo '---------------------------------------------------------------------------' + return +} +#If invoked as a command, use the arguments to actually run a function +(return 0 2>/dev/null) || $1 "${@:2}" diff --git a/confluent_osdeploy/suse15/profiles/server/scripts/getinstalldisk b/confluent_osdeploy/suse15/profiles/server/scripts/getinstalldisk new file mode 100644 index 00000000..4af31c0b --- /dev/null +++ b/confluent_osdeploy/suse15/profiles/server/scripts/getinstalldisk @@ -0,0 +1,88 @@ +import subprocess +import os + +class DiskInfo(object): + def __init__(self, devname): + self.name = devname + self.wwn = None + self.path = None + self.model = '' + self.size = 0 + self.driver = None + self.mdcontainer = '' + devnode = '/dev/{0}'.format(devname) + qprop = subprocess.check_output( + ['udevadm', 'info', '--query=property', devnode]) + if not isinstance(qprop, str): + qprop = qprop.decode('utf8') + for prop in qprop.split('\n'): + if '=' not in prop: + continue + k, v = prop.split('=', 1) + if k == 'DEVTYPE' and v != 'disk': + raise Exception('Not a disk') + elif k == 'DM_NAME': + raise Exception('Device Mapper') + elif k == 'ID_MODEL': + self.model = v + elif k == 'DEVPATH': + self.path = v + elif k == 'ID_WWN': + self.wwn = v + elif k == 'MD_CONTAINER': + self.mdcontainer = v + attrs = subprocess.check_output(['udevadm', 'info', '-a', devnode]) + if not isinstance(attrs, str): + attrs = attrs.decode('utf8') + for attr in attrs.split('\n'): + if '==' not in attr: + continue + k, v = attr.split('==', 1) + k = k.strip() + if k == 'ATTRS{size}': + self.size = v.replace('"', '') + elif (k == 'DRIVERS' and not self.driver + and v not in ('"sd"', '""')): + self.driver = v.replace('"', '') + if not self.driver and 'imsm' not in self.mdcontainer: + raise Exception("No driver detected") + + @property + def priority(self): + if self.model.lower() in ('thinksystem_m.2_vd', 'thinksystem m.2', 'thinksystem_m.2'): + return 0 + if 'imsm' in self.mdcontainer: + return 1 + if self.driver == 'ahci': + return 2 + if self.driver.startswith('megaraid'): + return 3 + if self.driver.startswith('mpt'): + return 4 + return 99 + + def __repr__(self): + return repr({ + 'name': self.name, + 'path': self.path, + 'wwn': self.wwn, + 'driver': self.driver, + 'size': self.size, + 'model': self.model, + }) + + +def main(): + disks = [] + for disk in sorted(os.listdir('/sys/class/block')): + try: + disk = DiskInfo(disk) + disks.append(disk) + except Exception as e: + print("Skipping {0}: {1}".format(disk, str(e))) + nd = [x.name for x in sorted(disks, key=lambda x: x.priority)] + if nd: + open('/tmp/installdisk', 'w').write(nd[0]) + +if __name__ == '__main__': + main() diff --git a/confluent_osdeploy/suse15/profiles/server/scripts/post.custom b/confluent_osdeploy/suse15/profiles/server/scripts/post.custom new file mode 100644 index 00000000..a841271b --- /dev/null +++ b/confluent_osdeploy/suse15/profiles/server/scripts/post.custom @@ -0,0 +1,16 @@ +#!/bin/sh + +# This script runs at the end of install in the installed system +# but still under the installer kernel. + +# This is a good place to run most customizations that do not have any +# dependency upon the install target kernel being active. + +# If there are dependencies on the kernel (drivers or special filesystems) +# then firstboot.sh would be the script to customize. + +. /etc/confluent/functions + +# Examples: +# run_remote script.sh +# run_remote_python script.py diff --git a/confluent_osdeploy/suse15/profiles/server/scripts/post.sh b/confluent_osdeploy/suse15/profiles/server/scripts/post.sh new file mode 100644 index 00000000..be4e2d80 --- /dev/null +++ b/confluent_osdeploy/suse15/profiles/server/scripts/post.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +# This script runs at the end of install in the installed system +# but still under the installer kernel. + +# This is a good place to run most customizations that do not have any +# dependency upon the install target kernel being active. + +# If there are dependencies on the kernel (drivers or special filesystems) +# then firstboot.sh would be the script to customize. + +chmod 700 /var/log/confluent +exec >> /var/log/confluent/confluent-post.log +exec 2>> /var/log/confluent/confluent-post.log +chmod 600 /var/log/confluent/confluent-post.log +confluent_mgr=$(grep ^deploy_server /etc/confluent/confluent.deploycfg|awk '{print $2}') +confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|sed -e 's/^profile: //') +nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}') +confluent_apikey=$(cat /etc/confluent/confluent.apikey) + +chmod 700 /etc/confluent +chmod og-rwx /etc/confluent/* + + +export confluent_mgr confluent_profile nodename +. /etc/confluent/functions + +# This will induce server side processing of the syncfile contents if +# present +run_remote_python syncfileclient + +run_remote post.custom + +# Also, scripts may be placed into 'post.d', e.g. post.d/01-runfirst.sh, post.d/02-runsecond.sh +run_remote_parts post.d + +# Induce execution of remote configuration, e.g. ansible plays in ansible/post.d/ +run_remote_config post.d + +curl -X POST -d 'status: staged' -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" https://$confluent_mgr/confluent-api/self/updatestatus + diff --git a/confluent_osdeploy/suse15/profiles/server/scripts/pre.sh b/confluent_osdeploy/suse15/profiles/server/scripts/pre.sh new file mode 100644 index 00000000..54fea7c2 --- /dev/null +++ b/confluent_osdeploy/suse15/profiles/server/scripts/pre.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +# This script runs before the installer executes, and sets up ssh during install as well +# as rewriting the autoyast file with any substitutions prior to it being evaluated for real + +exec >> /tmp/confluent-pre.log +exec 2>> /tmp/confluent-pre.log +chmod 600 /tmp/confluent-pre.log +nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}') +rootpw=$(grep rootpassword: /etc/confluent/confluent.deploycfg|sed -e 's/^rootpassword: //') +if [ "$rootpw" = "null" ]; then + rootpw="!" +fi +cryptboot=$(grep encryptboot: /etc/confluent/confluent.deploycfg|sed -e 's/^encryptboot: //') +if [ "$cryptboot" != "" ] && [ "$cryptboot" != "none" ] && [ "$cryptboot" != "null" ]; then + echo "****Encrypted boot requested, but not implemented for this OS, halting install" > /dev/console + [ -f '/tmp/autoconsdev' ] && (echo "****Encryptod boot requested, but not implemented for this OS,halting install" >> $(cat /tmp/autoconsdev)) + while :; do sleep 86400; done +fi + +mkdir ~/.ssh +cat /ssh/*pubkey > ~/.ssh/authorized_keys 2>/dev/null + +ssh-keygen -A +for i in /etc/ssh/ssh_host*key.pub; do + certname=${i/.pub/-cert.pub} + curl -f -X POST -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $(cat /etc/confluent/confluent.apikey)" -d @$i https://$confluent_mgr/confluent-api/self/sshcert > $certname + echo HostKey ${i%.pub} >> /etc/ssh/sshd_config + echo HostCertificate $certname >> /etc/ssh/sshd_config +done +/usr/sbin/sshd +curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/functions > /tmp/functions +. /tmp/functions +ntpcfg="" +if grep ^ntpservers: /etc/confluent/confluent.deploycfg > /dev/null; then + echo '' > /tmp/ntp.cfg + sed -n '/^ntpservers:/,/^[^-]/p' /etc/confluent/confluent.deploycfg | sed 1d|sed '$d'| sed -e 's/^- /
/' -e 's!$!
!' >> /tmp/ntp.cfg + echo '' >> /tmp/ntp.cfg + ntpcfg=$(paste -sd '' /tmp/ntp.cfg) +fi +mdadm --assemble --scan +run_remote_python getinstalldisk +if grep ^md /tmp/installdisk > /dev/null; then + for md in /dev/disk/*/*; do + rmd=$(readlink $md) + if echo $rmd|grep $(cat /tmp/installdisk)$ > /dev/null; then + echo ${md#/dev/} > /tmp/installdisk + fi + done +fi +sed -e s'!'%%INSTDISK%%'!'/dev/$(cat /tmp/installdisk)'!' -e s'!'%%NODENAME%%'!'$nodename'!' -e 's!!'"$ntpcfg"'!' -e "s?%%ROOTPASSWORD%%?${rootpw}?" /tmp/profile/autoinst.xml > /tmp/profile/modified.xml +if grep append /tmp/bootloader.xml > /dev/null; then + sed -i 's@@'"$(tr -d '\n' < /tmp/bootloader.xml)"'@' /tmp/profile/modified.xml +fi +sed -i 's#root#root'"$(tr -d '\n' < /tmp/rootkeys.xml)"'#' /tmp/profile/modified.xml +sed -i 's@/hwclock>@/hwclock>'"$(tr -d '\n' < /tmp/timezone)"'@' /tmp/profile/modified.xml +sed -i 's@@'"$(tr -d '\n' < /tmp/pkgurl)"'@' /tmp/profile/modified.xml diff --git a/confluent_osdeploy/suse15/profiles/server/scripts/prechroot.sh b/confluent_osdeploy/suse15/profiles/server/scripts/prechroot.sh new file mode 100644 index 00000000..527703e4 --- /dev/null +++ b/confluent_osdeploy/suse15/profiles/server/scripts/prechroot.sh @@ -0,0 +1,33 @@ +#!/bin/sh + +# This script runs when install is finished, but while the installer +# is still running, with the to-be-booted system mounted in /mnt + +# carry over deployment configuration and api key for OS install action +confluent_mgr=$(grep ^deploy_server /etc/confluent/confluent.deploycfg|awk '{print $2}') +confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|sed -e 's/^profile: //') +nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}') +export confluent_mgr confluent_profile nodename +mkdir -p /mnt/etc/confluent +chmod 700 /mnt/etc/confluent +cp /tmp/functions /mnt/etc/confluent/ +. /tmp/functions +cp -a /etc/confluent/* /mnt/etc/confluent/ +cp -a /tls /mnt/etc/confluent/ +cp -a /tls/* /mnt/var/lib/ca-certificates/openssl +cp -a /tls/* /mnt/var/lib/ca-certificates/pem +cp -a /tls/*.pem /mnt/etc/pki/trust/anchors +cat /tls/*.pem > /mnt/etc/confluent/ca.pem +mkdir -p /mnt/opt/confluent/bin +cp /opt/confluent/bin/apiclient /mnt/opt/confluent/bin/ + +run_remote setupssh.sh + +echo Port 22 >> /etc/ssh/sshd_config +echo Port 2222 >> /etc/ssh/sshd_config +echo Match LocalPort 22 >> /etc/ssh/sshd_config +echo " ChrootDirectory /mnt" >> /etc/ssh/sshd_config +kill -HUP $(cat /run/sshd.pid) +mkdir -p /mnt/var/log/confluent +cp /tmp/confluent*log /mnt/var/log/confluent + diff --git a/confluent_osdeploy/suse15/profiles/server/scripts/setupssh.sh b/confluent_osdeploy/suse15/profiles/server/scripts/setupssh.sh new file mode 100644 index 00000000..181e225a --- /dev/null +++ b/confluent_osdeploy/suse15/profiles/server/scripts/setupssh.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# Carry over install-time ssh material into installed system +mkdir -p /mnt/root/.ssh/ +chmod 700 /mnt/root/.ssh/ +cp /root/.ssh/authorized_keys /mnt/root/.ssh/ +chmd 600 /mnt/root/.ssh/authorized_keys +cp /etc/ssh/*key* /mnt/etc/ssh/ +for i in /etc/ssh/*-cert.pub; do + echo HostCertificate $i >> /mnt/etc/ssh/sshd_config +done +for i in /ssh/*.ca; do + echo '@cert-authority *' $(cat $i) >> /mnt/etc/ssh/ssh_known_hosts +done +# Enable ~/.shosts, for the sake of root user, who is forbidden from using shosts.equiv +echo IgnoreRhosts no >> /mnt/etc/ssh/sshd_config +echo HostbasedAuthentication yes >> /mnt/etc/ssh/sshd_config +echo HostbasedUsesNameFromPacketOnly yes >> /mnt/etc/ssh/sshd_config +echo Host '*' >> /mnt/etc/ssh/ssh_config +echo " HostbasedAuthentication yes" >> /mnt/etc/ssh/ssh_config +echo " EnableSSHKeysign yes" >> /mnt/etc/ssh/ssh_config +# Limit the attempts of using host key. This prevents client from using 3 or 4 +# authentication attempts through host based attempts +echo " HostbasedKeyTypes *ed25519*" >> /mnt/etc/ssh/ssh_config + +# In SUSE platform, setuid for ssh-keysign is required for host based, +# and also must be opted into. +echo /usr/lib/ssh/ssh-keysign root:root 4711 >> /mnt/etc/permissions.local +chmod 4711 /mnt/usr/lib/ssh/ssh-keysign + +# Download list of nodes from confluent, and put it into shosts.equiv (for most users) and .shosts (for root) +curl -f -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $(cat /etc/confluent/confluent.apikey)" https://$confluent_mgr/confluent-api/self/nodelist > /tmp/allnodes +cp /tmp/allnodes /mnt/root/.shosts +cp /tmp/allnodes /mnt/etc/ssh/shosts.equiv + diff --git a/confluent_osdeploy/suse15/profiles/server/scripts/syncfileclient b/confluent_osdeploy/suse15/profiles/server/scripts/syncfileclient new file mode 100644 index 00000000..3fdd1f08 --- /dev/null +++ b/confluent_osdeploy/suse15/profiles/server/scripts/syncfileclient @@ -0,0 +1,272 @@ +#!/usr/bin/python +import importlib +import tempfile +import json +import os +import shutil +import pwd +import grp +from importlib.machinery import SourceFileLoader +try: + apiclient = SourceFileLoader('apiclient', '/opt/confluent/bin/apiclient').load_module() +except FileNotFoundError: + apiclient = SourceFileLoader('apiclient', '/etc/confluent/apiclient').load_module() + + +def partitionhostsline(line): + comment = '' + try: + cmdidx = line.index('#') + comment = line[cmdidx:] + line = line[:cmdidx].strip() + except ValueError: + pass + if not line: + return '', [], comment + ipaddr, names = line.split(maxsplit=1) + names = names.split() + return ipaddr, names, comment + +class HostMerger(object): + def __init__(self): + self.byip = {} + self.byname = {} + self.sourcelines = [] + self.targlines = [] + + def read_source(self, sourcefile): + with open(sourcefile, 'r') as hfile: + self.sourcelines = hfile.read().split('\n') + while not self.sourcelines[-1]: + self.sourcelines = self.sourcelines[:-1] + for x in range(len(self.sourcelines)): + line = self.sourcelines[x] + currip, names, comment = partitionhostsline(line) + if currip: + self.byip[currip] = x + for name in names: + self.byname[name] = x + + def read_target(self, targetfile): + with open(targetfile, 'r') as hfile: + lines = hfile.read().split('\n') + if not lines[-1]: + lines = lines[:-1] + for y in range(len(lines)): + line = lines[y] + currip, names, comment = partitionhostsline(line) + if currip in self.byip: + x = self.byip[currip] + if self.sourcelines[x] is None: + # have already consumed this enntry + continue + self.targlines.append(self.sourcelines[x]) + self.sourcelines[x] = None + continue + for name in names: + if name in self.byname: + x = self.byname[name] + if self.sourcelines[x] is None: + break + self.targlines.append(self.sourcelines[x]) + self.sourcelines[x] = None + break + else: + self.targlines.append(line) + + def write_out(self, targetfile): + while not self.targlines[-1]: + self.targlines = self.targlines[:-1] + if not self.targlines: + break + while not self.sourcelines[-1]: + self.sourcelines = self.sourcelines[:-1] + if not self.sourcelines: + break + with open(targetfile, 'w') as hosts: + for line in self.targlines: + hosts.write(line + '\n') + for line in self.sourcelines: + if line is not None: + hosts.write(line + '\n') + + +class CredMerger: + def __init__(self): + try: + with open('/etc/login.defs', 'r') as ldefs: + defs = ldefs.read().split('\n') + except FileNotFoundError: + defs = [] + lkup = {} + self.discardnames = {} + self.shadowednames = {} + for line in defs: + try: + line = line[:line.index('#')] + except ValueError: + pass + keyval = line.split() + if len(keyval) < 2: + continue + lkup[keyval[0]] = keyval[1] + self.uidmin = int(lkup.get('UID_MIN', 1000)) + self.uidmax = int(lkup.get('UID_MAX', 60000)) + self.gidmin = int(lkup.get('GID_MIN', 1000)) + self.gidmax = int(lkup.get('GID_MAX', 60000)) + self.shadowlines = None + + def read_passwd(self, source, targfile=False): + self.read_generic(source, self.uidmin, self.uidmax, targfile) + + def read_group(self, source, targfile=False): + self.read_generic(source, self.gidmin, self.gidmax, targfile) + + def read_generic(self, source, minid, maxid, targfile): + if targfile: + self.targdata = [] + else: + self.sourcedata = [] + with open(source, 'r') as inputfile: + for line in inputfile.read().split('\n'): + try: + name, _, uid, _ = line.split(':', 3) + uid = int(uid) + except ValueError: + continue + if targfile: + if uid < minid or uid > maxid: + self.targdata.append(line) + else: + self.discardnames[name] = 1 + else: + if name[0] in ('+', '#', '@'): + self.sourcedata.append(line) + elif uid >= minid and uid <= maxid: + self.sourcedata.append(line) + + def read_shadow(self, source): + self.shadowlines = [] + try: + with open(source, 'r') as inshadow: + for line in inshadow.read().split('\n'): + try: + name, _ = line.split(':' , 1) + except ValueError: + continue + if name in self.discardnames: + continue + self.shadowednames[name] = 1 + self.shadowlines.append(line) + except FileNotFoundError: + return + + def write_out(self, outfile): + with open(outfile, 'w') as targ: + for line in self.targdata: + targ.write(line + '\n') + for line in self.sourcedata: + targ.write(line + '\n') + if outfile == '/etc/passwd': + if self.shadowlines is None: + self.read_shadow('/etc/shadow') + with open('/etc/shadow', 'w') as shadout: + for line in self.shadowlines: + shadout.write(line + '\n') + for line in self.sourcedata: + name, _ = line.split(':', 1) + if name[0] in ('+', '#', '@'): + continue + if name in self.shadowednames: + continue + shadout.write(name + ':!:::::::\n') + if outfile == '/etc/group': + if self.shadowlines is None: + self.read_shadow('/etc/gshadow') + with open('/etc/gshadow', 'w') as shadout: + for line in self.shadowlines: + shadout.write(line + '\n') + for line in self.sourcedata: + name, _ = line.split(':' , 1) + if name in self.shadowednames: + continue + shadout.write(name + ':!::\n') + +def appendonce(basepath, filename): + with open(filename, 'rb') as filehdl: + thedata = filehdl.read() + targname = filename.replace(basepath, '') + try: + with open(targname, 'rb') as filehdl: + targdata = filehdl.read() + except IOError: + targdata = b'' + if thedata in targdata: + return + with open(targname, 'ab') as targhdl: + targhdl.write(thedata) + +def synchronize(): + tmpdir = tempfile.mkdtemp() + appendoncedir = tempfile.mkdtemp() + try: + ac = apiclient.HTTPSClient() + data = json.dumps({'merge': tmpdir, 'appendonce': appendoncedir}) + status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data) + if status == 202: + lastrsp = '' + while status != 204: + status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles') + if not isinstance(rsp, str): + rsp = rsp.decode('utf8') + if status == 200: + lastrsp = rsp + pendpasswd = os.path.join(tmpdir, 'etc/passwd') + if os.path.exists(pendpasswd): + cm = CredMerger() + cm.read_passwd(pendpasswd, targfile=False) + cm.read_passwd('/etc/passwd', targfile=True) + cm.write_out('/etc/passwd') + pendgroup = os.path.join(tmpdir, 'etc/group') + if os.path.exists(pendgroup): + cm = CredMerger() + cm.read_group(pendgroup, targfile=False) + cm.read_group('/etc/group', targfile=True) + cm.write_out('/etc/group') + pendhosts = os.path.join(tmpdir, 'etc/hosts') + if os.path.exists(pendhosts): + cm = HostMerger() + cm.read_source(pendhosts) + cm.read_target('/etc/hosts') + cm.write_out('/etc/hosts') + for dirn in os.walk(appendoncedir): + for filen in dirn[2]: + appendonce(appendoncedir, os.path.join(dirn[0], filen)) + if lastrsp: + lastrsp = json.loads(lastrsp) + opts = lastrsp.get('options', {}) + for fname in opts: + uid = -1 + gid = -1 + for opt in opts[fname]: + if opt == 'owner': + try: + uid = pwd.getpwnam(opts[fname][opt]['name']).pw_uid + except KeyError: + uid = opts[fname][opt]['id'] + elif opt == 'group': + try: + gid = grp.getgrnam(opts[fname][opt]['name']).gr_gid + except KeyError: + gid = opts[fname][opt]['id'] + elif opt == 'permissions': + os.chmod(fname, int(opts[fname][opt], 8)) + if uid != -1 or gid != -1: + os.chown(fname, uid, gid) + finally: + shutil.rmtree(tmpdir) + shutil.rmtree(appendoncedir) + + +if __name__ == '__main__': + synchronize()