diff --git a/confluent_osdeploy/ubuntu18.04/initramfs/lib/debian-installer-startup.d/S25confluentinit b/confluent_osdeploy/ubuntu18.04/initramfs/lib/debian-installer-startup.d/S25confluentinit index 44156d36..cac4fdac 100644 --- a/confluent_osdeploy/ubuntu18.04/initramfs/lib/debian-installer-startup.d/S25confluentinit +++ b/confluent_osdeploy/ubuntu18.04/initramfs/lib/debian-installer-startup.d/S25confluentinit @@ -27,6 +27,7 @@ for i in /sys/class/net/*; do ip link set $(basename $i) up done cp -a /tls/* /etc/ssl/certs/ +mkdir -p /etc/confluent if [ -e /dev/disk/by-label/CNFLNT_IDNT ]; then tmnt=$(mktemp -d) tcfg=$(mktemp) @@ -34,6 +35,7 @@ if [ -e /dev/disk/by-label/CNFLNT_IDNT ]; then cd $tmnt deploysrvs=$(sed -n '/^deploy_servers:/,/^[^-]/p' cnflnt.yml |grep ^-|sed -e 's/^- //'|grep -v :) nodename=$(grep ^nodename: cnflnt.yml|cut -f 2 -d ' ') + echo NODENAME: $nodename > /etc/confluent/confluent.info sed -n '/^net_cfgs:/,/^[^- ]/{/^[^- ]/!p}' cnflnt.yml |sed -n '/^-/,/^-/{/^-/!p}'| sed -e 's/^[- ]*//'> $tcfg autoconfigmethod=$(grep ^ipv4_method: $tcfg) autoconfigmethod=${autoconfigmethod#ipv4_method: } @@ -67,8 +69,8 @@ if [ -e /dev/disk/by-label/CNFLNT_IDNT ]; then passcrypt=/tmp/cnflntcryptfile hmacfile=/tmp/cnflnthmacfile echo -n $(grep ^apitoken: cnflnt.yml|cut -d ' ' -f 2) > $hmackeyfile + /opt/confluent/bin/genpasshmac $passfile $passcrypt $hmacfile $hmackeyfile wget --header="CONFLUENT_NODENAME: $nodename" --header="CONFLUENT_CRYPTHMAC: $(cat $hmacfile)" --post-file=$passcrypt https://$mgr/confluent-api/self/registerapikey -O - --quiet - rm registerapikey cp $passfile /etc/confluent/confluent.apikey nic=$NICGUESS else @@ -76,6 +78,7 @@ else dhcpid=$(mktemp) mkdir -p /etc/confluent cp /tls/* /etc/ssl/certs/ + cat /tls/*.pem >> /etc/confluent/ca.pem for nic in $(ip link | grep mtu|grep -v LOOPBACK|cut -d: -f 2|sed -e 's/ //'); do ip link set $nic up done @@ -98,6 +101,7 @@ wget --header="CONFLUENT_NODENAME: $nodename" --header="CONFLUENT_APIKEY: $apike cd - predir=$(mktemp -d) cd $predir +cp /etc/confluent/deploycfg /etc/confluent/confluent.deploycfg profile=$(grep ^profile: /etc/confluent/deploycfg|cut -d ' ' -f 2) namesrvs=$(sed -n '/^nameservers:/,/^[^-]/p' /etc/confluent/deploycfg|grep ^- | cut -d ' ' -f 2|sed -e 's/ //') for namesrv in "$namesrvs"; do @@ -114,7 +118,7 @@ setdebopt time/zone $(grep ^timezone: /etc/confluent/deploycfg|cut -d ' ' -f 2|s ntpsrvs=$(sed -n '/^ntpservers:/,/^[^-]/p' /etc/confluent/deploycfg|grep ^- | cut -d ' ' -f 2|sed -e 's/ //') for ntpsrv in "$ntpsrvs"; do setdebopt clock-setup/ntp true boolean - setdebopt clock-setup/ntep-server $ntpsrv string + setdebopt clock-setup/ntp-server $ntpsrv string done setdebopt debian-installer/locale $(grep ^locale: /etc/confluent/deploycfg|cut -d ' ' -f 2) select domainname=$(grep ^dnsdomain: /etc/confluent/deploycfg|cut -d ' ' -f 2) @@ -127,14 +131,28 @@ fi wget https://$mgr/confluent-public/os/$profile/scripts/pre.sh chmod u+x pre.sh +wget https://$mgr/confluent-public/os/$profile/scripts/prechroot.sh +chmod u+x prechroot.sh +wget https://$mgr/confluent-public/os/$profile/scripts/post.sh +chmod u+x post.sh wget https://$mgr/confluent-public/os/$profile/preseed.cfg cat preseed.cfg >> /preseed.cfg +echo $mgr > /etc/confluent/deployer setdebopt auto-install/enable true boolean setdebopt partman/early_command $predir/pre.sh string +setdebopt preseed/late_command $predir/prechroot.sh string +mv $predir/post.sh /tmp/ cd - ip -4 a flush dev $nic setdebopt netcfg/choose_interface $nic select setdebopt netcfg/get_hostname $nodename string +setdebopt mirror/protocol https string +setdebopt mirror/country manual string +setdebopt mirror/https/hostname $mgr string +setdebopt mirror/https/directory /confluent-public/os/$profile/distribution string +setdebopt mirror/protocol https string +setdebopt mirror/https/proxy "" string +setdebopt apt-setup/security_host $mgr string if [ ! -e /dev/disk/by-label/CNFLNT_IDNT ]; then v4cfgmeth=$(grep ipv4_method: /etc/confluent/deploycfg |cut -d: -f 2|sed -e 's/ //') if [ "$v4cfgmeth" = "static" ]; then diff --git a/confluent_osdeploy/ubuntu18.04/profiles/default/initprofile.sh b/confluent_osdeploy/ubuntu18.04/profiles/default/initprofile.sh new file mode 100644 index 00000000..a306f6e1 --- /dev/null +++ b/confluent_osdeploy/ubuntu18.04/profiles/default/initprofile.sh @@ -0,0 +1,7 @@ +#!/bin/sh +sed -i 's/label: ubuntu/label: Ubuntu/' $2/profile.yaml && \ +ln -s $1/install/hwe-netboot/ubuntu-installer/amd64/linux $2/boot/kernel && \ +ln -s $1/install/hwe-netboot/ubuntu-installer/amd64/initrd.gz $2/boot/initramfs/distribution && \ +mkdir -p $2/boot/efi/boot && \ +ln -s $1/EFI/BOOT/* $2/boot/efi/boot + diff --git a/confluent_osdeploy/ubuntu18.04/profiles/default/preseed.cfg b/confluent_osdeploy/ubuntu18.04/profiles/default/preseed.cfg new file mode 100644 index 00000000..5454c1eb --- /dev/null +++ b/confluent_osdeploy/ubuntu18.04/profiles/default/preseed.cfg @@ -0,0 +1,24 @@ +d-i anna/choose_modules string openssh-server-udeb +d-i partman-auto/method string regular +d-i partman-lvm/device_remove_lvm boolean true +d-i partman-md/device_remove_md boolean true +d-i partman-auto/expert_recipe_file string /tmp/partitionfile +d-i partman/confirm_write_new_label boolean true +d-i partman/choose_partition select finish +d-i partman/confirm boolean true +d-i partman/confirm_nooverwrite boolean true +d-i preseed/late_command string sleep 86400 +d-i partman/early_command string sleep 86400 +d-i passwd/make-user boolean false +d-i clock-setup/utc boolean true +d-i apt-setup/multiverse boolean false +d-i apt-setup/universe boolean false +d-i apt-setup/backports boolean false +d-i apt-setup/updates boolean false +d-i grub-installer/only_debian boolean true +tasksel tasksel/first multiselect standard +d-i pkgsel/include string openssh-server curl +d-i pkgsel/update-policy select none +d-i pkgsel/updatedb boolean false + + diff --git a/confluent_osdeploy/ubuntu18.04/profiles/default/scripts/functions b/confluent_osdeploy/ubuntu18.04/profiles/default/scripts/functions new file mode 100644 index 00000000..026697b7 --- /dev/null +++ b/confluent_osdeploy/ubuntu18.04/profiles/default/scripts/functions @@ -0,0 +1,208 @@ +#!/bin/bash +function test_mgr() { + whost=$1 + if [[ "$whost" == *:* ]] && [[ "$whost" != *[* ]] ; then + whost="[$whost]" + fi + if curl -gs https://${whost}/confluent-api/ > /dev/null; then + return 0 + fi + return 1 +} + +function confluentpython() { + if [ -x /usr/libexec/platform-python ]; then + /usr/libexec/platform-python $* + elif [ -x /usr/bin/python3 ]; then + /usr/bin/python3 $* + elif [ -x /usr/bin/python ]; then + /usr/bin/python $* + elif [ -x /usr/bin/python2 ]; then + /usr/bin/python2 $* + fi +} + +function set_confluent_vars() { + if [ -z "$nodename" ]; then + nodename=$(grep ^NODENAME: /etc/confluent/confluent.info | awk '{print $2}') + fi + if [[ "$confluent_mgr" == *"%"* ]]; then + confluent_mgr="" + fi + if [ -z "$confluent_mgr" ]; then + confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //') + if ! test_mgr $confluent_mgr; then + confluent_mgr=$(grep ^deploy_server_v6: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //') + if [[ "$confluent_mgr" = *":"* ]]; then + confluent_mgr="[$confluent_mgr]" + fi + fi + if ! test_mgr $confluent_mgr; then + BESTMGRS=$(grep ^EXTMGRINFO: /etc/confluent/confluent.info | grep '|1$' | sed -e 's/EXTMGRINFO: //' -e 's/|.*//') + OKMGRS=$(grep ^EXTMGRINFO: /etc/confluent/confluent.info | grep '|0$' | sed -e 's/EXTMGRINFO: //' -e 's/|.*//') + for confluent_mgr in $BESTMGRS $OKMGRS; do + if [[ $confluent_mgr == *":"* ]]; then + confluent_mgr="[$confluent_mgr]" + fi + if test_mgr $confluent_mgr; then + break + fi + done + fi + fi + if [ -z "$confluent_profile" ]; then + confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //') + fi +} + +fetch_remote() { + curlargs="" + if [ -f /etc/confluent/ca.pem ]; then + curlargs=" --cacert /etc/confluent/ca.pem" + fi + set_confluent_vars + mkdir -p $(dirname $1) + whost=$confluent_mgr + if [[ "$whost" == *:* ]] && [[ "$whost" != *[* ]] ; then + whost="[$whost]" + fi + curl -gf -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1 + if [ $? != 0 ]; then echo $1 failed to download; return 1; fi +} + +source_remote_parts() { + confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX) + apiclient=/opt/confluent/bin/apiclient + if [ -f /etc/confluent/apiclient ]; then + apiclient=/etc/confluent/apiclient + fi + scriptlist=$(confluentpython $apiclient /confluent-api/self/scriptlist/$1|sed -e 's/^- //') + for script in $scriptlist; do + source_remote $1/$script + done + rm -rf $confluentscripttmpdir + unset confluentscripttmpdir +} + +run_remote_parts() { + confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX) + apiclient=/opt/confluent/bin/apiclient + if [ -f /etc/confluent/apiclient ]; then + apiclient=/etc/confluent/apiclient + fi + scriptlist=$(confluentpython $apiclient /confluent-api/self/scriptlist/$1|sed -e 's/^- //') + for script in $scriptlist; do + run_remote $1/$script + done + rm -rf $confluentscripttmpdir + unset confluentscripttmpdir +} + +source_remote() { + set_confluent_vars + unsettmpdir=0 + echo + echo '---------------------------------------------------------------------------' + echo Sourcing $1 from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/ + if [ -z "$confluentscripttmpdir" ]; then + confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX) + unsettmpdir=1 + fi + echo Sourcing from $confluentscripttmpdir + cd $confluentscripttmpdir + fetch_remote $1 + if [ $? != 0 ]; then echo $1 failed to download; return 1; fi + chmod +x $1 + cmd=$1 + shift + source ./$cmd + cd - > /dev/null + if [ "$unsettmpdir" = 1 ]; then + rm -rf $confluentscripttmpdir + unset confluentscripttmpdir + unsettmpdir=0 + fi + rm -rf $confluentscripttmpdir + return $retcode +} + +run_remote() { + requestedcmd="'$*'" + unsettmpdir=0 + set_confluent_vars + echo + echo '---------------------------------------------------------------------------' + echo Running $requestedcmd from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/ + if [ -z "$confluentscripttmpdir" ]; then + confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX) + unsettmpdir=1 + fi + echo Executing in $confluentscripttmpdir + cd $confluentscripttmpdir + fetch_remote $1 + if [ $? != 0 ]; then echo $requestedcmd failed to download; return 1; fi + chmod +x $1 + cmd=$1 + if [ -x /usr/bin/chcon ]; then + chcon system_u:object_r:bin_t:s0 $cmd + fi + shift + ./$cmd $* + retcode=$? + if [ $retcode -ne 0 ]; then + echo "$requestedcmd exited with code $retcode" + fi + cd - > /dev/null + if [ "$unsettmpdir" = 1 ]; then + rm -rf $confluentscripttmpdir + unset confluentscripttmpdir + unsettmpdir=0 + fi + return $retcode +} + +run_remote_python() { + echo + set_confluent_vars + if [ -f /etc/confluent/ca.pem ]; then + curlargs=" --cacert /etc/confluent/ca.pem" + fi + echo '---------------------------------------------------------------------------' + echo Running python script "'$*'" from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/ + confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX) + echo Executing in $confluentscripttmpdir + cd $confluentscripttmpdir + mkdir -p $(dirname $1) + whost=$confluent_mgr + if [[ "$whost" == *:* ]] && [[ "$whost" != *[* ]] ; then + whost="[$whost]" + fi + curl -gf -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1 + if [ $? != 0 ]; then echo "'$*'" failed to download; return 1; fi + confluentpython $* + retcode=$? + echo "'$*' exited with code $retcode" + cd - > /dev/null + rm -rf $confluentscripttmpdir + unset confluentscripttmpdir + return $retcode +} + +run_remote_config() { + echo + set_confluent_vars + apiclient=/opt/confluent/bin/apiclient + if [ -f /etc/confluent/apiclient ]; then + apiclient=/etc/confluent/apiclient + fi + echo '---------------------------------------------------------------------------' + echo Requesting to run remote configuration for "'$*'" from $confluent_mgr under profile $confluent_profile + confluentpython $apiclient /confluent-api/self/remoteconfig/"$*" -d {} + confluentpython $apiclient /confluent-api/self/remoteconfig/status -w 204 + echo + echo 'Completed remote configuration' + echo '---------------------------------------------------------------------------' + return +} +#If invoked as a command, use the arguments to actually run a function +(return 0 2>/dev/null) || $1 "${@:2}" diff --git a/confluent_osdeploy/ubuntu18.04/profiles/default/scripts/post.sh b/confluent_osdeploy/ubuntu18.04/profiles/default/scripts/post.sh new file mode 100755 index 00000000..5c3b032e --- /dev/null +++ b/confluent_osdeploy/ubuntu18.04/profiles/default/scripts/post.sh @@ -0,0 +1,75 @@ +#!/bin/bash +mkdir /run/sshd +mkdir /root/.ssh +cat /tmp/ssh/*pubkey >> /root/.ssh/authorized_keys +cat /tmp/ssh/*.ca | sed -e s/^/'@cert-authority * '/ >> /etc/ssh/ssh_known_hosts +chmod 700 /etc/confluent +chmod go-rwx /etc/confluent/* +for pubkey in /etc/ssh/ssh_host*key.pub; do + certfile=${pubkey/.pub/-cert.pub} + privfile=${pubkey%.pub} + python3 /opt/confluent/bin/apiclient /confluent-api/self/sshcert $pubkey > $certfile + if [ -s $certfile ]; then + if ! grep $certfile /etc/ssh/sshd_config; then + echo HostCertificate $certfile >> /etc/ssh/sshd_config + fi + fi + if ! grep $privfile /etc/ssh/sshd_config > /dev/null; then + echo HostKey $privfile >> /etc/ssh/sshd_config + fi +done +sshconf=/etc/ssh/ssh_config +if [ -d /etc/ssh/ssh_config.d/ ]; then + sshconf=/etc/ssh/ssh_config.d/01-confluent.conf +fi +echo 'Host *' >> $sshconf +echo ' HostbasedAuthentication yes' >> $sshconf +echo ' EnableSSHKeysign yes' >> $sshconf +echo ' HostbasedKeyTypes *ed25519*' >> $sshconf +confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg | awk '{print $2}') +python3 /opt/confluent/bin/apiclient /confluent-public/os/$confluent_profile/scripts/firstboot.sh > /etc/confluent/firstboot.sh +python3 /opt/confluent/bin/apiclient /confluent-public/os/$confluent_profile/scripts/functions > /etc/confluent/functions +chmod +x /etc/confluent/firstboot.sh +source /etc/confluent/functions +python3 /opt/confluent/bin/apiclient /confluent-api/self/nodelist | sed -e s/'^- //' > /tmp/allnodes +cp /tmp/allnodes /root/.shosts +cp /tmp/allnodes /etc/ssh/shosts.equiv +if grep ^ntpservers: /etc/confluent/confluent.deploycfg > /dev/null; then + ntps=$(sed -n '/^ntpservers:/,/^[^-]/p' /etc/confluent/confluent.deploycfg|sed 1d|sed '$d' | sed -e 's/^- //' | paste -sd ' ') + sed -i "s/#NTP=/NTP=$ntps/" /etc/systemd/timesyncd.conf +fi +textcons=$(grep ^textconsole: /etc/confluent/confluent.deploycfg |awk '{print $2}') +updategrub=0 +if [ "$textcons" = "true" ] && ! grep console= /proc/cmdline > /dev/null; then + cons="" + if [ -f /tmp/autocons.info ]; then + cons=$(cat /tmp/autocons.info) + fi + if [ ! -z "$cons" ]; then + sed -i 's/GRUB_CMDLINE_LINUX="\([^"]*\)"/GRUB_CMDLINE_LINUX="\1 console='${cons#/dev/}'"/' /etc/default/grub + updategrub=1 + fi +fi +kargs=$(python3 /opt/confluent/bin/apiclient /confluent-public/os/$confluent_profile/profile.yaml | grep ^installedargs: | sed -e 's/#.*//') +if [ ! -z "$kargs" ]; then + sed -i 's/GRUB_CMDLINE_LINUX="\([^"]*\)"/GRUB_CMDLINE_LINUX="\1 '"${kargs}"'"/' /etc/default/grub +fi + +if [ 1 = $updategrub ]; then + update-grub +fi + +if [ -e /sys/firmware/efi ]; then + bootnum=$(efibootmgr | grep ubuntu | sed -e 's/ .*//' -e 's/\*//' -e s/Boot//) + if [ ! -z "$bootnum" ]; then + currboot=$(efibootmgr | grep ^BootOrder: | awk '{print $2}') + nextboot=$(echo $currboot| awk -F, '{print $1}') + [ "$nextboot" = "$bootnum" ] || efibootmgr -o $bootnum,$currboot + efibootmgr -D + fi +fi +run_remote_python syncfileclient +run_remote_parts post.d +run_remote_config post + + diff --git a/confluent_osdeploy/ubuntu18.04/profiles/default/scripts/pre.sh b/confluent_osdeploy/ubuntu18.04/profiles/default/scripts/pre.sh new file mode 100755 index 00000000..be179c27 --- /dev/null +++ b/confluent_osdeploy/ubuntu18.04/profiles/default/scripts/pre.sh @@ -0,0 +1,43 @@ +#!/bin/sh +## Use the following option to add additional boot parameters for the +## installed system (if supported by the bootloader installer). +## Note: options passed to the installer will be added automatically. +#d-i debian-installer/add-kernel-opts string [from profile.yaml] +deploycfg=/etc/confluent/confluent.deploycfg +mgr=$(cat /etc/confluent/deployer) + +cryptboot=$(grep encryptboot: $deploycfg|sed -e 's/^encryptboot: //') +if [ "$cryptboot" != "" ] && [ "$cryptboot" != "none" ] && [ "$cryptboot" != "null" ]; then + echo "****Encrypted boot requested, but not implemented for this OS, halting install" > /dev/console + [ -f '/tmp/autoconsdev' ] && (echo "****Encryptod boot requested, but not implemented for this OS,halting install" >> $(cat /tmp/autoconsdev)) + while :; do sleep 86400; done +fi +cat > /usr/lib/live-installer.d/confluent-certs << EOF +#!/bin/sh +cp /tls/* /target/etc/ssl/certs/ +cat /tls/*.pem >> /target/etc/ssl/certs/ca-certificates.crt +EOF +chmod a+x /usr/lib/live-installer.d/confluent-certs +mkdir -p /.ssh/ +cat /ssh/*pubkey > /.ssh/authorized_keys +mkdir -p /etc/ssh +nodename=$(grep ^NODENAME: /etc/confluent/confluent.info|cut -d ' ' -f 2) +apikey=$(cat /etc/confluent/confluent.apikey) +ssh-keygen -A +for pubkey in /etc/ssh/ssh_host*key.pub; do + certfile=$(echo $pubkey | sed -e s/.pub/-cert.pub/) + keyfile=${pubkey%.pub} + wget --header="CONFLUENT_NODENAME: $nodename" --header="CONFLUENT_APIKEY: $apikey" --post-file=$pubkey https://$mgr/confluent-api/self/sshcert -O $certfile --quiet + echo HostKey $keyfile >> /etc/ssh/sshd_config + echo HostCertificate $certfile >> /etc/ssh/sshd_config +done +echo HostbasedAuthentication yes >> /etc/ssh/sshd_config +echo HostbasedUsesNameFromPacketOnly yes >> /etc/ssh/sshd_config +echo IgnoreRhosts no >> /etc/ssh/sshd_config +echo sshd:x:1:1::/run/sshd:/bin/false >> /etc/passwd +/usr/sbin/sshd +wget --header="CONFLUENT_NODENAME: $nodename" --header="CONFLUENT_APIKEY: $apikey" https://$mgr/confluent-api/self/nodelist -O /tmp/allnodes --quiet +#kill -HUP $(ps | grep -v grep | grep /usr/sbin/sshd | sed -e 's/^ *//'|cut -d ' ' -f 1) +#curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/getinstalldisk > /tmp/getinstalldisk +#python3 /tmp/getinstalldisk +#sed -i s!%%INSTALLDISK%%!/dev/$(cat /tmp/installdisk)! /autoinstall.yaml diff --git a/confluent_osdeploy/ubuntu18.04/profiles/default/scripts/prechroot.sh b/confluent_osdeploy/ubuntu18.04/profiles/default/scripts/prechroot.sh new file mode 100644 index 00000000..3f5f508b --- /dev/null +++ b/confluent_osdeploy/ubuntu18.04/profiles/default/scripts/prechroot.sh @@ -0,0 +1,13 @@ +#!/bin/sh +mount -o bind /sys /target/sys +mount -o bind /dev /target/dev +mount -o bind /dev/pts /target/dev/pts +mount -o bind /proc /target/proc +mount -o bind /dev/pts /target/dev/pts +mount -o bind /run /target/run +cp -a /etc/confluent /target/etc/confluent +cp -a /opt/confluent /target/opt/confluent +mv /tmp/post.sh /target/tmp/ +cp -a /ssh /tls /target/tmp +cat /tls/*.pem >> /target/etc/confluent/ca.pem +chroot /target bash /tmp/post.sh