2
0
mirror of https://github.com/xcat2/confluent.git synced 2025-08-26 13:10:38 +00:00

Merge branch 'lenovo:master' into master

This commit is contained in:
zhougj4
2021-05-24 10:46:50 +02:00
20 changed files with 653 additions and 23 deletions

View File

@@ -159,7 +159,7 @@ def parse_config_line(arguments, single=False):
if setmode is None:
setmode = True
if setmode != True:
bailout('Cannot do set and query in same command')
bailout('Cannot do set and query in same command: Query detected but "{0}" appears to be set'.format(param))
if '=' in param:
key, _, value = param.partition('=')
_assign_value()
@@ -171,7 +171,7 @@ def parse_config_line(arguments, single=False):
if setmode is None:
setmode = False
if setmode != False:
bailout('Cannot do set and query in same command')
bailout('Cannot do set and query in same command: Set mode detected but "{0}" appears to be a query'.format(param))
if '.' not in param:
if param == 'bmc':
printallbmc = True

View File

@@ -25,7 +25,7 @@ cd utils
make all
cp copernicus clortho autocons ../opt/confluent/bin
cd ..
for os in rhvh4 el7 genesis el8 suse15 ubuntu20.04; do
for os in rhvh4 el7 genesis el8 suse15 ubuntu20.04 coreos; do
mkdir ${os}out
cd ${os}out
cp -a ../opt .
@@ -48,7 +48,7 @@ cp -a esxi7out esxi6out
cp -a esxi7 esxi6
%install
for os in rhvh4 el7 el8 genesis suse15 ubuntu20.04 esxi6 esxi7; do
for os in rhvh4 el7 el8 genesis suse15 ubuntu20.04 esxi6 esxi7 coreos; do
mkdir -p %{buildroot}/opt/confluent/lib/osdeploy/$os/initramfs
mkdir -p %{buildroot}/opt/confluent/lib/osdeploy/$os/profiles
cp ${os}out/addons.* %{buildroot}/opt/confluent/lib/osdeploy/$os/initramfs

View File

@@ -0,0 +1 @@
/usr/lib/systemd/system/confluent-rootfs.service

View File

@@ -0,0 +1,106 @@
#!/bin/bash
TRIES=0
oum=$(umask)
umask 0077
mkdir -p /etc/confluent
echo -n > /etc/confluent/confluent.info
umask $oum
cd /sys/class/net
while ! grep ^EXTMGRINFO: /etc/confluent/confluent.info | awk -F'|' '{print $3}' | grep 1 >& /dev/null && [ "$TRIES" -lt 60 ]; do
TRIES=$((TRIES + 1))
for currif in *; do
ip link set $currif up
done
/opt/confluent/bin/copernicus -t > /etc/confluent/confluent.info
done
cd /
grep ^EXTMGRINFO: /etc/confluent/confluent.info || return 0 # Do absolutely nothing if no data at all yet
echo -n "" > /tmp/confluent.initq
# restart cmdline
echo -n "" > /etc/cmdline.d/01-confluent.conf
nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
cat /tls/*.pem > /etc/confluent/ca.pem
confluent_mgr=$(grep ^MANAGER: /etc/confluent/confluent.info|head -n 1 | awk '{print $2}')
if [[ $confluent_mgr == *%* ]]; then
echo $confluent_mgr | awk -F% '{print $2}' > /tmp/confluent.ifidx
ifidx=$(cat /tmp/confluent.ifidx)
ifname=$(ip link |grep ^$ifidx:|awk '{print $2}')
ifname=${ifname%:}
fi
needseal=1
oldumask=$(umask)
umask 0077
while [ -z "$confluent_apikey" ]; do
/opt/confluent/bin/clortho $nodename $confluent_mgr > /etc/confluent/confluent.apikey
if grep ^SEALED: /etc/confluent/confluent.apikey > /dev/null; then
needseal=0
sed -e s/^SEALED:// /etc/confluent/confluent.apikey | clevis-decrypt-tpm2 > /etc/confluent/confluent.apikey.decrypt
mv /etc/confluent/confluent.apikey.decrypt /etc/confluent/confluent.apikey
fi
confluent_apikey=$(cat /etc/confluent/confluent.apikey)
if [ -z "$confluent_apikey" ]; then
echo "Unable to acquire node api key, no TPM2 sealed nor fresh token available, retrying..."
sleep 10
fi
done
if [[ $confluent_mgr == *:* ]]; then
confluent_mgr="[$confluent_mgr]"
fi
if [ $needseal == 1 ]; then
sealed=$(echo $confluent_apikey | clevis-encrypt-tpm2 {})
if [ ! -z "$sealed" ]; then
curl -sf -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" -d $sealed https://$confluent_mgr/confluent-api/self/saveapikey
fi
fi
curl -sf -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" https://$confluent_mgr/confluent-api/self/deploycfg > /etc/confluent/confluent.deploycfg
umask $oldumask
autoconfigmethod=$(grep ipv4_method /etc/confluent/confluent.deploycfg)
autoconfigmethod=${autoconfigmethod#ipv4_method: }
confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg)
confluent_profile=${confluent_profile#profile: }
if [ "$autoconfigmethod" = "dhcp" ]; then
echo ip=$ifname:dhcp >> /etc/cmdline.d/01-confluent.conf
else
v4addr=$(grep ^ipv4_address: /etc/confluent/confluent.deploycfg)
v4addr=${v4addr#ipv4_address: }
v4gw=$(grep ^ipv4_gateway: /etc/confluent/confluent.deploycfg)
v4gw=${v4gw#ipv4_gateway: }
if [ "$v4gw" = "null" ]; then
v4gw=""
fi
v4nm=$(grep ipv4_netmask: /etc/confluent/confluent.deploycfg)
v4nm=${v4nm#ipv4_netmask: }
echo ip=$v4addr::$v4gw:$v4nm:$hostname:$ifname:none >> /etc/cmdline.d/01-confluent.conf
fi
nameserversec=0
while read -r entry; do
if [ $nameserversec = 1 ]; then
if [[ $entry == "-"* ]] && [[ $entry != "- ''" ]]; then
echo nameserver=${entry#- } >> /etc/cmdline.d/01-confluent.conf
continue
fi
fi
nameserversec=0
if [ "${entry%:*}" = "nameservers" ]; then
nameserversec=1
continue
fi
done < /etc/confluent/confluent.deploycfg
if [ -e /lib/nm-lib.sh ]; then
. /lib/nm-lib.sh
nm_generate_connections
if [[ "$ifname" == ib* ]]; then
sed -i s/type=ethernet/type=infiniband/ /run/NetworkManager/system-connections/$ifname.nmconnection
if ! grep '\[infiniband\]' /run/NetworkManager/system-connections/$ifname.nmconnection > /dev/null; then
echo >> /run/NetworkManager/system-connections/$ifname.nmconnection
echo '[infiniband]' >> /run/NetworkManager/system-connections/$ifname.nmconnection
echo transport-mode=datagram >> /run/NetworkManager/system-connections/$ifname.nmconnection
fi
fi
fi
cat /proc/cmdline /etc/cmdline.d/01-confluent.conf | tr '\n' ' ' > /run/fakecmdline
mount -o bind /run/fakecmdline /proc/cmdline
curl -sf https://$confluent_mgr/confluent-public/os/$confluent_profile/rootfs.img | rdcore stream-hash /etc/coreos-live-want-rootfs | bsdtar -xf - -C /

View File

@@ -0,0 +1,10 @@
cat /tls/*.0 >> /etc/pki/tls/certs/ca-bundle.crt
if ! grep console= /proc/cmdline >& /dev/null; then
autocons=$(/opt/confluent/bin/autocons)
if [ -n "$autocons" ]; then
echo console=$autocons |sed -e 's!/dev/!!' >> /tmp/01-autocons.conf
autocons=${autocons%,*}
echo $autocons > /tmp/01-autocons.devnode
echo "Detected firmware specified console at $(cat /tmp/01-autocons.conf)" > $autocons
fi
fi

View File

@@ -0,0 +1,28 @@
#!/bin/sh
# if ignition configuration is driving, defer to it
if [ ! -f /config.ign ]; then
nodename=$(grep ^NODENAME: /etc/confluent/confluent.info | awk '{print $2}')
confluent_mgr=$(grep ^MANAGER: /etc/confluent/confluent.info| head -n 1| awk '{print $2}' | sed -e s/%/%25/)
if [[ $confluent_mgr = *:* ]]; then
confluent_mgr=[$confluent_mgr]
fi
rootpassword=$(grep ^rootpassword: /etc/confluent/confluent.deploycfg)
rootpassword=${rootpassword#rootpassword: }
if [ "$rootpassword" = "null" ]; then
rootpassword=""
fi
if [ ! -z "$rootpassword" ]; then
sed -i "s@root:[^:]*:@root:$rootpassword:@" /sysroot/etc/shadow
fi
mount -o bind /dev /sysroot/dev
chroot /sysroot ssh-keygen -A
umount /sysroot/dev
for i in /sysroot/etc/ssh/ssh_host*key.pub; do
certname=${i/.pub/-cert.pub}
curl -f -X POST -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $(cat /etc/confluent/confluent.apikey)" -d @$i https://$confluent_mgr/confluent-api/self/sshcert > $certname
echo HostKey ${i%.pub} | sed -e 's!/sysroot!!' >> /sysroot/etc/ssh/sshd_config
echo HostCertificate $certname | sed -e 's!/sysroot!!' >> /sysroot/etc/ssh/sshd_config
done

View File

@@ -0,0 +1,19 @@
[Unit]
Description=Confluent initialization
DefaultDependencies=false
After=basic.target
# Network is enabled here
After=dracut-initqueue.service
Before=coreos-livepxe-rootfs.service
# If we fail, the boot will fail. Be explicit about it.
OnFailure=emergency.target
OnFailureJobMode=isolate
[Service]
Type=oneshot
RemainAfterExit=yes
ExecStart=/opt/confluent/bin/initconfluent.sh

View File

@@ -0,0 +1,8 @@
#!/bin/sh
ln -s $1/images/pxeboot/vmlinuz $2/boot/kernel && \
ln -s $1/images/pxeboot/initrd.img $2/boot/initramfs/distribution && \
ln -s $1/images/ignition.img $2/boot/initramfs/ignition.img && \
mkdir -p $2/boot/efi/boot/ && \
ln -s $1/images/pxeboot/rootfs.img $2/ && \
mcopy -i $1/images/efiboot.img ::efi/redhat/grubx64.efi $2/boot/efi/boot/ && \
mcopy -i $1/images/efiboot.img ::efi/boot/bootx64.efi $2/boot/efi/boot/

View File

@@ -0,0 +1,5 @@
label: RedHat CoreOS %%VERSION%% %%ARCH%% (Default Profile)
kernelargs: quiet random.trust_cpu=on ignition.firstboot ignition.platform.id=metal
#the above boots and will have ignition agent phone home and await instructions
#kernelargs: quiet random.trust_cpu=on ignition.firstboot ignition.platform.id=metal coreos.inst=yes coreos.inst.install_dev=sda
#the above runs the installer to write to disk

View File

@@ -5,7 +5,7 @@ dracut_install curl openssl tar cpio gzip lsmod ethtool xz lsmod ethtool
dracut_install modprobe touch echo cut wc bash netstat uniq grep ip hostname
dracut_install awk egrep dirname expr sort
dracut_install ssh sshd reboot parted mkfs mkfs.ext4 mkfs.xfs xfs_db mkswap
dracut_install efibootmgr
dracut_install efibootmgr uuidgen
dracut_install du df ssh-keygen scp clear dhclient
dracut_install /lib64/libnss_dns-2.28.so /lib64/libnss_dns.so.2
dracut_install /usr/lib64/libnl-3.so.200
@@ -15,13 +15,7 @@ inst /bin/bash /bin/sh
dracut_install /lib64/libfuse.so.2 /lib64/libfuse.so.2.9.7
dracut_install chown chroot dd expr kill parted rsync sort blockdev findfs insmod lvm
dracut_install /usr/lib/udev/rules.d/10-dm.rules /usr/sbin/dmsetup /usr/lib/udev/rules.d/95-dm-notify.rules
dracut_install /usr/lib/udev/rules.d/60-net.rules /lib/udev/rename_device /usr/lib/systemd/network/99-default.link
#this would be nfs with lock, but not needed, go nolock
#dracut_install mount.nfs rpcbind rpc.statd /etc/netconfig sm-notify

View File

@@ -5,7 +5,7 @@ instmods nvme
instmods cdc_ether
instmods mptctl
instmods mlx4_ib mlx5_ub ib_umad ib_ipoib
instmods i40e hfi1 bnxt_en qed qede
instmods ice i40e hfi1 bnxt_en qed qede
instmods dm-mod dm-log raid0 raid1 raid10 raid456 dm-raid dm-thin-pool dm-crypt dm-snapshot linear dm-era
# nfs root and optionally gocryptfs
instmods fuse overlay squashfs loop

View File

@@ -1,4 +1,5 @@
installroot=$1
destdir=$2
touch $installroot/.testcap
if setcap cap_net_raw+p $installroot/.testcap >& /dev/null; then
@@ -7,9 +8,29 @@ else
capsargs="--setopt=tsflags=nocapps"
fi
rm $installroot/.testcap
yum $capsargs --releasever=8 --installroot=/var/tmp/testgenimage/ --releasever=8.3 install '@minimal-environment' chrony kernel net-tools nfs-utils openssh-server rsync tar util-linux python3 tar dracut dracut-network ethtool parted openssl dhclient openssh-clients bash vim-minimal rpm iputils shim-x64 grub2-efi-x64
dracut -v --xz -N -m "stateless base terminfo" -f boot/initramfs-diskless 4.18.0-240.22.1.el8_3.x86_64
pkglist=$(cat $(dirname $0)/pkglist | tr "\r\n" " ")
mydir=$(dirname $0)
mkdir -p $installroot/proc $installroot/sys $installroot/dev
unshare -f -p -m bash -c "
mount -o bind /proc $installroot/proc
mount -o bind /sys $installroot/sys
mount -o bind /dev $installroot/dev
yum -y $capsargs --releasever=8 --installroot=$installroot install $pkglist
cp -a $dirname/dracut $installroot/usr/lib/dracut/modules.d/97diskless
chmod a+x $installroot/usr/lib/dracut/modules.d/97diskless/*
for kernel in $(ls $installroot/boot/vmlinuz-*|grep -v rescue|sed -e s/.*vmlinuz-//); do
echo -n "Creating diskless initramfs for $kernel"
chroot $installroot dracut -v --xz -N -m "diskless base terminfo" -f boot/initramfs-diskless-$kernel.img $kernel
done
"
latestkernel=$(ls $installroot/boot/vmlinuz-*|grep -v rescue|sed -e s/.*vmlinuz-//|tail -n 1)
mkdir -p $destdir/boot/efi/boot $destdir/boot/initramfs
cp $installroot/boot/vmlinuz-$latestkernel $destdir/boot/kernel
cp $installroot/boot/initramfs-diskless-$latestkernel.img $destdir/boot/initramfs/distribution
cp $installroot/boot/efi/EFI/BOOT/BOOTX64.EFI $destdir/boot/efi/boot
cp $installroot/boot/efi/EFI/centos/grubx64.efi $destdir/boot/efi/boot
# link kernel, initrd, grub, and shim as appropriate
# use xz, minimize https burden and transfer penalty
mksquashfs -comp xz
mksquashfs $installroot $destdir/rootimg.sfs -comp xz

View File

@@ -62,6 +62,8 @@ if [[ $confluent_mgr == *%* ]]; then
ifname=${ifname%:}
fi
needseal=1
oldumask=$(umask)
umask 0077
while [ -z "$confluent_apikey" ]; do
/opt/confluent/bin/clortho $nodename $confluent_mgr > /etc/confluent/confluent.apikey
if grep ^SEALED: /etc/confluent/confluent.apikey > /dev/null; then
@@ -85,6 +87,7 @@ if [ $needseal == 1 ]; then
fi
fi
curl -sf -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" https://$confluent_mgr/confluent-api/self/deploycfg > /etc/confluent/confluent.deploycfg
umask $oldumask
autoconfigmethod=$(grep ipv4_method /etc/confluent/confluent.deploycfg |awk '{print $2}')
if [ "$autoconfigmethod" = "dhcp" ]; then
echo -n "Attempting to use dhcp to bring up $ifname..."
@@ -106,7 +109,62 @@ else
if [ ! -z "$v4gw" ]; then
ip route add default via $v4gw
fi
mkdir -p /run/NetworkManager/system-connections
cat > /run/NetworkManager/system-connections/$ifname.nmconnection << EOC
[connection]
id=eno1
EOC
echo uuid=$(uuidgen) >> /run/NetworkManager/system-connections/$ifname.nmconnection
cat >> /run/NetworkManager/system-connections/$ifname.nmconnection << EOC
type=ethernet
autoconnect-retries=1
EOC
echo interface-name=$ifname >> /run/NetworkManager/system-connections/$ifname.nmconnection
cat >> /run/NetworkManager/system-connections/$ifname.nmconnection << EOC
multi-connect=1
permissions=
wait-device-timeout=60000
[ethernet]
mac-address-blacklist=
[ipv4]
EOC
echo address1=$v4addr/$v4nm >> /run/NetworkManager/system-connections/$ifname.nmconnection
if [ ! -z "$v4gw" ]; then
echo gateway=$v4gw >> /run/NetworkManager/system-connections/$ifname.nmconnection
fi
nameserversec=0
nameservers=""
while read -r entry; do
if [ $nameserversec = 1 ]; then
if [[ $entry == "-"* ]]; then
nameservers="$nameservers"${entry#- }";"
continue
fi
fi
nameserversec=0
if [ "${entry%:*}" = "nameservers" ]; then
nameserversec=1
continue
fi
done < /etc/confluent/confluent.deploycfg
echo dns=$nameservers >> /run/NetworkManager/system-connections/$ifname.nmconnection
dnsdomain=$(grep ^dnsdomain: /etc/confluent/confluent.deploycfg)
dnsdomain=${dnsdomain#dnsdomain: }
echo dns-search=$dnsdomain >> /run/NetworkManager/system-connections/$ifname.nmconnection
cat >> /run/NetworkManager/system-connections/$ifname.nmconnection << EOC
may-fail=false
method=manual
[ipv6]
addr-gen-mode=eui64
method=auto
[proxy]
EOC
fi
chmod 600 /run/NetworkManager/system-connections/*.nmconnection
echo -n "Initializing ssh..."
ssh-keygen -A
for pubkey in /etc/ssh/ssh_host*key.pub; do
@@ -129,8 +187,10 @@ for addr in $(grep ^MANAGER: /etc/confluent/confluent.info|awk '{print $2}'|sed
confluent_urls="$confluent_urls $confluent_proto://$addr/confluent-public/os/$confluent_profile/rootimg.sfs"
fi
done
confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg| awk '{print $2}')
confluent_urls="$confluent_urls https://$confluent_mgr/confluent-public/os/$confluent_profile/rootimg.sfs"
mkdir -p /mnt/remoteimg /mnt/remote /mnt/overlay
curlmount $confluent_urls /mnt/remoteimg
/opt/confluent/bin/urlmount $confluent_urls /mnt/remoteimg
mount -o loop,ro /mnt/remoteimg/*.sfs /mnt/remote
mount -t tmpfs overlay /mnt/overlay
mkdir -p /mnt/overlay/upper /mnt/overlay/work
@@ -139,6 +199,7 @@ mkdir -p /sysroot/etc/ssh
mkdir -p /sysroot/etc/confluent
mkdir -p /sysroot/root/.ssh
cp /root/.ssh/* /sysroot/root/.ssh
chmod 700 /sysroot/root/.ssh
cp /etc/confluent/* /sysroot/etc/confluent/
cp /etc/ssh/*key* /sysroot/etc/ssh/
for pubkey in /etc/ssh/ssh_host*key.pub; do
@@ -149,6 +210,7 @@ for pubkey in /etc/ssh/ssh_host*key.pub; do
fi
echo HostKey $privfile >> /sysroot/etc/ssh/sshd_config
done
mkdir -p /sysroot/dev /sysroot/sys /sysroot/proc /sysroot/run
if [ ! -z "$autocons" ]; then
autocons=${autocons%,*}
@@ -162,6 +224,39 @@ while [ ! -e /sysroot/sbin/init ]; do
sleep 1
done
done
rootpassword=$(grep ^rootpassword: /etc/confluent/confluent.deploycfg)
rootpassword=${rootpassword#rootpassword: }
if [ "$rootpassword" = "null" ]; then
rootpassword=""
fi
if [ ! -z "$rootpassword" ]; then
sed -i "s@root:[^:]*:@root:$rootpassword:@" /sysroot/etc/shadow
fi
for i in /ssh/*.ca; do
echo '@cert-authority *' $(cat $i) >> /sysroot/etc/ssh/ssh_known_hosts
done
echo HostbasedAuthentication yes >> /sysroot/etc/ssh/sshd_config
echo HostbasedUsesNameFromPacketOnly yes >> /sysroot/etc/ssh/sshd_config
echo IgnoreRhosts no >> /sysroot/etc/ssh/sshd_config
sshconf=/sysroot/etc/ssh/ssh_config
if [ -d /sysroot/etc/ssh/ssh_config.d/ ]; then
sshconf=/sysroot/etc/ssh/ssh_config.d/01-confluent.conf
fi
echo 'Host *' >> $sshconf
echo ' HostbasedAuthentication yes' >> $sshconf
echo ' EnableSSHKeysign yes' >> $sshconf
echo ' HostbasedKeyTypes *ed25519*' >> $sshconf
curl -sf -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $(cat /etc/confluent/confluent.apikey)" https://$confluent_mgr/confluent-api/self/nodelist > /sysroot/etc/ssh/shosts.equiv
cp /sysroot/etc/ssh/shosts.equiv /sysroot/root/.shosts
chmod 640 /sysroot/etc/ssh/*_key
chroot /sysroot chgrp ssh_keys /etc/ssh/*_key
chroot /sysroot cat /etc/confluent/ca.pem >> /etc/pki/tls/certs/ca-bundle.crt
curl -sf https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/onboot.service > /sysroot/etc/systemd/system/onboot.service
mkdir -p /sysroot/opt/confluent/bin
curl -sf https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/onboot.sh > /sysroot/opt/confluent/bin/onboot.sh
chmod +x /sysroot/opt/confluent/bin/onboot.sh
ln -s /etc/systemd/system/onboot.service /sysroot/etc/systemd/system/multi-user.target.wants/onboot.service
exec /opt/confluent/bin/start_root

View File

@@ -0,0 +1,11 @@
[Unit]
Description=Confluent onboot hook
Requires=network-online.target
After=network-online.target
[Service]
ExecStart=/opt/confluent/bin/onboot.sh
[Install]
WantedBy=multi-user.target

View File

@@ -0,0 +1,27 @@
#!/bin/sh
# This script is executed on each boot as it is
# completed. It is best to edit the middle of the file as
# noted below so custom commands are executed before
# the script notifies confluent that install is fully complete.
nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
confluent_apikey=$(cat /etc/confluent/confluent.apikey)
confluent_mgr=$(grep deploy_server /etc/confluent/confluent.deploycfg|awk '{print $2}')
confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|awk '{print $2}')
export nodename confluent_mgr confluent_profile
. /etc/confluent/functions
exec >> /var/log/confluent/confluent-onboot.log
exec 2>> /var/log/confluent/confluent-onboot.log
tail -f /var/log/confluent/confluent-onboot.log > /dev/console &
logshowpid=$!
run_remote onboot.custom
# onboot scripts may be placed into onboot.d, e.g. onboot.d/01-firstaction.sh, onboot.d/02-secondaction.sh
run_remote_parts onboot.d
# Induce execution of remote configuration, e.g. ansible plays in ansible/onboot.d/
run_remote_config onboot.d
#curl -X POST -d 'status: booted' -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" https://$confluent_mgr/confluent-api/self/updatestatus
kill $logshowpid

View File

@@ -276,6 +276,13 @@ def proxydhcp():
bootfile = b'confluent/x86_64/ipxe.efi'
elif disco['arch'] == 'bios-x86':
bootfile = b'confluent/x86_64/ipxe.kkpxe'
if len(bootfile) > 127:
log.log(
{'info': 'Boot offer cannot be made to {0} as the '
'profile name "{1}" is {2} characters longer than is supported '
'for this boot method.'.format(
node, profile, len(bootfile) - 127)})
continue
rpv[:240] = rqv[:240].tobytes()
rpv[0:1] = b'\x02'
rpv[108:108 + len(bootfile)] = bootfile
@@ -485,6 +492,13 @@ def check_reply(node, info, packet, sock, cfg, reqview):
)
if not isinstance(bootfile, bytes):
bootfile = bootfile.encode('utf8')
if len(bootfile) > 127:
log.log(
{'info': 'Boot offer cannot be made to {0} as the '
'profile name "{1}" is {2} characters longer than is supported '
'for this boot method.'.format(
node, profile, len(bootfile) - 127)})
return
repview[108:108 + len(bootfile)] = bootfile
repview[20:24] = myipn
gateway = None

View File

@@ -23,6 +23,7 @@ READFILES = set([
'media.2/products',
'.DISCINFO',
'.discinfo',
'zipl.prm',
])
HEADERSUMS = set([b'\x85\xeddW\x86\xc5\xbdhx\xbe\x81\x18X\x1e\xb4O\x14\x9d\x11\xb7C8\x9b\x97R\x0c-\xb8Ht\xcb\xb3'])
@@ -424,6 +425,37 @@ def _priv_check_oraclelinux(isoinfo):
return {'name': 'oraclelinux-{0}-{1}'.format(ver, arch), 'method': EXTRACT,
'category': 'el{0}'.format(major)}
def fixup_coreos(targpath):
# the efi boot image holds content that the init script would want
# to mcopy, but the boot sector is malformed usually, so change it to 1
# sector per track
if os.path.exists(targpath + '/images/efiboot.img'):
with open(targpath + '/images/efiboot.img', 'rb+') as bootimg:
bootimg.seek(0x18)
if bootimg.read != b'\x00\x00':
bootimg.seek(0x18)
bootimg.write(b'\x01')
def check_coreos(isoinfo):
arch = 'x86_64' # TODO: would check magic of vmlinuz to see which arch
if 'zipl.prm' in isoinfo[1]:
prodinfo = isoinfo[1]['zipl.prm']
if not isinstance(prodinfo, str):
prodinfo = prodinfo.decode('utf8')
for inf in prodinfo.split():
if inf.startswith('coreos.liveiso=rhcos-'):
ver = inf.split('-')[1]
return {'name': 'rhcos-{0}-{1}'.format(ver, arch),
'method': EXTRACT, 'category': 'coreos'}
elif inf.startswith('coreos.liveiso=fedore-coreos-'):
ver = inf.split('-')[2]
return {'name': 'fedoracoreos-{0}-{1}'.format(ver, arch),
'method': EXTRACT, 'category': 'coreos'}
def check_rhel(isoinfo):
ver = None
arch = None
@@ -502,7 +534,7 @@ def fingerprint(archive):
if fun.startswith('check_'):
name = globals()[fun](isoinfo)
if name:
return name, isoinfo[0]
return name, isoinfo[0], fun.replace('check_', '')
return None
else:
sum = hashlib.sha256(header)
@@ -514,7 +546,7 @@ def fingerprint(archive):
chunk = archive.read(32768)
imginfo = HASHPRINTS.get(sum.hexdigest(), None)
if imginfo:
return imginfo, None
return imginfo, None, None
def import_image(filename, callback, backend=False, mfd=None):
@@ -525,7 +557,7 @@ def import_image(filename, callback, backend=False, mfd=None):
identity = fingerprint(archive)
if not identity:
return -1
identity, imginfo = identity
identity, imginfo, funname = identity
targpath = identity['name']
distpath = '/var/lib/confluent/distributions/' + targpath
if identity.get('subname', None):
@@ -566,6 +598,8 @@ def import_image(filename, callback, backend=False, mfd=None):
del identity['subname']
with open(distpath + '/distinfo.yaml', 'w') as distinfo:
distinfo.write(yaml.dump(identity, default_flow_style=False))
if 'fixup_{0}'.format(funname) in globals():
globals()['fixup_{0}'.format(funname)](targpath)
callback({'progress': 1.0})
sys.stdout.write('\n')
@@ -648,7 +682,7 @@ class MediaImporter(object):
if not identity:
raise exc.InvalidArgumentException('Unsupported Media')
self.percent = 0.0
identity, _ = identity
identity, _, _ = identity
self.phase = 'copying'
if not identity:
raise Exception('Unrecognized OS Media')

View File

@@ -11,6 +11,16 @@ import tempfile
agent_pid = None
ready_keys = {}
_sshver = None
def sshver():
global _sshver
if _sshver is None:
p = subprocess.Popen(['ssh', '-V'], stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
_, output = p.communicate()
_sshver = float(output.split()[0].split(b'_')[1].split(b'p')[0])
return _sshver
def normalize_uid():
curruid = os.geteuid()
@@ -23,6 +33,8 @@ def normalize_uid():
def assure_agent():
if sshver() <= 7.6:
return
global agent_pid
if agent_pid is None:
sai = subprocess.check_output(['ssh-agent'])
@@ -41,6 +53,8 @@ def assure_agent():
os.environ[k] = v
def get_passphrase():
if sshver() <= 7.6:
return ''
# convert the master key to base64
# for use in ssh passphrase context
if cfm._masterkey is None:
@@ -106,8 +120,9 @@ def sign_host_key(pubkey, nodename, principals=()):
principals = set(principals)
principals.add(nodename)
principals = ','.join(sorted(principals))
flags = '-Us' if sshver() > 7.6 else '-s'
subprocess.check_call(
['ssh-keygen', '-Us', '/etc/confluent/ssh/ca.pub', '-I', nodename,
['ssh-keygen', flags, '/etc/confluent/ssh/ca.pub', '-I', nodename,
'-n', principals, '-h', pkeyname])
certname = pkeyname.replace('.pub', '-cert.pub')
with open(certname) as cert:

View File

@@ -4,5 +4,5 @@ instmods nvme
instmods cdc_ether
instmods mptctl
instmods mlx4_ib mlx5_ub ib_umad ib_ipoib
instmods i40e hfi1 bnxt_en qed qede
instmods ice i40e hfi1 bnxt_en qed qede
instmods dm-mod dm-log raid0 raid1 raid10 raid456 dm-raid dm-thin-pool dm-crypt dm-snapshot linear dm-era

242
misc/urlmount.c Normal file
View File

@@ -0,0 +1,242 @@
/*
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2021 Lenovo
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
*/
#define FUSE_USE_VERSION 26
#include <fuse.h>
#include <curl/curl.h>
#include <errno.h>
#include <stdlib.h>
#include <string.h>
#include <math.h>
#include <unistd.h>
CURL *curl;
char curlerror[CURL_ERROR_SIZE];
curl_off_t filesize;
typedef struct downloadbuffer {
char *response;
size_t completed;
size_t total;
} downloadbuffer;
#define MAX_FILE_LEN 1024
#define MAX_URL_PATHS 512
static char filename[MAX_FILE_LEN];
static int urlidx;
static char* urls[MAX_URL_PATHS];
static int http_readdir(const char *path, void *buf, fuse_fill_dir_t filler,
off_t offset, struct fuse_file_info *fi)
{
if (strcmp(path, "/") != 0) // We don't support subdirs
return -ENOENT;
filler(buf, ".", NULL, 0);
filler(buf, "..", NULL, 0);
filler(buf, filename + 1, NULL, 0);
return 0;
}
size_t fill_buffer(char *data, size_t size, size_t nmemb, downloadbuffer *userdata) {
size_t amount;
amount = size * nmemb;
if (userdata->total < amount + userdata->completed) return 0;
memcpy(&(userdata->response[userdata->completed]), data, amount);
userdata->completed += amount;
return amount;
}
static int http_read(const char *path, char *buf, size_t size, off_t offset,
struct fuse_file_info *fi) {
char headbuffer[512];
double dldbl = 0.0;
int startidx;
int reconnecting = 0;
FILE* fd;
startidx = urlidx;
memset(buf, 0, size);
curl_off_t downloaded;
//Would be needed for multithread, however preferring to conserve
//filehandles rather than go multithread
// Some comparisons showed that the threaded performance boost doesn't
// do even offset the overhead of the new curl handles, so better
// to use single threaded curl overall for now
//CURL *tmpcurl = curl_easy_duphandle(curl);
downloadbuffer dlbuf;
dlbuf.response = buf;
dlbuf.completed = 0;
dlbuf.total = size;
fd = NULL;
if (strcmp(path, filename) != 0) return -ENOENT;
memset(headbuffer, 0, 512);
snprintf(headbuffer, 512, "%ld-%ld", offset, offset + size - 1);
curl_easy_setopt(curl, CURLOPT_RANGE, headbuffer);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &dlbuf);
while (curl_easy_perform(curl) != CURLE_OK) {
reconnecting = 1;
fd = fopen("/dev/kmsg", "w+");
dlbuf.completed = 0;
fprintf(fd, "<1>urlmount: error while communicating with %s: %s\n", urls[urlidx], curlerror);
urlidx++;
if (urls[urlidx] == NULL)
urlidx = 0;
if (urlidx == startidx)
sleep(10);
fprintf(fd, "urlmount: Connecting to %s\n", urls[urlidx]);
curl_easy_setopt(curl, CURLOPT_URL, urls[urlidx]);
fclose(fd);
}
if (reconnecting) {
fd = fopen("/dev/kmsg", "w+");
fprintf(fd, "<1>urlmount: Successfully connected to %s\n", urls[urlidx]);
fclose(fd);
}
curl_easy_getinfo(curl, CURLINFO_SIZE_DOWNLOAD, &dldbl);
downloaded = round(dldbl);
//Would be needed for multithread
//curl_easy_cleanup(tmpcurl);
return downloaded;
}
static int http_open(const char *path, struct fuse_file_info *fi) {
if (strcmp(path, filename) != 0)
return -ENOENT;
if ((fi->flags & 3) != O_RDONLY)
return -EACCES;
return 0;
}
static void* http_init(struct fuse_conn_info *conn) {
// Because we fork, we need to redo curl
// or else suffer the wrath of NSS TLS
curl_global_init(CURL_GLOBAL_DEFAULT);
curl = curl_easy_init();
curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, curlerror);
//We want to consider error conditions fatal, rather than
//passing error text as data
curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1L);
curl_easy_setopt(curl, CURLOPT_TIMEOUT, 10L);
curl_easy_setopt(curl, CURLOPT_URL, urls[urlidx]);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fill_buffer);
return NULL;
}
static int http_getattr(const char *path, struct stat *st) {
memset(st, 0, sizeof(struct stat));
if (strcmp(path, "/") == 0) {
st->st_mode = S_IFDIR | 0555;
st->st_nlink = 2;
} else if (strcmp(path, filename) == 0) {
st->st_mode = S_IFREG | 0444;
st->st_nlink = 1;
st->st_size = filesize; // TODO: fix with curl HEAD
} else
return -ENOENT;
return 0;
}
static const struct fuse_operations http_ops = {
.getattr = http_getattr,
.readdir = http_readdir,
.read = http_read,
.open = http_open,
.init = http_init,
};
int main(int argc, char* argv[]) {
char *tmp;
double fsize;
unsigned int i;
int j;
j = open("/dev/urandom", O_RDONLY);
if (j <= 0 || read(j, (char*)&i, 4) < 0) {
i = time(NULL);
}
if (j > 0) {
close(j);
}
srand(i);
j = 0;
memset(urls, 0, 32*sizeof(char*));
urlidx = 0;
curl_global_init(CURL_GLOBAL_DEFAULT);
curl = curl_easy_init();
curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, curlerror);
//We want to consider error conditions fatal, rather than
//passing error text as data
curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1L);
curl_easy_setopt(curl, CURLOPT_TIMEOUT, 15L);
memset(filename, 0, MAX_FILE_LEN);
for (i=0; i < argc; i++) {
if (strstr(argv[i], ":") > 0) {
if (j < MAX_URL_PATHS) {
urls[j] = argv[i];
tmp = strrchr(urls[j++], '/');
strncpy(filename, tmp, MAX_FILE_LEN);
}
//Request single threaded mode, as curl would need more
// filehandles for multithread
argv[i] = "-s";
}
}
if (filename[0] == 0) {
fprintf(stderr, "No URL given in arguments\n");
exit(1);
}
for (i=0; urls[i] != NULL; i++) {
printf("Registering mount path: %s\n", urls[i]);
}
urlidx = rand() % j;
j = urlidx;
printf("Connecting to %s\n", urls[urlidx]);
curl_easy_setopt(curl, CURLOPT_URL, urls[urlidx]);
curl_easy_setopt(curl, CURLOPT_NOBODY, 1);
while (curl_easy_perform(curl) != CURLE_OK) {
fprintf(stderr, "urlmount: error while communicating with %s: %s\n", urls[urlidx++], curlerror);
if (urls[urlidx] == NULL)
urlidx = 0;
if (urlidx == j) {
fprintf(stderr, "urlmount: Unable to reach any target url, aborting\n");
exit(1);
}
printf("Connecting to %s\n", urls[urlidx]);
curl_easy_setopt(curl, CURLOPT_URL, urls[urlidx]);
}
printf("Successfully connected to %s\n", urls[urlidx]);
curl_easy_getinfo(curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &fsize);
filesize = round(fsize);
curl_easy_setopt(curl, CURLOPT_NOBODY, 0);
if (filesize < 1) {
fprintf(stderr, "Unable to reach designated URL\n");
exit(1);
}
if (!curl) {
fprintf(stderr, "Unable to initialize CURL!\n");
exit(1);
}
curl_easy_cleanup(curl);
curl_global_cleanup();
fuse_main(argc, argv, &http_ops, NULL);
}