From 4cb37a0a7d63c66a17556c1e68f689a9b9d7e1fd Mon Sep 17 00:00:00 2001 From: Bruce Potter Date: Sat, 26 Apr 2014 18:29:01 -0400 Subject: [PATCH] sysclone working in softlayer --- xCAT-SoftLayer/bin/modifygrub | 50 +++++++-- xCAT-SoftLayer/bin/pushinitrd | 57 +++++++++- .../si-post-install/14all.addnetworkdelay | 43 ++++++++ .../si-post-install/16all.updatenetwork | 101 ++++++++++++++++++ xCAT-SoftLayer/xCAT-SoftLayer.spec | 4 + xCAT-genesis-scripts/bin/doxcat | 33 +++--- 6 files changed, 262 insertions(+), 26 deletions(-) create mode 100755 xCAT-SoftLayer/si-post-install/14all.addnetworkdelay create mode 100755 xCAT-SoftLayer/si-post-install/16all.updatenetwork diff --git a/xCAT-SoftLayer/bin/modifygrub b/xCAT-SoftLayer/bin/modifygrub index 07ce948a9..a2e70caff 100755 --- a/xCAT-SoftLayer/bin/modifygrub +++ b/xCAT-SoftLayer/bin/modifygrub @@ -13,11 +13,12 @@ use Socket; my $HELP; my $VERBOSE; my $WAITTIME; +my $PROVMETHOD; my $XCATNETBOOTTITLE = 'xCAT network boot kernel and initrd'; my $usage = sub { my $exitcode = shift @_; - print "Usage: modifygrub [-?|-h|--help] [-v|--verbose] [-w ] \n\n"; + print "Usage: modifygrub [-?|-h|--help] [-v|--verbose] [-w ] [-p \n\n"; if (!$exitcode) { print "Modify the grub config file on the node to boot the specified kernel and initrd.\n"; } @@ -30,7 +31,7 @@ if (-f '/etc/os-release') { die "This script doesn't support ubuntu yet.\n"; } Getopt::Long::Configure("bundling"); #Getopt::Long::Configure("pass_through"); Getopt::Long::Configure("no_pass_through"); -if (!GetOptions('h|?|help' => \$HELP, 'v|verbose' => \$VERBOSE, 'w|waittime=s' => \$WAITTIME)) { $usage->(1); } +if (!GetOptions('h|?|help' => \$HELP, 'v|verbose' => \$VERBOSE, 'w|waittime=s' => \$WAITTIME, 'p|provmethod=s' => \$PROVMETHOD)) { $usage->(1); } if ($HELP) { $usage->(0); } if (scalar(@ARGV) != 4) { $usage->(1); } @@ -60,7 +61,7 @@ sub addKernelParms { $args->{kernelparms} =~ s//$nodename/g; # get node ip and add it to the kernel parms - my ($nic, $ip, $netmask, $gateway, $mac) = getNodeIpInfo($args); + my ($nic, $ip, $netmask, $network, $broadcast, $gateway, $mac) = getNodeIpInfo($args); if (!$ip) { die "Error: could not find the NIC that would connect to the xCAT mgmt node's IP (".$args->{mnip}.").\n"; } # if we are booting genesis, need to add the BOOTIF parm my $bootif; @@ -69,7 +70,16 @@ sub addKernelParms { $bootif =~ s/:/-/g; $bootif = "BOOTIF=01-$bootif"; } - $args->{kernelparms} .= " $bootif hostip=$ip netmask=$netmask gateway=$gateway dns=$mnip hostname=$nodename netdevice=$nic netwait=$WAITTIME textmode=1"; + + if (defined($PROVMETHOD) && $PROVMETHOD eq 'sysclone') { + # add additional parms for sysclone + # DEVICE=eth0 IPADDR=10.0.0.99 NETMASK=255.255.255.0 NETWORK=10.0.0.0 BROADCAST=10.0.0.255 GATEWAY=10.0.0.1 GATEWAYDEV=eth0 + #todo: should we also add ETHER_SLEEP=$WAITTIME textmode=1 dns=$mnip ? + $args->{kernelparms} .= " $bootif IPADDR=$ip NETMASK=$netmask NETWORK=$network BROADCAST=$broadcast GATEWAY=$gateway HOSTNAME=$nodename DEVICE=$nic GATEWAYDEV=$nic"; + } + else { # scripted install or genesis shell + $args->{kernelparms} .= " $bootif hostip=$ip netmask=$netmask gateway=$gateway dns=$mnip hostname=$nodename netdevice=$nic netwait=$WAITTIME textmode=1"; + } } @@ -88,7 +98,7 @@ sub getNodeIpInfo { if (($mactmp) = $line=~m|^\s+link/ether\s+(\S+) |) { $mac = $mactmp; } # got mac, remember it if (($iptmp) = $line=~m/^\s+inet\s+($ipprefix\S+) /) { $ipandmask = $iptmp; last; } # got ip, we are done } - my ($ip, $netmask) = convertIpAndMask($ipandmask); + my ($ip, $netmask, $network, $broadcast) = convertIpAndMask($ipandmask); # if the nic is a bonded nic (common on sl), then find the 1st real nic that is part of it my $realnic = $nic; @@ -117,18 +127,36 @@ sub getNodeIpInfo { } verbose("IP info: realnic=$realnic, ip=$ip, netmask=$netmask, gateway=$gateway, mac=$mac"); - return ($realnic, $ip, $netmask, $gateway, $mac); + return ($realnic, $ip, $netmask, $network, $broadcast, $gateway, $mac); } -# Convert an ip/mask in slash notation (like 10.0.0.1/26) to separate ip and netmask like 10.0.0.1 and 255.255.255.192 +# Convert an ip/mask in slash notation (like 10.1.1.1/26) to separate ip, netmask, network, and broadcast values, +# like: 10.1.1.1, 255.255.255.192, 10.1.1.0, 10.1.1.63 sub convertIpAndMask { my $ipandmask = shift @_; my ($ip, $masknum) = split('/', $ipandmask); - my $netbin = oct("0b" . '1' x $masknum . '0' x (32-$masknum)); # create a str like '1111100', then convert to binary - my @netarr=unpack('C4',pack('N',$netbin)); # separate into the 4 octets - my $netmask=join('.',@netarr); # put them together into the normal looking netmask - return ($ip, $netmask); + + # build the netmask + my $nmbin = oct("0b" . '1' x $masknum . '0' x (32-$masknum)); # create a str like '1111100', then convert to binary + my @nmarr=unpack('C4',pack('N',$nmbin)); # separate into the 4 octets + my $netmask=join('.',@nmarr); # put them together into the normal looking netmask + + # create binary form of ip + my @iparr=split(/\./,$ip); + my ( $ipbin ) = unpack('N', pack('C4',@iparr ) ); + + # Calculate network address by logical AND operation of ip & netmask and convert network address to IP address format + my $netbin = ( $ipbin & $nmbin ); + my @netarr=unpack('C4', pack('N',$netbin ) ); + my $network=join(".",@netarr); + + # Calculate broadcast address by inverting the netmask and adding it to the network address + my $bcbin = ( $ipbin & $nmbin ) + ( ~ $nmbin ); + my @bcarr=unpack('C4', pack('N',$bcbin ) ) ; + my $broadcast=join(".",@bcarr); + + return ($ip, $netmask, $network, $broadcast); } diff --git a/xCAT-SoftLayer/bin/pushinitrd b/xCAT-SoftLayer/bin/pushinitrd index b454a1f77..bcad30198 100755 --- a/xCAT-SoftLayer/bin/pushinitrd +++ b/xCAT-SoftLayer/bin/pushinitrd @@ -47,6 +47,8 @@ updateGrubOnNodes($noderange, \%bootparms); if ($bootparms{osimageprovmethod} eq 'install' && !$NOAUTOINST) { modifyAutoinstFiles($noderange, \%bootparms); } +if ($bootparms{osimageprovmethod} eq 'sysclone') { copySyscloneFiles(); } + exit(0); @@ -118,7 +120,7 @@ sub updateGrubOnNodes { my @output = runcmd('which modifygrub'); my $modifygrub = $output[0]; chomp($modifygrub); - my $cmd = "xdsh $nr -e $modifygrub $vtxt -w $WAITTIME " . remoteFilename($bootparms->{kernel}) . ' ' . remoteFilename($bootparms->{initrd}) . ' '; + my $cmd = "xdsh $nr -e $modifygrub $vtxt -w $WAITTIME -p " . $bootparms->{osimageprovmethod} . ' ' . remoteFilename($bootparms->{kernel}) . ' ' . remoteFilename($bootparms->{initrd}) . ' '; # we need to quote the kernel parms, both here when passing it to xdsh, and on the node # when xdsh is passing it to modifygrub. The way to get single quotes inside single quotes # is to quote each of the outer single quotes with double quotes. @@ -143,15 +145,55 @@ sub modifyAutoinstFiles { # Edit each file to have chroot.sles insert a wait at the end of /etc/init.d/network # this finds the end of boot.sh script (which is chroot.sles) my $search = '\n\]\]>\s*\s*\s*'; + + # hack the /etc/init.d/network script to put a wait in it my $file = '/mnt/etc/init.d/network'; # at this point in the installation, the permanent file system is just mounted #my $waitstring = 'echo Sleeping for 55s;sleep 55'; - # this is the string to insert in the nodes /etc/init.d/network script. It is a while loop pinging the mn, but some of the chars need to be escape for sed - my $waitstring = 'echo Waiting to reach xCAT mgmt node...;while \[ \$\(\(xcati+=1\)\) -le 60 \] \&\& ! ping -c2 -w3 ' . $bootparms->{mnip} .'; do echo i=\$xcati ; done; sleep 10'; + # this is the string to insert in the nodes /etc/init.d/network script. It is a while loop pinging the mn, but some of the chars need to be escaped for sed + my $waitstring = 'echo -n Waiting to reach xCAT mgmt node ' . $bootparms->{mnip} . '.;xcatretries=60;while \[ \$\(\(xcati+=1\)\) -le \$xcatretries \] \&\& ! ping -c2 -w3 ' . $bootparms->{mnip} .' \>\/dev\/null 2\>\&1; do echo -n .; done; if \[ \$xcati -le \$xcatretries \]; then echo success; else echo failed; fi; sleep 3'; # this crazy sed string is from google. It gathers up the whole file into the hold buffer, and then the substitution is done on the whole file my $sedstring = q|sed -n '1h;1!H;${;g;s/\(\t\treload_firewall\n\)\n/\1\t\t| . $waitstring . q(\n\n/g;p;}') . " $file > $file.new"; # finally create the perl replace string that will be used to modify the autoinst file my $replace = "$sedstring\nchmod 755 $file.new; mv -f $file.new $file"; + # Now instead we add a script that gets invoked by the OS after the nic is brought up + # these are specific to SLES + #my $netdir = '/etc/sysconfig/network'; + #my $filename = '/etc/sysconfig/network/if-up.d/xcat-sl-wait'; + #my $mnip = $bootparms->{mnip}; + #todo: to support rhel, use these values instead + #my $netdir='/etc/sysconfig/network-scripts'; + #my $filename='/sbin/ifup-local'; + + # this does not work, because midway thru the autoyast process, the if-up.d scripts do not seem to get invoked + # so autoyast fails to get he media + #my $replace = qq( +#FILENAME=$filename +#NETDIR=$netdir +#MNIP=$mnip +#); +# $replace .= q( +#cat >$FILENAME << EOF1 +#MNIP=$MNIP +#NETDIR=$NETDIR +#EOF1 +# +# this part of the file we do NOT want to expand the variables in the content +#cat >>$FILENAME << 'EOF2' +#NIC="$1" +# look in this ifcfg script to get the nics ip to see if this is the one we should be waiting on +#NICIP=`awk -F= '/^IPADDR/ {print $2}' $NETDIR/ifcfg-$NIC | tr -d \' ` +#if [ "${NICIP%.*.*}" != "${MNIP%.*.*}" ]; then exit; fi # hack: compare the 1st 2 octets +#echo -n Waiting to reach xCAT mgmt node $MNIP. +#xcatretries=60 +#while [ $((xcati+=1)) -le $xcatretries ] && ! ping -c2 -w3 $MNIP >/dev/null 2>&1; do echo -n .; done +#if [ $xcati -le $xcatretries ]; then echo " success"; else echo " failed"; fi +#sleep 3 +#EOF2 +# +#chmod +x $FILENAME +#); + # now actually update the file print "Updating /install/autoinst files.\n"; foreach my $n (@nodes) { @@ -162,6 +204,15 @@ sub modifyAutoinstFiles { } +# Copy softlayer specific systemimager post-install scripts to the systemimager location. +# These cause si to use static ip and insert a wait into the bring up of the network. +sub copySyscloneFiles { + my $cmd = "cp -f /opt/xcat/share/xcat/sysclone/post-install/* /install/sysclone/scripts/post-install"; + print "Copying SoftLayer-specific post scripts to the SystemImager post-install directory.\n"; + runcmd($cmd); +} + + # this is like multi-line sed replace function # Args: filename, search-string, replace-string sub sed { diff --git a/xCAT-SoftLayer/si-post-install/14all.addnetworkdelay b/xCAT-SoftLayer/si-post-install/14all.addnetworkdelay new file mode 100755 index 000000000..a1f205dfe --- /dev/null +++ b/xCAT-SoftLayer/si-post-install/14all.addnetworkdelay @@ -0,0 +1,43 @@ +#!/bin/bash + +# Add a script that will run when the network is brought up to wait until the +# xcat mn can be pinged. + +. /tmp/post-install/variables.txt + +if [ -d "/etc/sysconfig/network-scripts/" ];then + #redhat + NETDIR=/etc/sysconfig/network-scripts + filename=/sbin/ifup-local +elif [ -d "/etc/sysconfig/network/" ];then + #suse + NETDIR=/etc/sysconfig/network + filename=/etc/sysconfig/network/if-up.d/xcat-sl-wait +else + #ubuntu + echo "Does not support ubuntu." + exit 1 +fi + +# Create the wait script, overwriting a copy that may have come from the golden client +# (in case the mn/image server is different). +# this part of the file we want to expand the variables in the content +cat >$filename << EOF1 +MNIP=$IMAGESERVER +NETDIR=$NETDIR +EOF1 + +# this part of the file we do NOT want to expand the variables in the content +cat >>$filename << 'EOF2' +NIC="$1" +# look in this ifcfg script to get the nics ip to see if this is the one we should be waiting on +NICIP=`awk -F= '/^IPADDR/ {print $2}' $NETDIR/ifcfg-$NIC | tr -d \' ` +if [ "${NICIP%.*.*}" != "${MNIP%.*.*}" ]; then exit; fi # hack: compare the 1st 2 octets +echo -n Waiting to reach xCAT mgmt node $MNIP. +xcatretries=60 +while [ $((xcati+=1)) -le $xcatretries ] && ! ping -c2 -w3 $MNIP >/dev/null 2>&1; do echo -n .; done +if [ $xcati -le $xcatretries ]; then echo " success"; else echo " failed"; fi +sleep 3 +EOF2 + +chmod +x $filename \ No newline at end of file diff --git a/xCAT-SoftLayer/si-post-install/16all.updatenetwork b/xCAT-SoftLayer/si-post-install/16all.updatenetwork new file mode 100755 index 000000000..2d80690aa --- /dev/null +++ b/xCAT-SoftLayer/si-post-install/16all.updatenetwork @@ -0,0 +1,101 @@ +#!/bin/bash + +# Configure network settings after SI has installed the OS + +. /tmp/post-install/variables.txt + +# determine if we should be using a static ip or dhcp +staticIP () { + # Eventually we should use the SI variable IP_ASSIGNMENT_METHOD below to determine this. + # But this requires a patch in both xcat/sysclone (to set si_getimage -ip-assignment method) + # and SI (to set IP_ASSIGNMENT_METHOD as a result of that). Until both of those patches + # are in the main releases, assume that if we have set the IPADDR kernel parm for the boot + # kernel to use static ip, that the final OS should use static ip too. + # Note: the IPADDR environment variable will be set by SI, even if it got it thru dhcp, so + # that is not a reliable way to decide. + str=`cat /proc/cmdline` + #str='netmask=255.255.255.192 IPADDR=10.54.51.11 GATEWAY=10.54.51.1 dns=10.54.51.2 hostname=sap64-4 DEVICE=eth0' + for parm in $str; do + key=`echo $parm|awk -F= '{print $1}'` + value=`echo $parm|awk -F= '{print $2}'` + if [[ $key == "IPADDR" || $key == "ipaddr" ]]; then + return 0 # yes, we should use static ip + fi + done + if [[ -n $IP_ASSIGNMENT_METHOD && ${IP_ASSIGNMENT_METHOD,,} == "static" ]]; then + return 0 # this means yes/true + fi + return 1 +} + +#delete the udev rule in the image +rule_file=`ls /etc/udev/rules.d/*net_persistent_names.rules 2>/dev/null` +if [ -n "$rule_file" ];then + rm -f $rule_file +fi + +hostname $HOSTNAME + +device_names=`ifconfig -a | grep -i hwaddr | grep -i 'Ethernet' | grep -v usb| awk '{print $1}'` +str_cfg_file='' +if [ -d "/etc/sysconfig/network-scripts/" ];then + #redhat + grep -i HOSTNAME /etc/sysconfig/network + if [ $? -eq 0 ];then + sed -i "s/HOSTNAME=.*/HOSTNAME=$HOSTNAME/g" /etc/sysconfig/network + else + echo "HOSTNAME=$HOSTNAME" >> /etc/sysconfig/network + fi + if staticIP; then + # set static ip from variables in variables.txt + i="$DEVICE" + str_cfg_file="/etc/sysconfig/network-scripts/ifcfg-$i" + echo "DEVICE=$i" > $str_cfg_file + echo "BOOTPROTO=static" >> $str_cfg_file + echo "ONBOOT=yes" >> $str_cfg_file + echo "IPADDR=$IPADDR" >> $str_cfg_file + echo "NETMASK=$NETMASK" >> $str_cfg_file + echo "NETWORK=$NETWORK" >> $str_cfg_file + echo "BROADCAST=$BROADCAST" >> $str_cfg_file + #todo: add gateway config? Not sure, because the boot kernels gateway might not be the final OS gateway + else + # use dhcp for all nics + for i in $device_names;do + str_cfg_file="/etc/sysconfig/network-scripts/ifcfg-$i" + echo "DEVICE=$i" > $str_cfg_file + echo "BOOTPROTO=dhcp" >> $str_cfg_file + echo "NM_CONTROLLED=yes" >> $str_cfg_file + echo "ONBOOT=yes" >> $str_cfg_file + done + fi +elif [ -d "/etc/sysconfig/network/" ];then + #suse + echo "$HOSTNAME" > /etc/HOSTNAME + if staticIP; then + # set static ip from variables in variables.txt + i="$DEVICE" + str_cfg_file="/etc/sysconfig/network/ifcfg-$i" + echo "DEVICE=$i" > $str_cfg_file + echo "BOOTPROTO=static" >> $str_cfg_file + echo "STARTMODE=onboot" >> $str_cfg_file + echo "IPADDR=$IPADDR" >> $str_cfg_file + echo "NETMASK=$NETMASK" >> $str_cfg_file + echo "NETWORK=$NETWORK" >> $str_cfg_file + echo "BROADCAST=$BROADCAST" >> $str_cfg_file + #todo: add gateway config? Not sure, because the boot kernels gateway might not be the final OS gateway + else + # use dhcp for all nics + for i in $device_names;do + str_cfg_file="/etc/sysconfig/network/ifcfg-$i" + echo "DEVICE=$i" > $str_cfg_file + echo "BOOTPROTO=dhcp" >> $str_cfg_file + echo "STARTMODE=onboot" >> $str_cfg_file + echo "DHCLIENT_PRIMARY_DEVICE=yes" >> $str_cfg_file + done + fi +else + #ubuntu + echo "Does not support ubuntu." + exit 1 +fi + diff --git a/xCAT-SoftLayer/xCAT-SoftLayer.spec b/xCAT-SoftLayer/xCAT-SoftLayer.spec index df514ddfb..8e4420c72 100644 --- a/xCAT-SoftLayer/xCAT-SoftLayer.spec +++ b/xCAT-SoftLayer/xCAT-SoftLayer.spec @@ -47,6 +47,7 @@ mkdir -p $RPM_BUILD_ROOT/install/postscripts mkdir -p $RPM_BUILD_ROOT/%{prefix}/share/doc/packages/xCAT-SoftLayer mkdir -p $RPM_BUILD_ROOT/%{prefix}/share/man/man1 mkdir -p $RPM_BUILD_ROOT/%{prefix}/share/doc/man1 +mkdir -p $RPM_BUILD_ROOT/%{prefix}/share/xcat/sysclone/post-install cp -p -R share/xcat/install/* $RPM_BUILD_ROOT/%{prefix}/share/xcat/install/ @@ -56,6 +57,9 @@ chmod 755 $RPM_BUILD_ROOT/%{prefix}/bin/* cp -d postscripts/* $RPM_BUILD_ROOT/install/postscripts chmod 755 $RPM_BUILD_ROOT/install/postscripts/* +cp -d post-install/* $RPM_BUILD_ROOT/share/xcat/sysclone/post-install +chmod 755 $RPM_BUILD_ROOT/share/xcat/sysclone/post-install/* + cp LICENSE.html $RPM_BUILD_ROOT/%{prefix}/share/doc/packages/xCAT-SoftLayer chmod 644 $RPM_BUILD_ROOT/%{prefix}/share/doc/packages/xCAT-SoftLayer/* diff --git a/xCAT-genesis-scripts/bin/doxcat b/xCAT-genesis-scripts/bin/doxcat index c66f2f412..7e8a0a2f7 100755 --- a/xCAT-genesis-scripts/bin/doxcat +++ b/xCAT-genesis-scripts/bin/doxcat @@ -4,11 +4,13 @@ # Bug reported by Jeff Lang . Thanks, Jeff! # -modprobe acpi_cpufreq +modprobe acpi_cpufreq 2>/dev/null # on some machines this fails modprobe cpufreq_ondemand -for gov in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do - echo -n ondemand > $gov -done +if ls /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor &>/dev/null; then + for gov in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do + echo -n ondemand > $gov + done +fi if [ ! -z "$BOOTIF" ]; then BOOTIF=`echo $BOOTIF|sed -e s/01-// -e s/-/:/g` echo -n "Waiting for device with address $BOOTIF to appear.." @@ -117,14 +119,15 @@ mask2prefix() { # see if they specified static ip info, otherwise use dhcp for parm in `cat /proc/cmdline`; do key=`echo $parm|awk -F= '{print $1}'` - if [ "$key" = "hostip" ]; then - hostip=`echo $parm|awk -F= '{print $2}'` - elif [ "$key" = "netmask" ]; then - netmask=`echo $parm|awk -F= '{print $2}'` - elif [ "$key" = "gateway" ]; then - gateway=`echo $parm|awk -F= '{print $2}'` - elif [ "$key" = "netdevice" ]; then - netdevice=`echo $parm|awk -F= '{print $2}'` + value=`echo $parm|awk -F= '{print $2}'` + if [[ ${key,,} == "hostip" || ${key,,} == "ipaddr" ]]; then + hostip=$value + elif [[ ${key,,} == "netmask" ]]; then + netmask=$value + elif [[ ${key,,} == "gateway" ]]; then + gateway=$value + elif [[ ${key,,} == "netdevice" || ${key,,} == "device" ]]; then + netdevice=$value fi done if [[ -n $hostip && -n $netmask && -n $gateway && -n $netdevice ]]; then @@ -135,6 +138,12 @@ if [[ -n $hostip && -n $netmask && -n $gateway && -n $netdevice ]]; then ip addr add $hostip/$numbits broadcast $broadcast dev $netdevice scope global label $netdevice ip link set $netdevice up ip route replace to default via $gateway dev $netdevice + # in softlayer it takes up to 60 seconds for the nic to actually be able to communicate + echo -n Waiting to reach xCAT mgmt node $gateway. + xcatretries=60 + while [ $((xcati+=1)) -le $xcatretries ] && ! ping -c2 -w3 $gateway >/dev/null 2>&1; do echo -n .; done + if [ $xcati -le $xcatretries ]; then echo " success"; else echo " failed"; fi + sleep 3 else echo "Setting IP via DHCP..." dhclient -cf /etc/dhclient.conf -pf /var/run/dhclient.$bootnic.pid $bootnic &