From 672fd91279c8c1fb63564c45b9d8ad8243adbc97 Mon Sep 17 00:00:00 2001 From: linggao Date: Fri, 18 Jan 2013 18:06:10 +0000 Subject: [PATCH] added configib postscript git-svn-id: https://svn.code.sf.net/p/xcat/code/xcat-core/trunk@14922 8638fb3e-16cb-4fca-ae20-7b5d299a9bcd --- xCAT/postscripts/configib | 376 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 376 insertions(+) create mode 100755 xCAT/postscripts/configib diff --git a/xCAT/postscripts/configib b/xCAT/postscripts/configib new file mode 100755 index 000000000..2b73b6d23 --- /dev/null +++ b/xCAT/postscripts/configib @@ -0,0 +1,376 @@ +#!/bin/sh +# IBM(c) 2013 EPL license http://www.eclipse.org/legal/epl-v10.html + +# xCAT post script for configuring ib adaptors. +# The following are a sample of the env used: +# IBNICS=ib0,ib1 +# OSVER=sles11.1 +# NIC related attributes for the node for confignics postscript +# NICNODE=dxm360m4n04 +# NICIPS='eth0!10.1.34.242,ib1!15.0.1.242,ib2!15.0.2.242' +# NICHOSTNAMESUFFIXES='eth1!-eth1,ib1!-ib1,ib2!-ib2' +# NICTYPES='eth1!ethernet,ib1!infiniband,ib2!infiniband' +# NICNETWORKS='eth0!10_1_0_0-255_255_255_0,ib1!ib1,ib2!ib2' +# NETWORKS_LINES=3 +# NETWORKS_LINE1='netname=10_1_0_0-255_255_255_0||net=10.1.0.0||mask=255.255.255.0||mgtifname=eth1||gateway=||dhcpserver=10.1.0.205||tftpserver=10.1.0.205||nameservers=||ntpservers=||logservers=||dynamicrange=||staticrange=||staticrangeincrement=||nodehostname=||ddnsdomain=||vlanid=||domain=||disable=||comments=' +# NETWORKS_LINE2='netname=ib1||net=15.0.1.0||mask=255.255.255.0||mgtifname=||gateway=15.0.1.254||dhcpserver=||tftpserver=||nameservers=||ntpservers=||logservers=||dynamicrange=||staticrange=||staticrangeincrement=||nodehostname=||ddnsdomain=||vlanid=||domain=||disable=||comments=' +# NETWORKS_LINE3='netname=ib3||net=15.0.3.0||mask=255.255.255.0||mgtifname=||gateway=||dhcpserver=||tftpserver=||nameservers=||ntpservers=||logservers=||dynamicrange=||staticrange=||staticrangeincrement=||nodehostname=||ddnsdomain=||vlanid=||domain=||disable=||comments=' + + +#This is the number of ports for each ib adpator. It is decided by the user +#The valid values are 1 or 2. +portnum=1 + +# pmatch determines if 1st argument string is matched by 2nd argument pattern +pmatch () +{ + case $1 in + $2) return 0;; # zero return code means string matched by pattern + esac + + return 1 # non-zero return code means string not matched by pattern +} + +convert_netmask_to_cidr() { + let cidrnum=0 + OLFIFS=$IFS + IFS=. + for x in $1 + do + case $x in + 255) cidrnum=$(expr $cidrnum + 8);; + 254) cidrnum=$(expr $cidrnum + 7);; + 252) cidrnum=$(expr $cidrnum + 6);; + 248) cidrnum=$(expr $cidrnum + 5);; + 240) cidrnum=$(expr $cidrnum + 4);; + 224) cidrnum=$(expr $cidrnum + 3);; + 192) cidrnum=$(expr $cidrnum + 2);; + 128) cidrnum=$(expr $cidrnum + 1);; + 0);; + esac + done + IFS=$OLDIFS + echo $cidrnum +} + + +if [ -z "$IBNICS" ]; then + echo "nothing to do." +fi + + +#find correct IB driver +PLTFRM=`uname` +if [[ $OSVER == rhels5* || "$OSVER" == rhels6* ]] +then + ib_driver="rdma" + /sbin/service $ib_driver status + if [ $? -eq 1 ] + then + ib_driver="openibd" + /sbin/service $ib_driver status + if [ $? -eq 1 ] + then + echo "Not found the driver dameon: rdma or openibd" + logger -p local4.info -t xcat "Not found the driver dameon: rdma or openibd" + exit + fi + fi +else + ib_driver="openibd" +fi + +#make changes to the confiuration file +if [ $PLTFRM == "Linux" ] +then + if [ -f "/etc/rdma/rdma.conf" ] + then + sed -i "s/SDP_LOAD=yes/SDP_LOAD=no/g" /etc/rdma/rdma.conf + elif [ -f "/etc/infiniband/openib.conf" ] + then + sed -i "s/SDP_LOAD=yes/SDP_LOAD=no/g" /etc/infiniband/openib.conf + fi + + if [ -f "/etc/modprobe.conf" ] + then + if [ "$portnum" == "1" ]; then + sed -i "/options ib_ehca nr_ports/d" /etc/modprobe.conf + echo 'options ib_ehca nr_ports=1' >> /etc/modprobe.conf + else + sed -i "/options ib_ehca nr_ports=1/d" /etc/modprobe.conf + fi + sed -i "/options ib_ehca lock_hcalls/d" /etc/modprobe.conf + echo 'options ib_ehca lock_hcalls=0' >> /etc/modprobe.conf + fi + /sbin/chkconfig --level 2345 $ib_driver on + /sbin/service $ib_driver restart + sysctl -p +fi + +#clean up existing ib nic configrations +if [ $PLTFRM == "Linux" ] +then + + if [ -f /etc/redhat-release ] + then + OS_name="redhat" + elif [ -f /etc/SuSE-release ] + then + OS_name="suse" + else + echo "Unsupported to config IB on this OS!" + logger -p local4.info -t xcat "Unsupported to config IB on this OS!" + exit + fi + + if [ $OS_name == 'suse' ] + then + dir="/etc/sysconfig/network" + else + dir="/etc/sysconfig/network-scripts" + fi + `rm -f $dir/ifcfg-ib*` +else + loop_number=`lsdev | grep "IP over Infiniband Network Interface" | wc -l` + + num=0 + while [ $num -lt $loop_number ] + do + if [ "$portnum" == "1" ]; then + `rmdev -d -l ib$num >/dev/null 2>&1` + n=`expr $num % 2` + if [ $n == 1 ] + then + mlnum=`expr $num / 2` + `rmdev -d -l ml$mlnum >/dev/null 2>&1` + fi + else + num0=`expr $num \* 2` + num1=`expr $num0 + 1` + `rmdev -d -l ib$num0 >/dev/null 2>&1` + `rmdev -d -l ib$num1 >/dev/null 2>&1` + `rmdev -d -l ml$num >/dev/null 2>&1` + fi + num=`expr $num + 1` + done +fi + + + +HOST=`hostname -s` +goodnics="" +for nic in `echo "$IBNICS" | tr "," "\n"` +do + #Get nic ip + nicip="" + for tmp in `echo "$NICIPS" | tr "," "\n"` + do + nic_tmp=`echo "$tmp" | awk -F"!" '{print $1}'`; + if [ $nic_tmp == $nic ]; then + nicip=`echo "$tmp" | awk -F"!" '{print $2}'`; + break + fi + done + # echo "nic=$nic, nicip=$nicip" + + #get nic network name + nicnet="" + for tmp in `echo "$NICNETWORKS" | tr "," "\n"` + do + nic_tmp=`echo "$tmp" | awk -F"!" '{print $1}'`; + if [ $nic_tmp == $nic ]; then + nicnet=`echo "$tmp" | awk -F"!" '{print $2}'`; + break + fi + done + #echo "nic=$nic, nicnet=$nicnet" + + if [ -z "$nicnet" ]; then + echo "No network defined for $nic" + logger -p local4.info -t xcat "No network defined for $nic" + continue + fi + + #get netmask and gateway + index=0; + found=0 + while [ $index -lt $NETWORKS_LINES ] + do + index=$((index+1)) + eval netline=\$NETWORKS_LINE$index + if [[ -n "$netline" ]]; then + for tmp in `echo "$netline" | tr "\|\|" "\n"` + do + key=`echo "$tmp" | awk -F"=" '{print $1}'`; + case $key in + netname) + netname=`echo "$tmp" | awk -F"=" '{print $2}'` + if [ "$netname" != "$nicnet" ]; + then + break; + fi + ;; + net) + net=`echo "$tmp" | awk -F"=" '{print $2}'` + ;; + mask) + netmask=`echo "$tmp" | awk -F"=" '{print $2}'` + ;; + gateway) + gateway=`echo "$tmp" | awk -F"=" '{print $2}'` + found=1 + ;; + esac + if [ $found -eq 1 ]; then + break; + fi + done + if [ $found -eq 1 ]; then + break; + fi + fi + done + #echo "found=$found" + #echo "index=$index netname=$nicnet net=$net mask=$netmask gateway=$gateway" + + if [ "$found" == "0" ]; then + echo "Cannot find network $nicnet for $nic" + logger -p local4.info -t xcat "Cannot find network $nicnet for $nic" + continue + else + if [ -z "$goodnics" ]; then + goodnics=$nic + else + goodnics="$goodnics,$nic" + fi + fi + + #there should be only one gateway on a node and that should go through the to the xcat management node and would be set up from the install nic. Anything else should be a route + if [ "$gateway" == "" ]; then + gateway='' + fi + + + if [ $PLTFRM == "Linux" ] + then + # Issue openibd for Linux at boot time + + if [ -f /etc/sysctl.conf ] + then + sed -i "/net.ipv4.conf.$nic.arp_filter=1/d" /etc/sysctl.conf + sed -i "/net.ipv4.conf.$nic.arp_ignore=1/d" /etc/sysctl.conf + echo "net.ipv4.conf.$nic.arp_filter=1" >> /etc/sysctl.conf + echo "net.ipv4.conf.$nic.arp_ignore=1" >> /etc/sysctl.conf + fi + + # Write the info to the ifcfg file + echo "DEVICE=$nic +BOOTPROTO=static +IPADDR=$nicip +NETMASK=$netmask" > $dir/ifcfg-$nic + if [ -n "$gateway" ]; then + echo "GATEWAY=$gateway" >> $dir/ifcfg-$nic + fi + + if [[ "$OSVER" == rhels6* ]] + then + #get prefix from netmask, this is for IPv4 only + prefix=24 + if [ -n "$netmask" ]; then + prefix=$(convert_netmask_to_cidr $netmask) + fi + + echo "DEVICE=$nic +BOOTPROTO=static +IPADDR=$nicip +PREFIX=$prefix" > $dir/ifcfg-$nic + if [ -n "$gateway" ]; then + echo "GATEWAY=$gateway" >> $dir/ifcfg-$nic + fi + fi + + if [ $OS_name == 'redhat' ] + then + echo "ONBOOT=yes" >> $dir/ifcfg-$nic + else + echo "STARTMODE=auto" >> $dir/ifcfg-$nic + fi + elif [ $PLTFRM == "AIX" ]; then + if ( pmatch $nic "ml*" ); then #for ml* interface + num=${nic##ml} + mlt="mlt$num" + #Check whether the mlt is available + lsdev -C | grep $mlt | grep Available 2>&1 >/dev/null + if [ $? -ne 0 ] + then + echo "$mltnum is not available." + logger -p local4.info -t xcat "$mltnum is not available." + continue + fi + + #Check whether the ml0 is available + lsdev -C | grep $nic 2>&1 >/dev/null + if [ $? -ne 0 ] + then + cfgmgr 2>&1 >/dev/null + fi + + chdev -l $nic -a state=detach 2>&1 + chdev -l $nic -a netaddr=$nicip -a netmask=$netmask -a state=up 2>&1 + else #assume it is ib* + lsdev -C | grep icm | grep Available + if [ $? -ne 0 ] + then + mkdev -c management -s infiniband -t icm + if [ $? -ne 0 ] + then + mkdev -l icm + if [ $? -ne 0 ] + then + exit $? + fi + fi + fi + + #Configure the IB interfaces. Customize the port num. + num=${nic##ib} #this assumes that all the nics starts with 'ib' + if [ "$portnum" == "1" ]; then + iba_num=$num + ib_adapter="iba$iba_num" + port=1 + else + iba_num=`expr $num / 2` + ib_adapter="iba$iba_num" + if [ $(($num % 2)) == 0 ] + then + port=1 + else + port=2 + fi + fi + mkiba -a $nicip -i $nic -A $ib_adapter -p $port -P -1 -S up -m $netmask + fi + fi +done + +# echo "goodnics=$goodnics" +# Bringup all the ib interfaces +if [ $PLTFRM == "Linux" ] +then + + /sbin/service $ib_driver restart + sleep 10 + for nic in `echo "$goodnics" | tr "," "\n"` + do + sleep 5 + + if [[ "$OSVER" == rhels6* ]] + then + ip link set dev $nic up + else + ifup $nic + fi + done +fi + +