xcat-core/xCAT/postscripts/configib

#!/bin/sh 
# IBM(c) 2013 EPL license http://www.eclipse.org/legal/epl-v10.html

# xCAT post script for configuring ib adaptors.
# The following are a sample of the env used:
# NIC_IBNICS=ib0,ib1
# NIC_IBAPORTS=1 (or 2)
# OSVER=sles11.1
# NICNODE=dxm360m4n04 
# NICIPS='eth0!10.1.34.242,ib1!15.0.1.242,ib2!15.0.2.242'
# NICHOSTNAMESUFFIXES='eth1!-eth1,ib1!-ib1,ib2!-ib2'
# NICTYPES='eth1!ethernet,ib1!infiniband,ib2!infiniband'
# NICNETWORKS='eth0!10_1_0_0-255_255_255_0,ib1!ib1,ib2!ib2'
# NETWORKS_LINES=3
# NETWORKS_LINE1='netname=10_1_0_0-255_255_255_0||net=10.1.0.0||mask=255.255.255.0||mgtifname=eth1||gateway=<xcatmaster>||dhcpserver=10.1.0.205||tftpserver=10.1.0.205||nameservers=||ntpservers=||logservers=||dynamicrange=||staticrange=||staticrangeincrement=||nodehostname=||ddnsdomain=||vlanid=||domain=||disable=||comments='
# NETWORKS_LINE2='netname=ib1||net=15.0.1.0||mask=255.255.255.0||mgtifname=||gateway=15.0.1.254||dhcpserver=||tftpserver=||nameservers=||ntpservers=||logservers=||dynamicrange=||staticrange=||staticrangeincrement=||nodehostname=||ddnsdomain=||vlanid=||domain=||disable=||comments='
# NETWORKS_LINE3='netname=ib3||net=15.0.3.0||mask=255.255.255.0||mgtifname=||gateway=||dhcpserver=||tftpserver=||nameservers=||ntpservers=||logservers=||dynamicrange=||staticrange=||staticrangeincrement=||nodehostname=||ddnsdomain=||vlanid=||domain=||disable=||comments='


#This is the number of ports for each ib adpator.
portnum=1
if [ -n "$NIC_IBAPORTS" ]; then
    portnum=$NIC_IBAPORTS
fi

# pmatch determines if 1st argument string is matched by 2nd argument pattern
pmatch ()
{
  case $1 in
    $2) return 0;;  # zero return code means string matched by pattern
  esac

  return 1          # non-zero return code means string not matched by pattern
}

convert_netmask_to_cidr() {
    let cidrnum=0
    OLFIFS=$IFS
    IFS=.
    for x in $1
    do
        case $x in
            255) cidrnum=$(expr $cidrnum + 8);;
            254) cidrnum=$(expr $cidrnum + 7);;
            252) cidrnum=$(expr $cidrnum + 6);;
            248) cidrnum=$(expr $cidrnum + 5);;
            240) cidrnum=$(expr $cidrnum + 4);;
            224) cidrnum=$(expr $cidrnum + 3);;
            192) cidrnum=$(expr $cidrnum + 2);;
            128) cidrnum=$(expr $cidrnum + 1);;
            0);;
        esac
    done
    IFS=$OLDIFS
    echo $cidrnum
}
 

if [ -z "$NIC_IBNICS" ]; then
    echo "nothing to do."
fi


#find correct IB driver
PLTFRM=`uname`
if [[ $OSVER == rhels5*  || "$OSVER" == rhels6* ]]
then
    ib_driver="rdma"
    /sbin/service $ib_driver status
    if [ $? -eq 1 ]
    then
        ib_driver="openibd"
        /sbin/service $ib_driver status
        if [ $? -eq 1 ]
        then
            echo "Not found the driver dameon: rdma or openibd"
	    logger -p local4.info -t xcat "Not found the driver dameon: rdma or openibd"
            exit
        fi
    fi   
else
    ib_driver="openibd"
fi

#make changes to the confiuration file
if [ $PLTFRM == "Linux" ]
then
    if [ -f "/etc/rdma/rdma.conf" ]
    then
        sed -i "s/SDP_LOAD=yes/SDP_LOAD=no/g" /etc/rdma/rdma.conf
    elif [ -f "/etc/infiniband/openib.conf" ]
    then
        sed -i "s/SDP_LOAD=yes/SDP_LOAD=no/g" /etc/infiniband/openib.conf
    fi

    if [ -f "/etc/modprobe.conf" ]
    then
	if [ "$portnum" == "1" ]; then
            sed -i "/options ib_ehca nr_ports/d" /etc/modprobe.conf
            echo 'options ib_ehca nr_ports=1' >> /etc/modprobe.conf
	else
            sed -i "/options ib_ehca nr_ports=1/d" /etc/modprobe.conf
	fi
        sed -i "/options ib_ehca lock_hcalls/d" /etc/modprobe.conf
        echo 'options ib_ehca lock_hcalls=0' >> /etc/modprobe.conf
    fi
    /sbin/chkconfig --level 2345 $ib_driver on
    /sbin/service $ib_driver restart
    sysctl -p
fi

#clean up existing ib nic configrations
if [ $PLTFRM == "Linux" ]
then
    if [ -f /etc/redhat-release ]
    then
        OS_name="redhat"
    elif [ -f /etc/SuSE-release ]
    then
        OS_name="suse"
    else
        echo "Unsupported to config IB on this OS!"
	logger -p local4.info -t xcat "Unsupported to config IB on this OS!"
        exit
     fi
 
    if [ $OS_name == 'suse' ]
    then
         dir="/etc/sysconfig/network"
    else
         dir="/etc/sysconfig/network-scripts"
    fi
    for nic in `echo "$NIC_IBNICS" | tr "," "\n"`
    do     
	`rm -f $dir/ifcfg-$nic`
    done
else
    for nic in `echo "$NIC_IBNICS" | tr "," "\n"`
    do     
	`rmdev -d -l $nic >/dev/null 2>&1`
    done
fi


goodnics=""
for nic in `echo "$NIC_IBNICS" | tr "," "\n"`
do 
    #Get nic ip 
    nicip=""
    for tmp in `echo "$NICIPS" | tr "," "\n"`
    do
	nic_tmp=`echo "$tmp" | awk -F"!" '{print $1}'`;
	if [ $nic_tmp == $nic ]; then
	    nicip=`echo "$tmp" | awk -F"!" '{print $2}'`;
	    break
	fi
    done
   # echo "nic=$nic, nicip=$nicip"
    
   #get nic network name
    nicnet=""
   for tmp in `echo "$NICNETWORKS" | tr "," "\n"`
   do
       nic_tmp=`echo "$tmp" | awk -F"!" '{print $1}'`;
       if [ $nic_tmp == $nic ]; then
	   nicnet=`echo "$tmp" | awk -F"!" '{print $2}'`;
	   break
       fi
   done
   #echo "nic=$nic, nicnet=$nicnet"
   
   if [ -z "$nicnet" ]; then
       echo "No network defined for $nic"
       logger -p local4.info -t xcat "No network defined for $nic"
       continue
   fi   

   #get netmask and gateway
   index=0;
   found=0
   while [ $index -lt $NETWORKS_LINES ]
   do
       index=$((index+1))
       eval netline=\$NETWORKS_LINE$index
       if [[ -n "$netline" ]]; then
	   for tmp in `echo "$netline" | tr "\|" "\n"`
	   do
	       key=`echo "$tmp" | awk -F"=" '{print $1}'`;
	       case $key in
		   netname)
		       netname=`echo "$tmp" | awk -F"=" '{print $2}'`
		       if [ "$netname" != "$nicnet" ];
		       then
			   break;
		       fi
		       ;;
		   net)  
		       net=`echo "$tmp" | awk -F"=" '{print $2}'`
		       ;;
		   mask) 
		       netmask=`echo "$tmp" | awk -F"=" '{print $2}'`
		       ;; 
		   gateway) 
		       gateway=`echo "$tmp" | awk -F"=" '{print $2}'`
		       found=1
		       ;;
	       esac 
	       if [ $found -eq 1 ]; then
		   break;
	       fi
	   done
	   if [ $found -eq 1 ]; then
	       break;
	   fi
       fi
   done
   #echo "found=$found"
   #echo "index=$index netname=$nicnet net=$net mask=$netmask gateway=$gateway"

   if [ "$found" == "0" ]; then
       echo "Cannot find network $nicnet for $nic"
       logger -p local4.info -t xcat "Cannot find network $nicnet for $nic"
       continue
   else
       if [ -z "$goodnics" ]; then
	   goodnics=$nic
       else
	   goodnics="$goodnics,$nic"
       fi
   fi

   #there should be only one gateway on a node and that should go through the to the xcat management node and would be set up from the install nic. Anything else should be a route
   if [ "$gateway" == "<xcatmaster>" ]; then
       gateway=''
   fi

   
   if [ $PLTFRM == "Linux" ]
   then
       # Issue openibd for Linux at boot time
       
       if [ -f /etc/sysctl.conf ]
       then
           sed -i "/net.ipv4.conf.$nic.arp_filter=1/d" /etc/sysctl.conf
           sed -i "/net.ipv4.conf.$nic.arp_ignore=1/d" /etc/sysctl.conf
           echo "net.ipv4.conf.$nic.arp_filter=1" >> /etc/sysctl.conf
           echo "net.ipv4.conf.$nic.arp_ignore=1" >> /etc/sysctl.conf
       fi
       
       # Write the info to the ifcfg file
       echo "DEVICE=$nic
BOOTPROTO=static
IPADDR=$nicip
NETMASK=$netmask" > $dir/ifcfg-$nic
       if [ -n "$gateway" ]; then
	   echo "GATEWAY=$gateway" >> $dir/ifcfg-$nic
       fi
       
       if [[ "$OSVER" == rhels6* ]]
       then 
           #get prefix from netmask, this is for IPv4 only
	   prefix=24
	   if [ -n "$netmask" ]; then
	        prefix=$(convert_netmask_to_cidr $netmask)
	   fi   
        
	   echo "DEVICE=$nic
BOOTPROTO=static
IPADDR=$nicip
PREFIX=$prefix" > $dir/ifcfg-$nic
	   if [ -n "$gateway" ]; then
	       echo "GATEWAY=$gateway" >>  $dir/ifcfg-$nic
	   fi
       fi
    
       if [ $OS_name == 'redhat' ]
       then
           echo "ONBOOT=yes" >> $dir/ifcfg-$nic
       else 
           echo "STARTMODE=auto" >> $dir/ifcfg-$nic
       fi
   elif [ $PLTFRM == "AIX" ]; then
       if ( pmatch $nic "ml*" ); then #for ml* interface
	   num=${nic##ml}
	   mlt="mlt$num"
           #Check whether the mlt is available
	   lsdev -C | grep $mlt | grep Available 2>&1 >/dev/null
	   if [ $? -ne 0 ]
	   then
               echo "$mltnum is not available."
	       logger -p local4.info -t xcat "$mltnum is not available."
	       continue
	   fi
	   
           #Check whether the ml0 is available
	   lsdev -C | grep $nic 2>&1 >/dev/null
	   if [ $? -ne 0 ]
	   then
               cfgmgr 2>&1 >/dev/null
	   fi
	   
	   chdev -l $nic -a state=detach 2>&1
	   chdev -l $nic -a netaddr=$nicip -a netmask=$netmask -a state=up 2>&1
       else #assume it is ib*
	   lsdev -C | grep icm | grep Available
	   if [ $? -ne 0 ]
	   then
               mkdev -c management -s infiniband -t icm
               if [ $? -ne 0 ]
               then
		   mkdev -l icm
		   if [ $? -ne 0 ]
		   then
                       exit $?
		   fi
               fi
	   fi
	   
           #Configure the IB interfaces.  Customize the port num.
	   num=${nic##ib}   #this assumes that all the nics starts with 'ib'
	   if [ "$portnum" == "1" ]; then 
	       iba_num=$num
               ib_adapter="iba$iba_num"
               port=1
	   else
	       iba_num=`expr $num / 2`
	       ib_adapter="iba$iba_num"
	       if [ $(($num % 2)) == 0 ]
	       then
		   port=1
	       else
		   port=2
	       fi
	   fi
	   mkiba -a $nicip -i $nic -A $ib_adapter -p $port -P -1 -S up -m $netmask
       fi
   fi
done

# echo "goodnics=$goodnics"
# Bringup all the ib interfaces
if [ $PLTFRM == "Linux" ]
then

    /sbin/service $ib_driver restart
    for nic in `echo "$goodnics" | tr "," "\n"`
    do     
        sleep 5
             
        if [[ "$OSVER" == rhels6* ]]
        then
            ip link set dev $nic up
        else 
            ifup $nic     
        fi
    done
fi