2013-01-18 18:06:10 +00:00
#!/bin/sh
# IBM(c) 2013 EPL license http://www.eclipse.org/legal/epl-v10.html
# xCAT post script for configuring ib adaptors.
# The following are a sample of the env used:
2013-01-18 21:25:08 +00:00
# NIC_IBNICS=ib0,ib1
# NIC_IBAPORTS=1 (or 2)
2013-01-18 18:06:10 +00:00
# OSVER=sles11.1
2013-01-18 21:25:08 +00:00
# NICNODE=dxm360m4n04
2013-01-18 18:06:10 +00:00
# NICIPS='eth0!10.1.34.242,ib1!15.0.1.242,ib2!15.0.2.242'
# NICHOSTNAMESUFFIXES='eth1!-eth1,ib1!-ib1,ib2!-ib2'
# NICTYPES='eth1!ethernet,ib1!infiniband,ib2!infiniband'
# NICNETWORKS='eth0!10_1_0_0-255_255_255_0,ib1!ib1,ib2!ib2'
# NETWORKS_LINES=3
# NETWORKS_LINE1='netname=10_1_0_0-255_255_255_0||net=10.1.0.0||mask=255.255.255.0||mgtifname=eth1||gateway=<xcatmaster>||dhcpserver=10.1.0.205||tftpserver=10.1.0.205||nameservers=||ntpservers=||logservers=||dynamicrange=||staticrange=||staticrangeincrement=||nodehostname=||ddnsdomain=||vlanid=||domain=||disable=||comments='
# NETWORKS_LINE2='netname=ib1||net=15.0.1.0||mask=255.255.255.0||mgtifname=||gateway=15.0.1.254||dhcpserver=||tftpserver=||nameservers=||ntpservers=||logservers=||dynamicrange=||staticrange=||staticrangeincrement=||nodehostname=||ddnsdomain=||vlanid=||domain=||disable=||comments='
# NETWORKS_LINE3='netname=ib3||net=15.0.3.0||mask=255.255.255.0||mgtifname=||gateway=||dhcpserver=||tftpserver=||nameservers=||ntpservers=||logservers=||dynamicrange=||staticrange=||staticrangeincrement=||nodehostname=||ddnsdomain=||vlanid=||domain=||disable=||comments='
2013-01-18 21:25:08 +00:00
#This is the number of ports for each ib adpator.
2013-01-18 18:06:10 +00:00
portnum=1
2013-01-23 22:14:26 +00:00
if [ -n "$NIC_IBAPORTS" ]; then
2013-01-18 21:25:08 +00:00
portnum=$NIC_IBAPORTS
fi
2013-01-18 18:06:10 +00:00
# pmatch determines if 1st argument string is matched by 2nd argument pattern
pmatch ()
{
case $1 in
$2) return 0;; # zero return code means string matched by pattern
esac
return 1 # non-zero return code means string not matched by pattern
}
convert_netmask_to_cidr() {
let cidrnum=0
OLFIFS=$IFS
IFS=.
for x in $1
do
case $x in
255) cidrnum=$(expr $cidrnum + 8);;
254) cidrnum=$(expr $cidrnum + 7);;
252) cidrnum=$(expr $cidrnum + 6);;
248) cidrnum=$(expr $cidrnum + 5);;
240) cidrnum=$(expr $cidrnum + 4);;
224) cidrnum=$(expr $cidrnum + 3);;
192) cidrnum=$(expr $cidrnum + 2);;
128) cidrnum=$(expr $cidrnum + 1);;
0);;
esac
done
IFS=$OLDIFS
echo $cidrnum
}
2013-01-18 21:25:08 +00:00
if [ -z "$NIC_IBNICS" ]; then
2013-01-18 18:06:10 +00:00
echo "nothing to do."
fi
#find correct IB driver
PLTFRM=`uname`
if [[ $OSVER == rhels5* || "$OSVER" == rhels6* ]]
then
ib_driver="rdma"
/sbin/service $ib_driver status
if [ $? -eq 1 ]
then
ib_driver="openibd"
/sbin/service $ib_driver status
if [ $? -eq 1 ]
then
echo "Not found the driver dameon: rdma or openibd"
logger -p local4.info -t xcat "Not found the driver dameon: rdma or openibd"
exit
fi
fi
else
ib_driver="openibd"
fi
#make changes to the confiuration file
if [ $PLTFRM == "Linux" ]
then
if [ -f "/etc/rdma/rdma.conf" ]
then
sed -i "s/SDP_LOAD=yes/SDP_LOAD=no/g" /etc/rdma/rdma.conf
elif [ -f "/etc/infiniband/openib.conf" ]
then
sed -i "s/SDP_LOAD=yes/SDP_LOAD=no/g" /etc/infiniband/openib.conf
fi
if [ -f "/etc/modprobe.conf" ]
then
if [ "$portnum" == "1" ]; then
sed -i "/options ib_ehca nr_ports/d" /etc/modprobe.conf
echo 'options ib_ehca nr_ports=1' >> /etc/modprobe.conf
else
sed -i "/options ib_ehca nr_ports=1/d" /etc/modprobe.conf
fi
sed -i "/options ib_ehca lock_hcalls/d" /etc/modprobe.conf
echo 'options ib_ehca lock_hcalls=0' >> /etc/modprobe.conf
fi
/sbin/chkconfig --level 2345 $ib_driver on
/sbin/service $ib_driver restart
sysctl -p
fi
#clean up existing ib nic configrations
if [ $PLTFRM == "Linux" ]
then
if [ -f /etc/redhat-release ]
then
OS_name="redhat"
elif [ -f /etc/SuSE-release ]
then
OS_name="suse"
else
echo "Unsupported to config IB on this OS!"
logger -p local4.info -t xcat "Unsupported to config IB on this OS!"
exit
fi
if [ $OS_name == 'suse' ]
then
dir="/etc/sysconfig/network"
else
dir="/etc/sysconfig/network-scripts"
fi
2013-01-21 20:48:04 +00:00
for nic in `echo "$NIC_IBNICS" | tr "," "\n"`
do
`rm -f $dir/ifcfg-$nic`
done
2013-01-18 18:06:10 +00:00
else
2013-01-21 20:48:04 +00:00
for nic in `echo "$NIC_IBNICS" | tr "," "\n"`
do
`rmdev -d -l $nic >/dev/null 2>&1`
2013-01-18 18:06:10 +00:00
done
fi
goodnics=""
2013-01-18 21:25:08 +00:00
for nic in `echo "$NIC_IBNICS" | tr "," "\n"`
2013-01-18 18:06:10 +00:00
do
#Get nic ip
nicip=""
for tmp in `echo "$NICIPS" | tr "," "\n"`
do
nic_tmp=`echo "$tmp" | awk -F"!" '{print $1}'`;
if [ $nic_tmp == $nic ]; then
nicip=`echo "$tmp" | awk -F"!" '{print $2}'`;
break
fi
done
# echo "nic=$nic, nicip=$nicip"
#get nic network name
nicnet=""
for tmp in `echo "$NICNETWORKS" | tr "," "\n"`
do
nic_tmp=`echo "$tmp" | awk -F"!" '{print $1}'`;
if [ $nic_tmp == $nic ]; then
nicnet=`echo "$tmp" | awk -F"!" '{print $2}'`;
break
fi
done
#echo "nic=$nic, nicnet=$nicnet"
if [ -z "$nicnet" ]; then
echo "No network defined for $nic"
logger -p local4.info -t xcat "No network defined for $nic"
continue
fi
#get netmask and gateway
index=0;
found=0
while [ $index -lt $NETWORKS_LINES ]
do
index=$((index+1))
eval netline=\$NETWORKS_LINE$index
if [[ -n "$netline" ]]; then
for tmp in `echo "$netline" | tr "\|\|" "\n"`
do
key=`echo "$tmp" | awk -F"=" '{print $1}'`;
case $key in
netname)
netname=`echo "$tmp" | awk -F"=" '{print $2}'`
if [ "$netname" != "$nicnet" ];
then
break;
fi
;;
net)
net=`echo "$tmp" | awk -F"=" '{print $2}'`
;;
mask)
netmask=`echo "$tmp" | awk -F"=" '{print $2}'`
;;
gateway)
gateway=`echo "$tmp" | awk -F"=" '{print $2}'`
found=1
;;
esac
if [ $found -eq 1 ]; then
break;
fi
done
if [ $found -eq 1 ]; then
break;
fi
fi
done
#echo "found=$found"
#echo "index=$index netname=$nicnet net=$net mask=$netmask gateway=$gateway"
if [ "$found" == "0" ]; then
echo "Cannot find network $nicnet for $nic"
logger -p local4.info -t xcat "Cannot find network $nicnet for $nic"
continue
else
if [ -z "$goodnics" ]; then
goodnics=$nic
else
goodnics="$goodnics,$nic"
fi
fi
#there should be only one gateway on a node and that should go through the to the xcat management node and would be set up from the install nic. Anything else should be a route
if [ "$gateway" == "<xcatmaster>" ]; then
gateway=''
fi
if [ $PLTFRM == "Linux" ]
then
# Issue openibd for Linux at boot time
if [ -f /etc/sysctl.conf ]
then
sed -i "/net.ipv4.conf.$nic.arp_filter=1/d" /etc/sysctl.conf
sed -i "/net.ipv4.conf.$nic.arp_ignore=1/d" /etc/sysctl.conf
echo "net.ipv4.conf.$nic.arp_filter=1" >> /etc/sysctl.conf
echo "net.ipv4.conf.$nic.arp_ignore=1" >> /etc/sysctl.conf
fi
# Write the info to the ifcfg file
echo "DEVICE=$nic
BOOTPROTO=static
IPADDR=$nicip
NETMASK=$netmask" > $dir/ifcfg-$nic
if [ -n "$gateway" ]; then
echo "GATEWAY=$gateway" >> $dir/ifcfg-$nic
fi
if [[ "$OSVER" == rhels6* ]]
then
#get prefix from netmask, this is for IPv4 only
prefix=24
if [ -n "$netmask" ]; then
prefix=$(convert_netmask_to_cidr $netmask)
fi
echo "DEVICE=$nic
BOOTPROTO=static
IPADDR=$nicip
PREFIX=$prefix" > $dir/ifcfg-$nic
if [ -n "$gateway" ]; then
echo "GATEWAY=$gateway" >> $dir/ifcfg-$nic
fi
fi
if [ $OS_name == 'redhat' ]
then
echo "ONBOOT=yes" >> $dir/ifcfg-$nic
else
echo "STARTMODE=auto" >> $dir/ifcfg-$nic
fi
elif [ $PLTFRM == "AIX" ]; then
if ( pmatch $nic "ml*" ); then #for ml* interface
num=${nic##ml}
mlt="mlt$num"
#Check whether the mlt is available
lsdev -C | grep $mlt | grep Available 2>&1 >/dev/null
if [ $? -ne 0 ]
then
echo "$mltnum is not available."
logger -p local4.info -t xcat "$mltnum is not available."
continue
fi
#Check whether the ml0 is available
lsdev -C | grep $nic 2>&1 >/dev/null
if [ $? -ne 0 ]
then
cfgmgr 2>&1 >/dev/null
fi
chdev -l $nic -a state=detach 2>&1
chdev -l $nic -a netaddr=$nicip -a netmask=$netmask -a state=up 2>&1
else #assume it is ib*
lsdev -C | grep icm | grep Available
if [ $? -ne 0 ]
then
mkdev -c management -s infiniband -t icm
if [ $? -ne 0 ]
then
mkdev -l icm
if [ $? -ne 0 ]
then
exit $?
fi
fi
fi
#Configure the IB interfaces. Customize the port num.
num=${nic##ib} #this assumes that all the nics starts with 'ib'
if [ "$portnum" == "1" ]; then
iba_num=$num
ib_adapter="iba$iba_num"
port=1
else
iba_num=`expr $num / 2`
ib_adapter="iba$iba_num"
if [ $(($num % 2)) == 0 ]
then
port=1
else
port=2
fi
fi
mkiba -a $nicip -i $nic -A $ib_adapter -p $port -P -1 -S up -m $netmask
fi
fi
done
# echo "goodnics=$goodnics"
# Bringup all the ib interfaces
if [ $PLTFRM == "Linux" ]
then
/sbin/service $ib_driver restart
for nic in `echo "$goodnics" | tr "," "\n"`
do
sleep 5
if [[ "$OSVER" == rhels6* ]]
then
ip link set dev $nic up
else
ifup $nic
fi
done
fi