377 lines
10 KiB
Plaintext
377 lines
10 KiB
Plaintext
|
#!/bin/sh
|
||
|
|
# IBM(c) 2013 EPL license http://www.eclipse.org/legal/epl-v10.html
|
||
|
|
|
||
|
|
# xCAT post script for configuring ib adaptors.
|
||
|
|
# The following are a sample of the env used:
|
||
|
|
# IBNICS=ib0,ib1
|
||
|
|
# OSVER=sles11.1
|
||
|
|
# NIC related attributes for the node for confignics postscript
|
||
|
|
# NICNODE=dxm360m4n04
|
||
|
|
# NICIPS='eth0!10.1.34.242,ib1!15.0.1.242,ib2!15.0.2.242'
|
||
|
|
# NICHOSTNAMESUFFIXES='eth1!-eth1,ib1!-ib1,ib2!-ib2'
|
||
|
|
# NICTYPES='eth1!ethernet,ib1!infiniband,ib2!infiniband'
|
||
|
|
# NICNETWORKS='eth0!10_1_0_0-255_255_255_0,ib1!ib1,ib2!ib2'
|
||
|
|
# NETWORKS_LINES=3
|
||
|
|
# NETWORKS_LINE1='netname=10_1_0_0-255_255_255_0||net=10.1.0.0||mask=255.255.255.0||mgtifname=eth1||gateway=<xcatmaster>||dhcpserver=10.1.0.205||tftpserver=10.1.0.205||nameservers=||ntpservers=||logservers=||dynamicrange=||staticrange=||staticrangeincrement=||nodehostname=||ddnsdomain=||vlanid=||domain=||disable=||comments='
|
||
|
|
# NETWORKS_LINE2='netname=ib1||net=15.0.1.0||mask=255.255.255.0||mgtifname=||gateway=15.0.1.254||dhcpserver=||tftpserver=||nameservers=||ntpservers=||logservers=||dynamicrange=||staticrange=||staticrangeincrement=||nodehostname=||ddnsdomain=||vlanid=||domain=||disable=||comments='
|
||
|
|
# NETWORKS_LINE3='netname=ib3||net=15.0.3.0||mask=255.255.255.0||mgtifname=||gateway=||dhcpserver=||tftpserver=||nameservers=||ntpservers=||logservers=||dynamicrange=||staticrange=||staticrangeincrement=||nodehostname=||ddnsdomain=||vlanid=||domain=||disable=||comments='
|
||
|
|
|
||
|
|
|
||
|
|
#This is the number of ports for each ib adpator. It is decided by the user
|
||
|
|
#The valid values are 1 or 2.
|
||
|
|
portnum=1
|
||
|
|
|
||
|
|
# pmatch determines if 1st argument string is matched by 2nd argument pattern
|
||
|
|
pmatch ()
|
||
|
|
{
|
||
|
|
case $1 in
|
||
|
|
$2) return 0;; # zero return code means string matched by pattern
|
||
|
|
esac
|
||
|
|
|
||
|
|
return 1 # non-zero return code means string not matched by pattern
|
||
|
|
}
|
||
|
|
|
||
|
|
convert_netmask_to_cidr() {
|
||
|
|
let cidrnum=0
|
||
|
|
OLFIFS=$IFS
|
||
|
|
IFS=.
|
||
|
|
for x in $1
|
||
|
|
do
|
||
|
|
case $x in
|
||
|
|
255) cidrnum=$(expr $cidrnum + 8);;
|
||
|
|
254) cidrnum=$(expr $cidrnum + 7);;
|
||
|
|
252) cidrnum=$(expr $cidrnum + 6);;
|
||
|
|
248) cidrnum=$(expr $cidrnum + 5);;
|
||
|
|
240) cidrnum=$(expr $cidrnum + 4);;
|
||
|
|
224) cidrnum=$(expr $cidrnum + 3);;
|
||
|
|
192) cidrnum=$(expr $cidrnum + 2);;
|
||
|
|
128) cidrnum=$(expr $cidrnum + 1);;
|
||
|
|
0);;
|
||
|
|
esac
|
||
|
|
done
|
||
|
|
IFS=$OLDIFS
|
||
|
|
echo $cidrnum
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
if [ -z "$IBNICS" ]; then
|
||
|
|
echo "nothing to do."
|
||
|
|
fi
|
||
|
|
|
||
|
|
|
||
|
|
#find correct IB driver
|
||
|
|
PLTFRM=`uname`
|
||
|
|
if [[ $OSVER == rhels5* || "$OSVER" == rhels6* ]]
|
||
|
|
then
|
||
|
|
ib_driver="rdma"
|
||
|
|
/sbin/service $ib_driver status
|
||
|
|
if [ $? -eq 1 ]
|
||
|
|
then
|
||
|
|
ib_driver="openibd"
|
||
|
|
/sbin/service $ib_driver status
|
||
|
|
if [ $? -eq 1 ]
|
||
|
|
then
|
||
|
|
echo "Not found the driver dameon: rdma or openibd"
|
||
|
|
logger -p local4.info -t xcat "Not found the driver dameon: rdma or openibd"
|
||
|
|
exit
|
||
|
|
fi
|
||
|
|
fi
|
||
|
|
else
|
||
|
|
ib_driver="openibd"
|
||
|
|
fi
|
||
|
|
|
||
|
|
#make changes to the confiuration file
|
||
|
|
if [ $PLTFRM == "Linux" ]
|
||
|
|
then
|
||
|
|
if [ -f "/etc/rdma/rdma.conf" ]
|
||
|
|
then
|
||
|
|
sed -i "s/SDP_LOAD=yes/SDP_LOAD=no/g" /etc/rdma/rdma.conf
|
||
|
|
elif [ -f "/etc/infiniband/openib.conf" ]
|
||
|
|
then
|
||
|
|
sed -i "s/SDP_LOAD=yes/SDP_LOAD=no/g" /etc/infiniband/openib.conf
|
||
|
|
fi
|
||
|
|
|
||
|
|
if [ -f "/etc/modprobe.conf" ]
|
||
|
|
then
|
||
|
|
if [ "$portnum" == "1" ]; then
|
||
|
|
sed -i "/options ib_ehca nr_ports/d" /etc/modprobe.conf
|
||
|
|
echo 'options ib_ehca nr_ports=1' >> /etc/modprobe.conf
|
||
|
|
else
|
||
|
|
sed -i "/options ib_ehca nr_ports=1/d" /etc/modprobe.conf
|
||
|
|
fi
|
||
|
|
sed -i "/options ib_ehca lock_hcalls/d" /etc/modprobe.conf
|
||
|
|
echo 'options ib_ehca lock_hcalls=0' >> /etc/modprobe.conf
|
||
|
|
fi
|
||
|
|
/sbin/chkconfig --level 2345 $ib_driver on
|
||
|
|
/sbin/service $ib_driver restart
|
||
|
|
sysctl -p
|
||
|
|
fi
|
||
|
|
|
||
|
|
#clean up existing ib nic configrations
|
||
|
|
if [ $PLTFRM == "Linux" ]
|
||
|
|
then
|
||
|
|
|
||
|
|
if [ -f /etc/redhat-release ]
|
||
|
|
then
|
||
|
|
OS_name="redhat"
|
||
|
|
elif [ -f /etc/SuSE-release ]
|
||
|
|
then
|
||
|
|
OS_name="suse"
|
||
|
|
else
|
||
|
|
echo "Unsupported to config IB on this OS!"
|
||
|
|
logger -p local4.info -t xcat "Unsupported to config IB on this OS!"
|
||
|
|
exit
|
||
|
|
fi
|
||
|
|
|
||
|
|
if [ $OS_name == 'suse' ]
|
||
|
|
then
|
||
|
|
dir="/etc/sysconfig/network"
|
||
|
|
else
|
||
|
|
dir="/etc/sysconfig/network-scripts"
|
||
|
|
fi
|
||
|
|
`rm -f $dir/ifcfg-ib*`
|
||
|
|
else
|
||
|
|
loop_number=`lsdev | grep "IP over Infiniband Network Interface" | wc -l`
|
||
|
|
|
||
|
|
num=0
|
||
|
|
while [ $num -lt $loop_number ]
|
||
|
|
do
|
||
|
|
if [ "$portnum" == "1" ]; then
|
||
|
|
`rmdev -d -l ib$num >/dev/null 2>&1`
|
||
|
|
n=`expr $num % 2`
|
||
|
|
if [ $n == 1 ]
|
||
|
|
then
|
||
|
|
mlnum=`expr $num / 2`
|
||
|
|
`rmdev -d -l ml$mlnum >/dev/null 2>&1`
|
||
|
|
fi
|
||
|
|
else
|
||
|
|
num0=`expr $num \* 2`
|
||
|
|
num1=`expr $num0 + 1`
|
||
|
|
`rmdev -d -l ib$num0 >/dev/null 2>&1`
|
||
|
|
`rmdev -d -l ib$num1 >/dev/null 2>&1`
|
||
|
|
`rmdev -d -l ml$num >/dev/null 2>&1`
|
||
|
|
fi
|
||
|
|
num=`expr $num + 1`
|
||
|
|
done
|
||
|
|
fi
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
HOST=`hostname -s`
|
||
|
|
goodnics=""
|
||
|
|
for nic in `echo "$IBNICS" | tr "," "\n"`
|
||
|
|
do
|
||
|
|
#Get nic ip
|
||
|
|
nicip=""
|
||
|
|
for tmp in `echo "$NICIPS" | tr "," "\n"`
|
||
|
|
do
|
||
|
|
nic_tmp=`echo "$tmp" | awk -F"!" '{print $1}'`;
|
||
|
|
if [ $nic_tmp == $nic ]; then
|
||
|
|
nicip=`echo "$tmp" | awk -F"!" '{print $2}'`;
|
||
|
|
break
|
||
|
|
fi
|
||
|
|
done
|
||
|
|
# echo "nic=$nic, nicip=$nicip"
|
||
|
|
|
||
|
|
#get nic network name
|
||
|
|
nicnet=""
|
||
|
|
for tmp in `echo "$NICNETWORKS" | tr "," "\n"`
|
||
|
|
do
|
||
|
|
nic_tmp=`echo "$tmp" | awk -F"!" '{print $1}'`;
|
||
|
|
if [ $nic_tmp == $nic ]; then
|
||
|
|
nicnet=`echo "$tmp" | awk -F"!" '{print $2}'`;
|
||
|
|
break
|
||
|
|
fi
|
||
|
|
done
|
||
|
|
#echo "nic=$nic, nicnet=$nicnet"
|
||
|
|
|
||
|
|
if [ -z "$nicnet" ]; then
|
||
|
|
echo "No network defined for $nic"
|
||
|
|
logger -p local4.info -t xcat "No network defined for $nic"
|
||
|
|
continue
|
||
|
|
fi
|
||
|
|
|
||
|
|
#get netmask and gateway
|
||
|
|
index=0;
|
||
|
|
found=0
|
||
|
|
while [ $index -lt $NETWORKS_LINES ]
|
||
|
|
do
|
||
|
|
index=$((index+1))
|
||
|
|
eval netline=\$NETWORKS_LINE$index
|
||
|
|
if [[ -n "$netline" ]]; then
|
||
|
|
for tmp in `echo "$netline" | tr "\|\|" "\n"`
|
||
|
|
do
|
||
|
|
key=`echo "$tmp" | awk -F"=" '{print $1}'`;
|
||
|
|
case $key in
|
||
|
|
netname)
|
||
|
|
netname=`echo "$tmp" | awk -F"=" '{print $2}'`
|
||
|
|
if [ "$netname" != "$nicnet" ];
|
||
|
|
then
|
||
|
|
break;
|
||
|
|
fi
|
||
|
|
;;
|
||
|
|
net)
|
||
|
|
net=`echo "$tmp" | awk -F"=" '{print $2}'`
|
||
|
|
;;
|
||
|
|
mask)
|
||
|
|
netmask=`echo "$tmp" | awk -F"=" '{print $2}'`
|
||
|
|
;;
|
||
|
|
gateway)
|
||
|
|
gateway=`echo "$tmp" | awk -F"=" '{print $2}'`
|
||
|
|
found=1
|
||
|
|
;;
|
||
|
|
esac
|
||
|
|
if [ $found -eq 1 ]; then
|
||
|
|
break;
|
||
|
|
fi
|
||
|
|
done
|
||
|
|
if [ $found -eq 1 ]; then
|
||
|
|
break;
|
||
|
|
fi
|
||
|
|
fi
|
||
|
|
done
|
||
|
|
#echo "found=$found"
|
||
|
|
#echo "index=$index netname=$nicnet net=$net mask=$netmask gateway=$gateway"
|
||
|
|
|
||
|
|
if [ "$found" == "0" ]; then
|
||
|
|
echo "Cannot find network $nicnet for $nic"
|
||
|
|
logger -p local4.info -t xcat "Cannot find network $nicnet for $nic"
|
||
|
|
continue
|
||
|
|
else
|
||
|
|
if [ -z "$goodnics" ]; then
|
||
|
|
goodnics=$nic
|
||
|
|
else
|
||
|
|
goodnics="$goodnics,$nic"
|
||
|
|
fi
|
||
|
|
fi
|
||
|
|
|
||
|
|
#there should be only one gateway on a node and that should go through the to the xcat management node and would be set up from the install nic. Anything else should be a route
|
||
|
|
if [ "$gateway" == "<xcatmaster>" ]; then
|
||
|
|
gateway=''
|
||
|
|
fi
|
||
|
|
|
||
|
|
|
||
|
|
if [ $PLTFRM == "Linux" ]
|
||
|
|
then
|
||
|
|
# Issue openibd for Linux at boot time
|
||
|
|
|
||
|
|
if [ -f /etc/sysctl.conf ]
|
||
|
|
then
|
||
|
|
sed -i "/net.ipv4.conf.$nic.arp_filter=1/d" /etc/sysctl.conf
|
||
|
|
sed -i "/net.ipv4.conf.$nic.arp_ignore=1/d" /etc/sysctl.conf
|
||
|
|
echo "net.ipv4.conf.$nic.arp_filter=1" >> /etc/sysctl.conf
|
||
|
|
echo "net.ipv4.conf.$nic.arp_ignore=1" >> /etc/sysctl.conf
|
||
|
|
fi
|
||
|
|
|
||
|
|
# Write the info to the ifcfg file
|
||
|
|
echo "DEVICE=$nic
|
||
|
|
BOOTPROTO=static
|
||
|
|
IPADDR=$nicip
|
||
|
|
NETMASK=$netmask" > $dir/ifcfg-$nic
|
||
|
|
if [ -n "$gateway" ]; then
|
||
|
|
echo "GATEWAY=$gateway" >> $dir/ifcfg-$nic
|
||
|
|
fi
|
||
|
|
|
||
|
|
if [[ "$OSVER" == rhels6* ]]
|
||
|
|
then
|
||
|
|
#get prefix from netmask, this is for IPv4 only
|
||
|
|
prefix=24
|
||
|
|
if [ -n "$netmask" ]; then
|
||
|
|
prefix=$(convert_netmask_to_cidr $netmask)
|
||
|
|
fi
|
||
|
|
|
||
|
|
echo "DEVICE=$nic
|
||
|
|
BOOTPROTO=static
|
||
|
|
IPADDR=$nicip
|
||
|
|
PREFIX=$prefix" > $dir/ifcfg-$nic
|
||
|
|
if [ -n "$gateway" ]; then
|
||
|
|
echo "GATEWAY=$gateway" >> $dir/ifcfg-$nic
|
||
|
|
fi
|
||
|
|
fi
|
||
|
|
|
||
|
|
if [ $OS_name == 'redhat' ]
|
||
|
|
then
|
||
|
|
echo "ONBOOT=yes" >> $dir/ifcfg-$nic
|
||
|
|
else
|
||
|
|
echo "STARTMODE=auto" >> $dir/ifcfg-$nic
|
||
|
|
fi
|
||
|
|
elif [ $PLTFRM == "AIX" ]; then
|
||
|
|
if ( pmatch $nic "ml*" ); then #for ml* interface
|
||
|
|
num=${nic##ml}
|
||
|
|
mlt="mlt$num"
|
||
|
|
#Check whether the mlt is available
|
||
|
|
lsdev -C | grep $mlt | grep Available 2>&1 >/dev/null
|
||
|
|
if [ $? -ne 0 ]
|
||
|
|
then
|
||
|
|
echo "$mltnum is not available."
|
||
|
|
logger -p local4.info -t xcat "$mltnum is not available."
|
||
|
|
continue
|
||
|
|
fi
|
||
|
|
|
||
|
|
#Check whether the ml0 is available
|
||
|
|
lsdev -C | grep $nic 2>&1 >/dev/null
|
||
|
|
if [ $? -ne 0 ]
|
||
|
|
then
|
||
|
|
cfgmgr 2>&1 >/dev/null
|
||
|
|
fi
|
||
|
|
|
||
|
|
chdev -l $nic -a state=detach 2>&1
|
||
|
|
chdev -l $nic -a netaddr=$nicip -a netmask=$netmask -a state=up 2>&1
|
||
|
|
else #assume it is ib*
|
||
|
|
lsdev -C | grep icm | grep Available
|
||
|
|
if [ $? -ne 0 ]
|
||
|
|
then
|
||
|
|
mkdev -c management -s infiniband -t icm
|
||
|
|
if [ $? -ne 0 ]
|
||
|
|
then
|
||
|
|
mkdev -l icm
|
||
|
|
if [ $? -ne 0 ]
|
||
|
|
then
|
||
|
|
exit $?
|
||
|
|
fi
|
||
|
|
fi
|
||
|
|
fi
|
||
|
|
|
||
|
|
#Configure the IB interfaces. Customize the port num.
|
||
|
|
num=${nic##ib} #this assumes that all the nics starts with 'ib'
|
||
|
|
if [ "$portnum" == "1" ]; then
|
||
|
|
iba_num=$num
|
||
|
|
ib_adapter="iba$iba_num"
|
||
|
|
port=1
|
||
|
|
else
|
||
|
|
iba_num=`expr $num / 2`
|
||
|
|
ib_adapter="iba$iba_num"
|
||
|
|
if [ $(($num % 2)) == 0 ]
|
||
|
|
then
|
||
|
|
port=1
|
||
|
|
else
|
||
|
|
port=2
|
||
|
|
fi
|
||
|
|
fi
|
||
|
|
mkiba -a $nicip -i $nic -A $ib_adapter -p $port -P -1 -S up -m $netmask
|
||
|
|
fi
|
||
|
|
fi
|
||
|
|
done
|
||
|
|
|
||
|
|
# echo "goodnics=$goodnics"
|
||
|
|
# Bringup all the ib interfaces
|
||
|
|
if [ $PLTFRM == "Linux" ]
|
||
|
|
then
|
||
|
|
|
||
|
|
/sbin/service $ib_driver restart
|
||
|
|
sleep 10
|
||
|
|
for nic in `echo "$goodnics" | tr "," "\n"`
|
||
|
|
do
|
||
|
|
sleep 5
|
||
|
|
|
||
|
|
if [[ "$OSVER" == rhels6* ]]
|
||
|
|
then
|
||
|
|
ip link set dev $nic up
|
||
|
|
else
|
||
|
|
ifup $nic
|
||
|
|
fi
|
||
|
|
done
|
||
|
|
fi
|
||
|
|
|
||
|
|
|