arif/xcat-core
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Files
9c89936b6939a59b8457bc2445e08fff1f5e9f68
xcat-core/xCAT/postscripts/configib

498 lines
17 KiB
Bash
Executable File
Raw Blame History

#!/bin/sh
# IBM(c) 2013 EPL license http://www.eclipse.org/legal/epl-v10.html
# xCAT post script for configuring ib adaptors.
# Work for both IPv4 and IPv6
# The following are a sample of the env used:
# NIC_IBNICS=ib0,ib1
# NIC_IBAPORTS=1 (or 2)
# OSVER=sles11.1
# NICNODE=dxm360m4n04
# NICIPS='eth0!10.1.34.242,ib1!15.0.1.242,ib2!15.0.2.242'
# NICHOSTNAMESUFFIXES='eth1!-eth1,ib1!-ib1,ib2!-ib2'
# NICTYPES='eth1!ethernet,ib1!infiniband,ib2!infiniband'
# NICNETWORKS='eth0!10_1_0_0-255_255_255_0,ib1!ib1,ib2!ib2'
# NETWORKS_LINES=3
# NETWORKS_LINE1='netname=10_1_0_0-255_255_255_0||net=10.1.0.0||mask=255.255.255.0||mgtifname=eth1||gateway=<xcatmaster>||dhcpserver=10.1.0.205||tftpserver=10.1.0.205||nameservers=||ntpservers=||logservers=||dynamicrange=||staticrange=||staticrangeincrement=||nodehostname=||ddnsdomain=||vlanid=||domain=||disable=||comments='
# NETWORKS_LINE2='netname=ib1||net=15.0.1.0||mask=255.255.255.0||mgtifname=||gateway=15.0.1.254||dhcpserver=||tftpserver=||nameservers=||ntpservers=||logservers=||dynamicrange=||staticrange=||staticrangeincrement=||nodehostname=||ddnsdomain=||vlanid=||domain=||disable=||comments='
# NETWORKS_LINE3='netname=ib3||net=15.0.3.0||mask=255.255.255.0||mgtifname=||gateway=||dhcpserver=||tftpserver=||nameservers=||ntpservers=||logservers=||dynamicrange=||staticrange=||staticrangeincrement=||nodehostname=||ddnsdomain=||vlanid=||domain=||disable=||comments='
#This is the number of ports for each ib adpator.
portnum=1
if [ -n "$NIC_IBAPORTS" ]; then
portnum=$NIC_IBAPORTS
fi
# pmatch determines if 1st argument string is matched by 2nd argument pattern
pmatch ()
{
case $1 in
$2) return 0;; # zero return code means string matched by pattern
esac
return 1 # non-zero return code means string not matched by pattern
}
convert_netmask_to_cidr() {
let cidrnum=0
OLFIFS=$IFS
IFS=.
for x in $1
do
case $x in
255) cidrnum=$(expr $cidrnum + 8);;
254) cidrnum=$(expr $cidrnum + 7);;
252) cidrnum=$(expr $cidrnum + 6);;
248) cidrnum=$(expr $cidrnum + 5);;
240) cidrnum=$(expr $cidrnum + 4);;
224) cidrnum=$(expr $cidrnum + 3);;
192) cidrnum=$(expr $cidrnum + 2);;
128) cidrnum=$(expr $cidrnum + 1);;
0);;
esac
done
IFS=$OLDIFS
echo $cidrnum
}
if [ -z "$NIC_IBNICS" ]; then
echo "nothing to do."
fi
#find correct IB driver
PLTFRM=`uname`
if [[ $OSVER == rhels5* || "$OSVER" == rhels6* ]]
then
ib_driver="rdma"
/sbin/service $ib_driver status
if [ $? -eq 1 ]
then
ib_driver="openibd"
/sbin/service $ib_driver status
if [ $? -eq 1 ]
then
echo "Not found the driver dameon: rdma or openibd"
logger -p local4.info -t xcat "Not found the driver dameon: rdma or openibd"
exit
fi
fi
else
ib_driver="openibd"
fi
#make changes to the confiuration file
if [ $PLTFRM == "Linux" ]
then
if [ -f "/etc/rdma/rdma.conf" ]
then
sed -i "s/SDP_LOAD=yes/SDP_LOAD=no/g" /etc/rdma/rdma.conf
elif [ -f "/etc/infiniband/openib.conf" ]
then
sed -i "s/SDP_LOAD=yes/SDP_LOAD=no/g" /etc/infiniband/openib.conf
fi
if [ -f "/etc/modprobe.conf" ]
then
if [ "$portnum" == "1" ]; then
sed -i "/options ib_ehca nr_ports/d" /etc/modprobe.conf
echo 'options ib_ehca nr_ports=1' >> /etc/modprobe.conf
else
sed -i "/options ib_ehca nr_ports=1/d" /etc/modprobe.conf
fi
sed -i "/options ib_ehca lock_hcalls/d" /etc/modprobe.conf
echo 'options ib_ehca lock_hcalls=0' >> /etc/modprobe.conf
fi
/sbin/chkconfig --level 2345 $ib_driver on
/sbin/service $ib_driver restart
sysctl -p
fi
#clean up existing ib nic configrations
if [ $PLTFRM == "Linux" ]
then
if [ -f /etc/redhat-release ]
then
OS_name="redhat"
elif [ -f /etc/SuSE-release ]
then
OS_name="suse"
else
echo "Unsupported to config IB on this OS!"
logger -p local4.info -t xcat "Unsupported to config IB on this OS!"
exit
fi
if [ $OS_name == 'suse' ]
then
dir="/etc/sysconfig/network"
else
dir="/etc/sysconfig/network-scripts"
fi
for nic in `echo "$NIC_IBNICS" | tr "," "\n"`
do
`rm -f $dir/ifcfg-$nic` 2>&1 1>/dev/null
# nic aliases
`rm -f $dir/ifcfg-$nic:*` 2>&1 1>/dev/null
done
else
loop_number=`lsdev | grep "IP over Infiniband Network Interface" | wc -l`
num=0
while [ $num -lt $loop_number ]
do
if [ "$portnum" == "2" ]; then
num0=`expr $num \* 2`
num1=`expr $num0 + 1`
`rmdev -d -l ib$num0 >/dev/null 2>&1`
`rmdev -d -l ib$num1 >/dev/null 2>&1`
`rmdev -d -l ml$num >/dev/null 2>&1`
num=`expr $num + 1`
else
`rmdev -d -l ib$num >/dev/null 2>&1`
n=`expr $num % 2`
if [ $n == 1 ]
then
mlnum=`expr $num / 2`
`rmdev -d -l ml$mlnum >/dev/null 2>&1`
fi
num=`expr $num + 1`
fi
done
`rmdev -dl icm >/dev/null 2>&1`
#restart the managers
mkdev -c management -s infiniband -t icm
cfgmgr
fi
goodnics=""
for nic in `echo "$NIC_IBNICS" | tr "," "\n"`
do
#Get nic ips
nicips=""
for tmp in `echo "$NICIPS" | tr "," "\n"`
do
nic_tmp=`echo "$tmp" | awk -F"!" '{print $1}'`;
if [ $nic_tmp == $nic ]; then
# nicips=ip1|ip2|ip3
nicips=`echo "$tmp" | awk -F"!" '{print $2}'`;
break
fi
done
# echo "nic=$nic, nicip=$nicip"
#get nic networks name
nicnets=""
for tmp in `echo "$NICNETWORKS" | tr "," "\n"`
do
nic_tmp=`echo "$tmp" | awk -F"!" '{print $1}'`;
if [ $nic_tmp == $nic ]; then
# nicnets=net1|net2|net3
nicnets=`echo "$tmp" | awk -F"!" '{print $2}'`;
break
fi
done
#echo "nic=$nic, nicnet=$nicnet"
if [ -z "$nicnets" ]; then
echo "No network defined for $nic"
logger -p local4.info -t xcat "No network defined for $nic"
continue
fi
ipindex=0
for nicip in `echo $nicips | tr "|" "\n"`
do
ipindex=`expr $ipindex + 1`
nicnet=`echo $nicnets | cut -d '|' -f$ipindex`
#get netmask and gateway
index=0;
found=0
while [ $index -lt $NETWORKS_LINES ]
do
index=$((index+1))
eval netline=\$NETWORKS_LINE$index
if [[ -n "$netline" ]]; then
for tmp in `echo "$netline" | tr "\|" "\n"`
do
key=`echo "$tmp" | awk -F"=" '{print $1}'`;
case $key in
netname)
netname=`echo "$tmp" | awk -F"=" '{print $2}'`
if [ "$netname" != "$nicnet" ];
then
break;
fi
;;
net)
net=`echo "$tmp" | awk -F"=" '{print $2}'`
;;
mask)
netmask=`echo "$tmp" | awk -F"=" '{print $2}'`
# remove the prefix "/" from ipv6 mask
if echo $netmask | grep "/" 2>&1 1>/dev/null
then
netmask=`echo $netmask | awk -F'/' '{print $2}'`
fi
;;
gateway)
gateway=`echo "$tmp" | awk -F"=" '{print $2}'`
found=1
;;
esac
if [ $found -eq 1 ]; then
break;
fi
done # end for tmp in `echo "$netline" | tr "\|" "\n"`
if [ $found -eq 1 ]; then
break;
fi
fi # end if [[ -n "$netline" ]]
done # end while [ $index -lt $NETWORKS_LINES ]
#echo "found=$found"
#echo "index=$index netname=$nicnet net=$net mask=$netmask gateway=$gateway"
# Setup goodnics list
if [ "$found" == "0" ]; then
echo "Cannot find network $nicnet for $nic"
logger -p local4.info -t xcat "Cannot find network $nicnet for $nic"
continue
else
if [ -z "$goodnics" ]; then
goodnics=$nic
else
goodnics="$goodnics,$nic"
fi
fi
#there should be only one gateway on a node and that should go through the to the xcat management node and would be set up from the install nic. Anything else should be a route
if [ "$gateway" == "<xcatmaster>" ]; then
gateway=''
fi
if [ $PLTFRM == "Linux" ]
then
# Issue openibd for Linux at boot time
if [ -f /etc/sysctl.conf ]
then
sed -i "/net.ipv4.conf.$nic.arp_filter=1/d" /etc/sysctl.conf 2>&1 1>/dev/null
sed -i "/net.ipv4.conf.$nic.arp_ignore=1/d" /etc/sysctl.conf 2>&1 1>/dev/null
cfg="net.ipv4.conf.$nic.arp_filter=1"
grep "$cfg" /etc/sysctl.conf 2>&1 1>/dev/null
if [ $? -ne 0 ]
then
echo "net.ipv4.conf.$nic.arp_filter=1" >> /etc/sysctl.conf
fi
cfg="net.ipv4.conf.$nic.arp_ignore=1"
grep "$cfg" /etc/sysctl.conf 2>&1 1>/dev/null
if [ $? -ne 0 ]
then
echo "net.ipv4.conf.$nic.arp_ignore=1" >> /etc/sysctl.conf
fi
fi
if [ $OS_name == 'suse' ]
then
# First ip address
if [ $ipindex -eq 1 ]
then
# Write the info to the ifcfg file
echo "DEVICE=$nic
BOOTPROTO=static
STARTMODE=onboot
IPADDR=$nicip" > $dir/ifcfg-$nic
# ipv6
if echo $nicip | grep : 2>&1 1>/dev/null
then
echo "PREFIXLEN=$netmask" >> $dir/ifcfg-$nic
else
echo "NETMASK=$netmask" >> $dir/ifcfg-$nic
fi
if [ -n "$gateway" ]; then
# Do not add duplicate default gateway entry
grep -E "default\\s+$gateway\\s+" /etc/sysconfig/network/routes 2>&1 1>/dev/null
if [ $? -ne 0 ]; then
echo "default $gateway - -" >> /etc/sysconfig/network/routes
fi
fi
else # not the first ip address
echo "LABEL_$ipindex=$ipindex
IPADDR_$ipindex=$nicip" >> $dir/ifcfg-$nic
# ipv6
if echo $nicip | grep : 2>&1 1>/dev/null
then
echo "PREFIXLEN_$ipindex=$netmask" >> $dir/ifcfg-$nic
else
echo "NETMASK_$ipindex=$netmask" >> $dir/ifcfg-$nic
fi
if [ -n "$gateway" ]; then
# Do not add duplicate default gateway entry
grep -E "default\\s+$gateway\\s+" /etc/sysconfig/network/routes 2>&1 1>/dev/null
if [ $? -ne 0 ]; then
echo "default $gateway - -" >> /etc/sysconfig/network/routes
fi
fi
fi # end if [ $ipindex -eq 1 ]
elif [ $OS_name == 'redhat' ]
then
# First ip address
if [ $ipindex -eq 1 ]
then
# Write the info to the ifcfg file
echo "DEVICE=$nic
BOOTPROTO=static
ONBOOT=yes
IPADDR=$nicip" > $dir/ifcfg-$nic
# ipv6
if echo $nicip | grep : 2>&1 1>/dev/null
then
echo "PREFIXLEN=$netmask" >> $dir/ifcfg-$nic
else
if [[ "$OSVER" == rhels6* ]]
then
#get prefix from netmask, this is for IPv4 only
prefix=24
prefix=$(convert_netmask_to_cidr $netmask)
echo "PREFIX=$prefix" >> $dir/ifcfg-$nic
else
echo "NETMASK=$netmask" >> $dir/ifcfg-$nic
fi
fi
if [ -n "$gateway" ]; then
echo "GATEWAY=$gateway" >> $dir/ifcfg-$nic
fi
else # not the first ip address
# ipv6
if echo $nicip | grep : 2>&1 1>/dev/null
then
grep "IPV6INIT" $dir/ifcfg-$nic 2>&1 1>/dev/null
# The first ipv6 address
if [ $? -ne 0 ]
then
echo "IPV6INIT=yes
IPV6ADDR=$nicip/$netmask" >> $dir/ifcfg-$nic
else
echo "IPV6ADDR_SECONDARIES=$nicip/$netmask" >> $dir/ifcfg-$nic
fi
if [ -n "$gateway" ]; then
echo "IPV6_DEFAULTGW=$gateway" >> $dir/ifcfg-$nic
fi
else # ipv4 address
echo "DEVICE=$nic:$ipindex
BOOTPROTO=static
ONBOOT=yes
IPADDR=$nicip" > $dir/ifcfg-$nic:$ipindex
if [[ "$OSVER" == rhels6* ]]
then
#get prefix from netmask, this is for IPv4 only
prefix=24
prefix=$(convert_netmask_to_cidr $netmask)
echo "PREFIX=$prefix" >> $dir/ifcfg-$nic:$ipindex
else
echo "NETMASK=$netmask" >> $dir/ifcfg-$nic:$ipindex
fi
if [ -n "$gateway" ]; then
echo "GATEWAY=$gateway" >> $dir/ifcfg-$nic:$ipindex
fi
# need to run ifup eth1:1 for RedHat
goodnics="$goodnics,$nic:$ipindex"
fi
fi # end not the first ip address
else
echo "Unsupported operating system"
logger -p local4.err -t xcat "Unsupported operating system"
fi
elif [ $PLTFRM == "AIX" ]; then
if ( pmatch $nic "ml*" ); then #for ml* interface
num=${nic##ml}
mlt="mlt$num"
#Check whether the mlt is available
lsdev -C | grep $mlt | grep Available 2>&1 >/dev/null
if [ $? -ne 0 ]
then
echo "$mltnum is not available."
logger -p local4.info -t xcat "$mltnum is not available."
continue
fi
#Check whether the ml0 is available
lsdev -C | grep $nic 2>&1 >/dev/null
if [ $? -ne 0 ]
then
cfgmgr 2>&1 >/dev/null
fi
chdev -l $nic -a state=detach 2>&1
chdev -l $nic -a netaddr=$nicip -a netmask=$netmask -a state=up 2>&1
else #assume it is ib*
#lsdev -C | grep icm | grep Available
#if [ $? -ne 0 ]
#then
# mkdev -c management -s infiniband -t icm
# if [ $? -ne 0 ]
# then
# mkdev -l icm
# if [ $? -ne 0 ]
# then
# exit $?
# fi
# fi
# fi
#Configure the IB interfaces. Customize the port num.
num=${nic##ib} #this assumes that all the nics starts with 'ib'
if [ "$portnum" == "1" ]; then
iba_num=$num
ib_adapter="iba$iba_num"
port=1
else
iba_num=`expr $num / 2`
ib_adapter="iba$iba_num"
if [ $(($num % 2)) == 0 ]
then
port=1
else
port=2
fi
fi
if [ $NODE ]; then
shortname=$NODE
else
shortname=`hostname -s`
fi
chdev -l $ib_adapter -a hostname=$shortname
mkiba -a $nicip -i $nic -A $ib_adapter -p $port -P -1 -S up -m $netmask
fi # end assume it is ib*
fi # end if AIX
done # end for nicip
done # end for nic
# echo "goodnics=$goodnics"
# Bringup all the ib interfaces
if [ $PLTFRM == "Linux" ]
then
/sbin/service $ib_driver restart
for nic in `echo "$goodnics" | tr "," "\n"`
do
sleep 5
if [[ "$OSVER" == rhels6* ]]
then
ip link set dev $nic up
else
ifup $nic
fi
done
fi
Reference in New Issue View Git Blame Copy Permalink