diff --git a/xCAT-test/autotest/testcase/infiniband/cases0 b/xCAT-test/autotest/testcase/infiniband/cases0 new file mode 100644 index 000000000..2da6c8d92 --- /dev/null +++ b/xCAT-test/autotest/testcase/infiniband/cases0 @@ -0,0 +1,40 @@ +start:rhel_infiniband_diskless_install +description:Integration test of RHEL 7 and MLNX OFED for diskless installation. Following variables need to be set in the test configuration file, ISO, CN, MLNX, DKMS. Read comments in rhel-infiniband-diskless-install.sh for details. +cmd:RHEL_ISO="$$ISO" LINUX_DISTRO="GETNODEATTR($$CN, os)" COMPUTE_NODE="$$CN" MLNX_ISO="$$MLNX" DKMS_RPM="$$DKMS" /opt/xcat/share/xcat/tools/autotest/testcase/infiniband/rhel-infiniband-diskless-install.sh +check:rc==0 +end + +start:rhel_infiniband_diskfull_install +description:Integration test of RHEL 7 and MLNX OFED for diskfull installation. Following variables need to be set in the test configuration file, ISO, CN, MLNX, DKMS. Read comments in rhel-infiniband-diskfull-install.sh for details. +cmd:RHEL_ISO="$$ISO" LINUX_DISTRO="GETNODEATTR($$CN, os)" COMPUTE_NODE="$$CN" MLNX_ISO="$$MLNX" DKMS_RPM="$$DKMS" /opt/xcat/share/xcat/tools/autotest/testcase/infiniband/rhel-infiniband-diskfull-install.sh +check:rc==0 +end + +start:confignetwork_ib_ipoib +description: this case is to test confignetwork could configure the first infiniband interface, ib0, successfully with updatenode. +cmd:lsdef $$CN;if [ $? -eq 0 ]; then lsdef -l $$CN -z >/tmp/CN.standa ;fi +check:rc==0 +cmd:xdsh $$CN "mkdir -p /tmp/backupnet/" +check:rc==0 +cmd:if grep SUSE /etc/*release;then xdsh $$CN "cp -f /etc/sysconfig/network/ifcfg-* /tmp/backupnet/"; elif grep "Red Hat" /etc/*release;then xdsh $$CN "cp -f /etc/sysconfig/network-scripts/ifcfg-* /tmp/backupnet/"; elif grep Ubuntu /etc/*release;then xdsh $$CN "cp -f /etc/network/interfaces.d/* /tmp/backupnet/";else echo "Sorry,this is not supported os"; fi +check:rc==0 +cmd:mkdef -t network -o 11_1_0_0-255_255_0_0 net=11.1.0.0 mask=255.255.0.0 mgtifname=ib0 +check:rc==0 +cmd:chdef $$CN nicips.ib0=11.1.0.100 nictypes.ib0=infiniband nicnetworks.ib0=11_1_0_0-255_255_0_0 +check:rc==0 +cmd:updatenode $$CN -P confignetwork +check:rc==0 +cmd:if grep SUSE /etc/*release;then xdsh $$CN "grep 11.1.0.100 /etc/sysconfig/network/ifcfg-ib0"; elif grep "Red Hat" /etc/*release;then xdsh $$CN "grep 11.1.0.100 /etc/sysconfig/network-scripts/ifcfg-ib0"; elif grep Ubuntu /etc/*release;then xdsh $$CN "grep 11.1.0.100 /etc/network/interfaces.d/ib0";else echo "Sorry,this is not supported os"; fi +check:output=~11.1.0.100 +cmd:xdsh $$CN "ping -c 1 11.1.0.100" +check:rc==0 +# Clean up +cmd:xdsh $$CN "ip addr del 11.1.0.100/16 dev ib0" +cmd:rmdef -t network -o 11_1_0_0-255_255_0_0 +cmd:if grep SUSE /etc/*release;then xdsh $$CN "rm -rf /etc/sysconfig/network/ifcfg-ib0"; elif grep "Red Hat" /etc/*release;then xdsh $$CN "rm -rf /etc/sysconfig/network-scripts/ifcfg-ib0"; elif grep Ubuntu /etc/*release;then xdsh $$CN "rm -rf /etc/network/interfaces.d/ib0";else echo "Sorry,this is not supported os"; fi +check:rc==0 +cmd:if [ -e /tmp/CN.standa ]; then rmdef $$CN; cat /tmp/CN.standa | mkdef -z; rm -rf /tmp/CN.standa; fi +check:rc==0 +cmd:if grep SUSE /etc/*release;then xdsh $$CN "cp -f /tmp/backupnet/* /etc/sysconfig/network/"; elif grep "Red Hat" /etc/*release;then xdsh $$CN "cp -f /tmp/backupnet/* /etc/sysconfig/network-scripts/"; elif grep Ubuntu /etc/*release;then xdsh $$CN "cp -f /tmp/backupnet/* /etc/network/interfaces.d/";else echo "Sorry,this is not supported os"; fi +cmd:xdsh $$CN "rm -rf /tmp/backupnet/" +end diff --git a/xCAT-test/autotest/testcase/infiniband/rhel-infiniband-diskfull-install.sh b/xCAT-test/autotest/testcase/infiniband/rhel-infiniband-diskfull-install.sh new file mode 100755 index 000000000..7085a7545 --- /dev/null +++ b/xCAT-test/autotest/testcase/infiniband/rhel-infiniband-diskfull-install.sh @@ -0,0 +1,183 @@ +#!/bin/bash + +######## +# +# For manually run this script in a standalone test environment without xCAT-test, +# do the following steps. +# +# * Set all the variables in LINE 11, 13, 16, and 17. +# * Download all the ISO files, RPMs needed, and put them in ${SOURCE_DIR} +# * If you intend to specify all the packages explicitly, set variable in LINE 61, 62, 63. +# + +[ -n "$LINUX_DISTRO" ] && +LINUX_DISTRO="rhels7.4" +[ -n "$LINUX_ARCH" ] && +LINUX_ARCH="ppc64le" + +[ -n "$COMPUTE_NODE" ] && +COMPUTE_NODE="nonexistent" +SOURCE_DIR="/path/to/source" + +######## + +# $SOURCE_DIR is a directory this test case will be searched for. +# Files with the name looked like the following will be searched. +# +# -rw-r--r-- 1 nobody nobody 79404 Jul 27 01:20 dkms-2.3-5.20170523git8c3065c.el7.noarch.rpm +# -rw-r--r-- 1 nobody nobody 29085696 Jul 9 09:24 mlnx-en-4.1-1.0.2.0-rhel7.3-ppc64le.iso +# -rw-r--r-- 2 nobody nobody 3188944896 Oct 31 2016 RHEL-7.3-20161019.0-Server-ppc64le-dvd1.iso + +######## +# Auto detect all the source packages from the ${SOURCE_DIR} + +[ -d "${SOURCE_DIR}" ] +[ "$?" -ne "0" ] && echo "Directory ${SOURCE_DIR} not found." >&2 && exit 1 + +declare DKMS_RPM +declare RHEL_ISO +declare MLNX_ISO + +for f in "${SOURCE_DIR}"/* +do + r="$(realpath "${f}")" + [ -f "${r}" ] || continue + case "${r##*/}" in + "RHEL-"*"-"*"-Server-${LINUX_ARCH}-dvd1.iso") + RHEL_ISO="${r}" + ;; + "MLNX_OFED_LINUX-"*"-"*"-${LINUX_DISTRO/s/}-${LINUX_ARCH}.iso") + MLNX_ISO="${r}" + ;; + "dkms-"*".el7.noarch.rpm") + DKMS_RPM="${r}" + ;; + esac +done + +######## +# Override the auto detect results here. + +#MLNX_ISO="${SOURCE_DIR}/mlnx-en-4.1-1.0.2.0-rhel7.3-ppc64le.iso" +#RHEL_ISO="${SOURCE_DIR}/RHEL-7.3-20161019.0-Server-ppc64le-dvd1.iso" +#DKMS_RPM="${SOURCE_DIR}/dkms-2.3-5.20170523git8c3065c.el7.noarch.rpm" + +######## +echo "Red Hat Enterprise Linux Server ISO" +echo "===================================" +echo "${RHEL_ISO}" +echo +echo "Mellanox EN Driver for Linux" +echo "============================" +echo "${MLNX_ISO}" +echo +echo "DKMS RPM" +echo "========" +echo "${DKMS_RPM}" +echo + +echo "The files listed above were found and will be used for this test case" +echo "Press Ctrl-C to abort!" +for t in {5..1} +do + echo -n " ... ${t}" + sleep 1 + echo -n -e "\b\b\b\b\b\b" +done +######## + +OSIMAGE_NAME="${LINUX_DISTRO}-${LINUX_ARCH}-install-mlnx" +OSIMAGE_OTHERPKGDIR="/install/post/otherpkgs/${LINUX_DISTRO}/${LINUX_ARCH}" +OSIMAGE_ROOTIMGDIR="/install/install/${LINUX_DISTRO}/${LINUX_ARCH}/${OSIMAGE_NAME}" + +[ -f "${RHEL_ISO}" ] +[ "$?" -ne "0" ] && echo "File ${RHEL_ISO} not found." >&2 && exit 1 +copycds "${RHEL_ISO}" +[ "$?" -ne "0" ] && echo "Copy CD failed." >&2 && exit 1 + +rmdef -t osimage "${OSIMAGE_NAME}" +mkdef -z <<-EOF +# + +${OSIMAGE_NAME}: + objtype=osimage + imagetype=linux + osarch=${LINUX_ARCH} + osdistroname=${LINUX_DISTRO}-${LINUX_ARCH} + osname=Linux + osvers=${LINUX_DISTRO} + otherpkgdir="${OSIMAGE_OTHERPKGDIR}" + otherpkglist=/install/custom/install/rh/mlnx.${LINUX_DISTRO%%.*}.${LINUX_ARCH}.otherpkgs.pkglist + pkgdir=/install/${LINUX_DISTRO}/${LINUX_ARCH} + pkglist=/install/custom/install/rh/mlnx.${LINUX_DISTRO%%.*}.${LINUX_ARCH}.pkglist + profile=compute + provmethod=install + template=/opt/xcat/share/xcat/install/rh/compute.${LINUX_DISTRO%%.*}.tmpl +EOF +[ "$?" -ne "0" ] && echo "Make osimage definition failed." >&2 && exit 1 + +mkdir -p /install/mlnx +cp "${MLNX_ISO}" /install/mlnx + +mkdir -p /install/custom/install/rh + +( + cat /opt/xcat/share/xcat/install/rh/compute.${LINUX_DISTRO%%.*}.pkglist + + cat <<-EOF + + # For MLNX OFED support + EOF + cat /opt/xcat/share/xcat/ib/netboot/rh/ib.${LINUX_DISTRO%%.*}.${LINUX_ARCH}.pkglist +) >"/install/custom/install/rh/mlnx.${LINUX_DISTRO%%.*}.${LINUX_ARCH}.pkglist" + +( + : +) >"/install/custom/install/rh/mlnx.${LINUX_DISTRO%%.*}.${LINUX_ARCH}.otherpkgs.pkglist" + +cp /opt/xcat/share/xcat/ib/scripts/Mellanox/mlnxofed_ib_install.v2 \ + /install/postscripts/mlnxofed_ib_install.v2 + +chdef "${COMPUTE_NODE}" "postscripts=syslog,remoteshell,syncfiles,mlnxofed_ib_install.v2 -p /install/mlnx/${MLNX_ISO##*/} -m --add-kernel-support" + +rm -rf "${OSIMAGE_OTHERPKGDIR}" +mkdir -p "${OSIMAGE_OTHERPKGDIR}" + +mkdir -p "${OSIMAGE_OTHERPKGDIR}"/dkms +[ -f "${DKMS_RPM}" ] +[ "$?" -ne "0" ] && echo "File ${DKMS_RPM} not found." >&2 +cp "${DKMS_RPM}" "${OSIMAGE_OTHERPKGDIR}/dkms" + +( cd "${OSIMAGE_OTHERPKGDIR}" && createrepo . ) + +makedhcp -n +rinstall "${COMPUTE_NODE}" "osimage=${OSIMAGE_NAME}" + +INSTALL_TIMEOUT=1800 +declare -i WAIT=0 + +while sleep 10 +do + (( WAIT += 10 )) + nodestat "${COMPUTE_NODE}" | grep ': sshd$' + [ "$?" -eq "0" ] && break + [ "${WAIT}" -le "${INSTALL_TIMEOUT}" ] + [ "$?" -ne "0" ] && echo "Operating system installation failed." >&2 && exit 1 +done + +# For workaround the GitHub issue #3549 +sleep 5 + +xdsh "${COMPUTE_NODE}" date +[ "$?" -ne "0" ] && echo "Failed connect to compute node via SSH." >&2 && exit 1 + +xdsh "${COMPUTE_NODE}" 'rpm -qa' | grep 'mlnx' +[ "$?" -ne "0" ] && echo "MLNX OFED installation checking failed." >&2 && exit 1 + +xdsh "${COMPUTE_NODE}" 'lspci' +xdsh "${COMPUTE_NODE}" 'lsslot -c pci' +xdsh "${COMPUTE_NODE}" 'lsslot' +xdsh "${COMPUTE_NODE}" 'ibv_devinfo' +xdsh "${COMPUTE_NODE}" 'iblinkinfo' + +exit 0 diff --git a/xCAT-test/autotest/testcase/infiniband/rhel-infiniband-diskless-install.sh b/xCAT-test/autotest/testcase/infiniband/rhel-infiniband-diskless-install.sh new file mode 100755 index 000000000..f29cc891e --- /dev/null +++ b/xCAT-test/autotest/testcase/infiniband/rhel-infiniband-diskless-install.sh @@ -0,0 +1,199 @@ +#!/bin/bash + +######## +# +# For manually run this script in a standalone test environment without xCAT-test, +# do the following steps. +# +# * Set all the variables in LINE 11, 13, 16, and 17. +# * Download all the ISO files, RPMs needed, and put them in ${SOURCE_DIR} +# * If you intend to specify all the packages explicitly, set variable in LINE 61, 62, 63 +# + +[ -n "$LINUX_DISTRO" ] && +LINUX_DISTRO="rhels7.4" +[ -n "$LINUX_ARCH" ] && +LINUX_ARCH="ppc64le" + +[ -n "$COMPUTE_NODE" ] && +COMPUTE_NODE="nonexistent" +SOURCE_DIR="/path/to/source" + +######## + +# $SOURCE_DIR is a directory this test case will be searched for. +# Files with the name looked like the following will be searched. +# +# -rw-r--r-- 1 nobody nobody 79404 Jul 27 01:20 dkms-2.3-5.20170523git8c3065c.el7.noarch.rpm +# -rw-r--r-- 1 nobody nobody 29085696 Jul 9 09:24 mlnx-en-4.1-1.0.2.0-rhel7.3-ppc64le.iso +# -rw-r--r-- 2 nobody nobody 3188944896 Oct 31 2016 RHEL-7.3-20161019.0-Server-ppc64le-dvd1.iso + +######## +# Auto detect all the source packages from the ${SOURCE_DIR} + +[ -d "${SOURCE_DIR}" ] +[ "$?" -ne "0" ] && echo "Directory ${SOURCE_DIR} not found." >&2 && exit 1 + +declare DKMS_RPM +declare RHEL_ISO +declare MLNX_ISO + +for f in "${SOURCE_DIR}"/* +do + r="$(realpath "${f}")" + [ -f "${r}" ] || continue + case "${r##*/}" in + "RHEL-"*"-"*"-Server-${LINUX_ARCH}-dvd1.iso") + RHEL_ISO="${r}" + ;; + "MLNX_OFED_LINUX-"*"-"*"-${LINUX_DISTRO/s/}-${LINUX_ARCH}.iso") + MLNX_ISO="${r}" + ;; + "dkms-"*".el7.noarch.rpm") + DKMS_RPM="${r}" + ;; + esac +done + +######## +# Override the auto detect results here. + +#MLNX_ISO="${SOURCE_DIR}/mlnx-en-4.1-1.0.2.0-rhel7.3-ppc64le.iso" +#RHEL_ISO="${SOURCE_DIR}/RHEL-7.3-20161019.0-Server-ppc64le-dvd1.iso" +#DKMS_RPM="${SOURCE_DIR}/dkms-2.3-5.20170523git8c3065c.el7.noarch.rpm" + +######## +echo "Red Hat Enterprise Linux Server ISO" +echo "===================================" +echo "${RHEL_ISO}" +echo +echo "Mellanox EN Driver for Linux" +echo "============================" +echo "${MLNX_ISO}" +echo +echo "DKMS RPM" +echo "========" +echo "${DKMS_RPM}" +echo + +echo "The files listed above were found and will be used for this test case" +echo "Press Ctrl-C to abort!" +for t in {5..1} +do + echo -n " ... ${t}" + sleep 1 + echo -n -e "\b\b\b\b\b\b" +done +######## + +OSIMAGE_NAME="${LINUX_DISTRO}-${LINUX_ARCH}-netboot-mlnx" +OSIMAGE_OTHERPKGDIR="/install/post/otherpkgs/${LINUX_DISTRO}/${LINUX_ARCH}" +OSIMAGE_ROOTIMGDIR="/install/netboot/${LINUX_DISTRO}/${LINUX_ARCH}/${OSIMAGE_NAME}" + +[ -f "${RHEL_ISO}" ] +[ "$?" -ne "0" ] && echo "File ${RHEL_ISO} not found." >&2 && exit 1 +copycds "${RHEL_ISO}" +[ "$?" -ne "0" ] && echo "Copy CD failed." >&2 && exit 1 + +rmdef -t osimage "${OSIMAGE_NAME}" +mkdef -z <<-EOF +# + +${OSIMAGE_NAME}: + objtype=osimage + exlist=/opt/xcat/share/xcat/netboot/rh/compute.${LINUX_DISTRO%%.*}.${LINUX_ARCH}.exlist + imagetype=linux + osarch=${LINUX_ARCH} + osdistroname=${LINUX_DISTRO}-${LINUX_ARCH} + osname=Linux + osvers=${LINUX_DISTRO} + otherpkgdir="${OSIMAGE_OTHERPKGDIR}" + otherpkglist=/install/custom/netboot/rh/mlnx.${LINUX_DISTRO%%.*}.${LINUX_ARCH}.otherpkgs.pkglist + permission=755 + pkgdir=/install/${LINUX_DISTRO}/${LINUX_ARCH} + pkglist=/install/custom/netboot/rh/mlnx.${LINUX_DISTRO%%.*}.${LINUX_ARCH}.pkglist + postinstall=/install/custom/netboot/rh/mlnx.${LINUX_DISTRO%%.*}.${LINUX_ARCH}.postinstall + profile=compute + provmethod=netboot + rootimgdir=${OSIMAGE_ROOTIMGDIR} +EOF +[ "$?" -ne "0" ] && echo "Make osimage definition failed." >&2 && exit 1 + +mkdir -p /install/mlnx +cp "${MLNX_ISO}" /install/mlnx + +mkdir -p /install/custom/netboot/rh + +( + cat /opt/xcat/share/xcat/netboot/rh/compute.${LINUX_DISTRO%%.*}.${LINUX_ARCH}.pkglist + + cat <<-EOF + + # For MLNX OFED support + EOF + cat /opt/xcat/share/xcat/ib/netboot/rh/ib.${LINUX_DISTRO%%.*}.${LINUX_ARCH}.pkglist +) >"/install/custom/netboot/rh/mlnx.${LINUX_DISTRO%%.*}.${LINUX_ARCH}.pkglist" + +( + : +) >"/install/custom/netboot/rh/mlnx.${LINUX_DISTRO%%.*}.${LINUX_ARCH}.otherpkgs.pkglist" + +cp /opt/xcat/share/xcat/ib/scripts/Mellanox/mlnxofed_ib_install.v2 \ + /install/postscripts/mlnxofed_ib_install.v2 +( + cat "/opt/xcat/share/xcat/netboot/rh/compute.${LINUX_DISTRO%%.*}.${LINUX_ARCH}.postinstall" + cat <<-EOF + + NODESETSTATE=genimage IMGROOTPATH=${OSIMAGE_ROOTIMGDIR}/rootimg bash -x /install/postscripts/mlnxofed_ib_install.v2 -p /install/mlnx/${MLNX_ISO##*/} -m --add-kernel-support + EOF +) >"/install/custom/netboot/rh/mlnx.${LINUX_DISTRO%%.*}.${LINUX_ARCH}.postinstall" +chmod 0755 "/install/custom/netboot/rh/mlnx.${LINUX_DISTRO%%.*}.${LINUX_ARCH}.postinstall" + +rm -rf "${OSIMAGE_OTHERPKGDIR}" +mkdir -p "${OSIMAGE_OTHERPKGDIR}" + +mkdir -p "${OSIMAGE_OTHERPKGDIR}"/dkms +[ -f "${DKMS_RPM}" ] +[ "$?" -ne "0" ] && echo "File ${DKMS_RPM} not found." >&2 +cp "${DKMS_RPM}" "${OSIMAGE_OTHERPKGDIR}/dkms" + +( cd "${OSIMAGE_OTHERPKGDIR}" && createrepo . ) + +rm -rf "${OSIMAGE_ROOTIMGDIR}" + +genimage "${OSIMAGE_NAME}" +[ "$?" -ne "0" ] && echo "genimage failed" >&2 && exit 1 +packimage "${OSIMAGE_NAME}" +[ "$?" -ne "0" ] && echo "packimage failed" >&2 && exit 1 + +makedhcp -n +rinstall "${COMPUTE_NODE}" "osimage=${OSIMAGE_NAME}" + +NETBOOT_TIMEOUT=600 +declare -i WAIT=0 + +while sleep 10 +do + (( WAIT += 10 )) + nodestat "${COMPUTE_NODE}" | grep ': sshd$' + [ "$?" -eq "0" ] && break + [ "${WAIT}" -le "${NETBOOT_TIMEOUT}" ] + [ "$?" -ne "0" ] && echo "Netboot failed." >&2 && exit 1 +done + +# For workaround the GitHub issue #3549 +sleep 5 + +xdsh "${COMPUTE_NODE}" date +[ "$?" -ne "0" ] && echo "Failed connect to compute node via SSH." >&2 && exit 1 + +xdsh "${COMPUTE_NODE}" 'rpm -qa' | grep 'mlnx' +[ "$?" -ne "0" ] && echo "MLNX OFED installation checking failed." >&2 && exit 1 + +xdsh "${COMPUTE_NODE}" 'lspci' +xdsh "${COMPUTE_NODE}" 'lsslot -c pci' +xdsh "${COMPUTE_NODE}" 'lsslot' +xdsh "${COMPUTE_NODE}" 'ibv_devinfo' +xdsh "${COMPUTE_NODE}" 'iblinkinfo' + +exit 0