From 748d4647448816fc49220e0a65e01426bd1e807c Mon Sep 17 00:00:00 2001 From: GONG Jie Date: Thu, 17 Aug 2017 13:44:14 +0800 Subject: [PATCH 01/10] Test cases of compute node deployment with MLNX OFED InfiniBand drivers --- .../rhel-infiniband-diskfull-install.sh | 160 ++++++++++++++++ .../rhel-infiniband-diskless-install.sh | 176 ++++++++++++++++++ 2 files changed, 336 insertions(+) create mode 100755 xCAT-test/autotest/testcase/infiniband/rhel-infiniband-diskfull-install.sh create mode 100755 xCAT-test/autotest/testcase/infiniband/rhel-infiniband-diskless-install.sh diff --git a/xCAT-test/autotest/testcase/infiniband/rhel-infiniband-diskfull-install.sh b/xCAT-test/autotest/testcase/infiniband/rhel-infiniband-diskfull-install.sh new file mode 100755 index 000000000..ec277a729 --- /dev/null +++ b/xCAT-test/autotest/testcase/infiniband/rhel-infiniband-diskfull-install.sh @@ -0,0 +1,160 @@ +#!/bin/bash + +######## +# Set all the variables below + +LINUX_DISTRO="rhels7.4" +LINUX_ARCH="ppc64le" + +COMPUTE_NODE="c910f03c01p10" +SOURCE_DIR="/media/xcat" + +######## + +# $SOURCE_DIR is a directory this test case will be searched for. +# Files with the name looked like the following will be searched. +# +# -rw-r--r-- 1 nobody nobody 79404 Jul 27 01:20 dkms-2.3-5.20170523git8c3065c.el7.noarch.rpm +# -rw-r--r-- 1 nobody nobody 29085696 Jul 9 09:24 mlnx-en-4.1-1.0.2.0-rhel7.3-ppc64le.iso +# -rw-r--r-- 2 nobody nobody 3188944896 Oct 31 2016 RHEL-7.3-20161019.0-Server-ppc64le-dvd1.iso + +######## +# Auto detect all the source packages from the ${SOURCE_DIR} + +[ -d "${SOURCE_DIR}" ] +[ "$?" -ne "0" ] && echo "Directory ${SOURCE_DIR} not found." >&2 && exit 1 + +declare DKMS_RPM +declare RHEL_ISO +declare MLNX_ISO + +for f in "${SOURCE_DIR}"/* +do + r="$(realpath "${f}")" + [ -f "${r}" ] || continue + case "${r##*/}" in + "RHEL-"*"-"*"-Server-${LINUX_ARCH}-dvd1.iso") + RHEL_ISO="${r}" + ;; + "MLNX_OFED_LINUX-"*"-"*"-${LINUX_DISTRO/s/}-${LINUX_ARCH}.iso") + MLNX_ISO="${r}" + ;; + "dkms-"*".el7.noarch.rpm") + DKMS_RPM="${r}" + ;; + esac +done + +######## +# Override the auto detect results here. + +#MLNX_ISO="${SOURCE_DIR}/mlnx-en-4.1-1.0.2.0-rhel7.3-ppc64le.iso" +#RHEL_ISO="${SOURCE_DIR}/RHEL-7.3-20161019.0-Server-ppc64le-dvd1.iso" +#DKMS_RPM="${SOURCE_DIR}/dkms-2.3-5.20170523git8c3065c.el7.noarch.rpm" + +######## +echo "Red Hat Enterprise Linux Server ISO" +echo "===================================" +echo "${RHEL_ISO}" +echo +echo "Mellanox EN Driver for Linux" +echo "============================" +echo "${MLNX_ISO}" +echo +echo "DKMS RPM" +echo "========" +echo "${DKMS_RPM}" +echo + +echo "The files listed above were found and will be used for this test case" +echo "Press Ctrl-C to abort!" +for t in {5..1} +do + echo -n " ... ${t}" + sleep 1 + echo -n -e "\b\b\b\b\b\b" +done +######## + +OSIMAGE_NAME="${LINUX_DISTRO}-${LINUX_ARCH}-install-mlnx" +OSIMAGE_OTHERPKGDIR="/install/post/otherpkgs/${LINUX_DISTRO}/${LINUX_ARCH}" +OSIMAGE_ROOTIMGDIR="/install/install/${LINUX_DISTRO}/${LINUX_ARCH}/${OSIMAGE_NAME}" + +[ -f "${RHEL_ISO}" ] +[ "$?" -ne "0" ] && echo "File ${RHEL_ISO} not found." >&2 && exit 1 +copycds "${RHEL_ISO}" +[ "$?" -ne "0" ] && echo "Copy CD failed." >&2 && exit 1 + +rmdef -t osimage "${OSIMAGE_NAME}" +mkdef -z <<-EOF +# + +${OSIMAGE_NAME}: + objtype=osimage + imagetype=linux + osarch=${LINUX_ARCH} + osdistroname=${LINUX_DISTRO}-${LINUX_ARCH} + osname=Linux + osvers=${LINUX_DISTRO} + otherpkgdir="${OSIMAGE_OTHERPKGDIR}" + otherpkglist=/install/custom/install/rh/mlnx.${LINUX_DISTRO%%.*}.${LINUX_ARCH}.otherpkgs.pkglist + pkgdir=/install/${LINUX_DISTRO}/${LINUX_ARCH} + pkglist=/install/custom/install/rh/mlnx.${LINUX_DISTRO%%.*}.${LINUX_ARCH}.pkglist + profile=compute + provmethod=install + template=/opt/xcat/share/xcat/install/rh/compute.${LINUX_DISTRO%%.*}.tmpl +EOF +[ "$?" -ne "0" ] && echo "Make osimage definition failed." >&2 && exit 1 + +mkdir -p /install/mlnx +cp "${MLNX_ISO}" /install/mlnx + +mkdir -p /install/custom/install/rh + +( + cat /opt/xcat/share/xcat/install/rh/compute.${LINUX_DISTRO%%.*}.pkglist + + cat <<-EOF + + # For MLNX OFED support + EOF + cat /opt/xcat/share/xcat/ib/netboot/rh/ib.${LINUX_DISTRO%%.*}.${LINUX_ARCH}.pkglist +) >"/install/custom/install/rh/mlnx.${LINUX_DISTRO%%.*}.${LINUX_ARCH}.pkglist" + +( + : +) >"/install/custom/install/rh/mlnx.${LINUX_DISTRO%%.*}.${LINUX_ARCH}.otherpkgs.pkglist" + +cp /opt/xcat/share/xcat/ib/scripts/Mellanox/mlnxofed_ib_install.v2 \ + /install/postscripts/mlnxofed_ib_install.v2 + +chdef "${COMPUTE_NODE}" "postscripts=syslog,remoteshell,syncfiles,mlnxofed_ib_install.v2 -p /install/mlnx/${MLNX_ISO##*/} -m --add-kernel-support" + +rm -rf "${OSIMAGE_OTHERPKGDIR}" +mkdir -p "${OSIMAGE_OTHERPKGDIR}" + +makedhcp -n +rinstall "${COMPUTE_NODE}" "osimage=${OSIMAGE_NAME}" + +NETBOOT_TIMEOUT=600 +declare -i WAIT=0 + +while sleep 10 +do + (( WAIT += 10 )) + nodestat "${COMPUTE_NODE}" | grep ': sshd$' + [ "$?" -eq "0" ] && break + [ "${WAIT}" -le "${NETBOOT_TIMEOUT}" ] + [ "$?" -ne "0" ] && echo "Netboot failed" >&2 && exit 1 +done + +# For workaround the GitHub issue #3549 +sleep 5 + +xdsh "${COMPUTE_NODE}" date +[ "$?" -ne "0" ] && echo "Failed connect to compute node via SSH." >&2 && exit 1 + +#xdsh "${COMPUTE_NODE}" 'rpm -q cuda' | grep ': cuda-' +#[ "$?" -ne "0" ] && echo "CUDA installation checking failed" >&2 && exit 1 + +exit 0 diff --git a/xCAT-test/autotest/testcase/infiniband/rhel-infiniband-diskless-install.sh b/xCAT-test/autotest/testcase/infiniband/rhel-infiniband-diskless-install.sh new file mode 100755 index 000000000..d9eb79eac --- /dev/null +++ b/xCAT-test/autotest/testcase/infiniband/rhel-infiniband-diskless-install.sh @@ -0,0 +1,176 @@ +#!/bin/bash + +######## +# Set all the variables below + +LINUX_DISTRO="rhels7.4" +LINUX_ARCH="ppc64le" + +COMPUTE_NODE="c910f03c01p10" +SOURCE_DIR="/media/xcat" + +######## + +# $SOURCE_DIR is a directory this test case will be searched for. +# Files with the name looked like the following will be searched. +# +# -rw-r--r-- 1 nobody nobody 79404 Jul 27 01:20 dkms-2.3-5.20170523git8c3065c.el7.noarch.rpm +# -rw-r--r-- 1 nobody nobody 29085696 Jul 9 09:24 mlnx-en-4.1-1.0.2.0-rhel7.3-ppc64le.iso +# -rw-r--r-- 2 nobody nobody 3188944896 Oct 31 2016 RHEL-7.3-20161019.0-Server-ppc64le-dvd1.iso + +######## +# Auto detect all the source packages from the ${SOURCE_DIR} + +[ -d "${SOURCE_DIR}" ] +[ "$?" -ne "0" ] && echo "Directory ${SOURCE_DIR} not found." >&2 && exit 1 + +declare DKMS_RPM +declare RHEL_ISO +declare MLNX_ISO + +for f in "${SOURCE_DIR}"/* +do + r="$(realpath "${f}")" + [ -f "${r}" ] || continue + case "${r##*/}" in + "RHEL-"*"-"*"-Server-${LINUX_ARCH}-dvd1.iso") + RHEL_ISO="${r}" + ;; + "MLNX_OFED_LINUX-"*"-"*"-${LINUX_DISTRO/s/}-${LINUX_ARCH}.iso") + MLNX_ISO="${r}" + ;; + "dkms-"*".el7.noarch.rpm") + DKMS_RPM="${r}" + ;; + esac +done + +######## +# Override the auto detect results here. + +#MLNX_ISO="${SOURCE_DIR}/mlnx-en-4.1-1.0.2.0-rhel7.3-ppc64le.iso" +#RHEL_ISO="${SOURCE_DIR}/RHEL-7.3-20161019.0-Server-ppc64le-dvd1.iso" +#DKMS_RPM="${SOURCE_DIR}/dkms-2.3-5.20170523git8c3065c.el7.noarch.rpm" + +######## +echo "Red Hat Enterprise Linux Server ISO" +echo "===================================" +echo "${RHEL_ISO}" +echo +echo "Mellanox EN Driver for Linux" +echo "============================" +echo "${MLNX_ISO}" +echo +echo "DKMS RPM" +echo "========" +echo "${DKMS_RPM}" +echo + +echo "The files listed above were found and will be used for this test case" +echo "Press Ctrl-C to abort!" +for t in {5..1} +do + echo -n " ... ${t}" + sleep 1 + echo -n -e "\b\b\b\b\b\b" +done +######## + +OSIMAGE_NAME="${LINUX_DISTRO}-${LINUX_ARCH}-netboot-mlnx" +OSIMAGE_OTHERPKGDIR="/install/post/otherpkgs/${LINUX_DISTRO}/${LINUX_ARCH}" +OSIMAGE_ROOTIMGDIR="/install/netboot/${LINUX_DISTRO}/${LINUX_ARCH}/${OSIMAGE_NAME}" + +[ -f "${RHEL_ISO}" ] +[ "$?" -ne "0" ] && echo "File ${RHEL_ISO} not found." >&2 && exit 1 +copycds "${RHEL_ISO}" +[ "$?" -ne "0" ] && echo "Copy CD failed." >&2 && exit 1 + +rmdef -t osimage "${OSIMAGE_NAME}" +mkdef -z <<-EOF +# + +${OSIMAGE_NAME}: + objtype=osimage + exlist=/opt/xcat/share/xcat/netboot/rh/compute.${LINUX_DISTRO%%.*}.${LINUX_ARCH}.exlist + imagetype=linux + osarch=${LINUX_ARCH} + osdistroname=${LINUX_DISTRO}-${LINUX_ARCH} + osname=Linux + osvers=${LINUX_DISTRO} + otherpkgdir="${OSIMAGE_OTHERPKGDIR}" + otherpkglist=/install/custom/netboot/rh/mlnx.${LINUX_DISTRO%%.*}.${LINUX_ARCH}.otherpkgs.pkglist + permission=755 + pkgdir=/install/${LINUX_DISTRO}/${LINUX_ARCH} + pkglist=/install/custom/netboot/rh/mlnx.${LINUX_DISTRO%%.*}.${LINUX_ARCH}.pkglist + postinstall=/install/custom/netboot/rh/mlnx.${LINUX_DISTRO%%.*}.${LINUX_ARCH}.postinstall + profile=compute + provmethod=netboot + rootimgdir=${OSIMAGE_ROOTIMGDIR} +EOF +[ "$?" -ne "0" ] && echo "Make osimage definition failed." >&2 && exit 1 + +mkdir -p /install/mlnx +cp "${MLNX_ISO}" /install/mlnx + +mkdir -p /install/custom/netboot/rh + +( + cat /opt/xcat/share/xcat/netboot/rh/compute.${LINUX_DISTRO%%.*}.${LINUX_ARCH}.pkglist + + cat <<-EOF + + # For MLNX OFED support + EOF + cat /opt/xcat/share/xcat/ib/netboot/rh/ib.${LINUX_DISTRO%%.*}.${LINUX_ARCH}.pkglist +) >"/install/custom/netboot/rh/mlnx.${LINUX_DISTRO%%.*}.${LINUX_ARCH}.pkglist" + +( + : +) >"/install/custom/netboot/rh/mlnx.${LINUX_DISTRO%%.*}.${LINUX_ARCH}.otherpkgs.pkglist" + +cp /opt/xcat/share/xcat/ib/scripts/Mellanox/mlnxofed_ib_install.v2 \ + /install/postscripts/mlnxofed_ib_install.v2 +( + cat "/opt/xcat/share/xcat/netboot/rh/compute.${LINUX_DISTRO%%.*}.${LINUX_ARCH}.postinstall" + cat <<-EOF + + NODESETSTATE=genimage IMGROOTPATH=${OSIMAGE_ROOTIMGDIR}/rootimg bash -x /install/postscripts/mlnxofed_ib_install.v2 -p /install/mlnx/${MLNX_ISO##*/} -m --add-kernel-support + EOF +) >"/install/custom/netboot/rh/mlnx.${LINUX_DISTRO%%.*}.${LINUX_ARCH}.postinstall" +chmod 0755 "/install/custom/netboot/rh/mlnx.${LINUX_DISTRO%%.*}.${LINUX_ARCH}.postinstall" + +rm -rf "${OSIMAGE_OTHERPKGDIR}" +mkdir -p "${OSIMAGE_OTHERPKGDIR}" + +rm -rf "${OSIMAGE_ROOTIMGDIR}" + +genimage "${OSIMAGE_NAME}" +[ "$?" -ne "0" ] && echo "genimage failed" >&2 && exit 1 +packimage "${OSIMAGE_NAME}" +[ "$?" -ne "0" ] && echo "packimage failed" >&2 && exit 1 + +makedhcp -n +rinstall "${COMPUTE_NODE}" "osimage=${OSIMAGE_NAME}" + +NETBOOT_TIMEOUT=600 +declare -i WAIT=0 + +while sleep 10 +do + (( WAIT += 10 )) + nodestat "${COMPUTE_NODE}" | grep ': sshd$' + [ "$?" -eq "0" ] && break + [ "${WAIT}" -le "${NETBOOT_TIMEOUT}" ] + [ "$?" -ne "0" ] && echo "Netboot failed" >&2 && exit 1 +done + +# For workaround the GitHub issue #3549 +sleep 5 + +xdsh "${COMPUTE_NODE}" date +[ "$?" -ne "0" ] && echo "Failed connect to compute node via SSH." >&2 && exit 1 + +#xdsh "${COMPUTE_NODE}" 'rpm -q cuda' | grep ': cuda-' +#[ "$?" -ne "0" ] && echo "CUDA installation checking failed" >&2 && exit 1 + +exit 0 From a4c5da3972dbdb96a907aac5718141574fd2820a Mon Sep 17 00:00:00 2001 From: GONG Jie Date: Thu, 17 Aug 2017 16:06:32 +0800 Subject: [PATCH 02/10] Add test case confignetwork_ib_ipoib --- xCAT-test/autotest/testcase/infiniband/cases0 | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 xCAT-test/autotest/testcase/infiniband/cases0 diff --git a/xCAT-test/autotest/testcase/infiniband/cases0 b/xCAT-test/autotest/testcase/infiniband/cases0 new file mode 100644 index 000000000..a4b366c1a --- /dev/null +++ b/xCAT-test/autotest/testcase/infiniband/cases0 @@ -0,0 +1,25 @@ +start:confignetwork_ib_ipoib +description: this case is to test confignetwork could configure secondarynic successfully with updatenode. +cmd:lsdef $$CN;if [ $? -eq 0 ]; then lsdef -l $$CN -z >/tmp/CN.standa ;fi +check:rc==0 +cmd:xdsh $$CN "mkdir -p /tmp/backupnet/" +check:rc==0 +cmd:if grep SUSE /etc/*release;then xdsh $$CN "cp -f /etc/sysconfig/network/ifcfg-* /tmp/backupnet/"; elif grep "Red Hat" /etc/*release;then xdsh $$CN "cp -f /etc/sysconfig/network-scripts/ifcfg-* /tmp/backupnet/"; elif grep Ubuntu /etc/*release;then xdsh $$CN "cp -f /etc/network/interfaces.d/* /tmp/backupnet/";else echo "Sorry,this is not supported os"; fi +check:rc==0 +cmd:mkdef -t network -o 11_1_0_0-255_255_0_0 net=11.1.0.0 mask=255.255.0.0 mgtifname=ib0 +check:rc==0 +cmd:chdef $$CN nicips.ib0=11.1.0.100 nictypes.ib0=infiniband nicnetworks.ib0=11_1_0_0-255_255_0_0 +check:rc==0 +cmd:updatenode $$CN -P confignetwork +check:rc==0 +cmd:if grep SUSE /etc/*release;then xdsh $$CN "grep 11.1.0.100 /etc/sysconfig/network/ifcfg-ib0"; elif grep "Red Hat" /etc/*release;then xdsh $$CN "grep 11.1.0.100 /etc/sysconfig/network-scripts/ifcfg-ib0"; elif grep Ubuntu /etc/*release;then xdsh $$CN "grep 11.1.0.100 /etc/network/interfaces.d/ib0";else echo "Sorry,this is not supported os"; fi +check:output=~11.1.0.100 +cmd:xdsh $$CN "ip addr del 11.1.0.100/16 dev ib0" +cmd:rmdef -t network -o 11_1_0_0-255_255_0_0 +cmd:if grep SUSE /etc/*release;then xdsh $$CN "rm -rf /etc/sysconfig/network/ifcfg-ib0"; elif grep "Red Hat" /etc/*release;then xdsh $$CN "rm -rf /etc/sysconfig/network-scripts/ifcfg-ib0"; elif grep Ubuntu /etc/*release;then xdsh $$CN "rm -rf /etc/network/interfaces.d/ib0";else echo "Sorry,this is not supported os"; fi +check:rc==0 +cmd:if [ -e /tmp/CN.standa ]; then rmdef $$CN; cat /tmp/CN.standa | mkdef -z; rm -rf /tmp/CN.standa; fi +check:rc==0 +cmd:if grep SUSE /etc/*release;then xdsh $$CN "cp -f /tmp/backupnet/* /etc/sysconfig/network/"; elif grep "Red Hat" /etc/*release;then xdsh $$CN "cp -f /tmp/backupnet/* /etc/sysconfig/network-scripts/"; elif grep Ubuntu /etc/*release;then xdsh $$CN "cp -f /tmp/backupnet/* /etc/network/interfaces.d/";else echo "Sorry,this is not supported os"; fi +cmd:xdsh $$CN "rm -rf /tmp/backupnet/" +end From c6c8370d488ca0d2d0ea97f7334c7c4f51a27def Mon Sep 17 00:00:00 2001 From: GONG Jie Date: Mon, 21 Aug 2017 13:22:17 +0800 Subject: [PATCH 03/10] Revise InfiniBand test case --- .../infiniband/rhel-infiniband-diskless-install.sh | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/xCAT-test/autotest/testcase/infiniband/rhel-infiniband-diskless-install.sh b/xCAT-test/autotest/testcase/infiniband/rhel-infiniband-diskless-install.sh index d9eb79eac..18f61a70e 100755 --- a/xCAT-test/autotest/testcase/infiniband/rhel-infiniband-diskless-install.sh +++ b/xCAT-test/autotest/testcase/infiniband/rhel-infiniband-diskless-install.sh @@ -142,6 +142,13 @@ chmod 0755 "/install/custom/netboot/rh/mlnx.${LINUX_DISTRO%%.*}.${LINUX_ARCH}.po rm -rf "${OSIMAGE_OTHERPKGDIR}" mkdir -p "${OSIMAGE_OTHERPKGDIR}" +mkdir -p "${OSIMAGE_OTHERPKGDIR}"/dkms +[ -f "${DKMS_RPM}" ] +[ "$?" -ne "0" ] && echo "File ${DKMS_RPM} not found." >&2 && exit 1 +cp "${DKMS_RPM}" "${OSIMAGE_OTHERPKGDIR}/dkms" + +( cd "${OSIMAGE_OTHERPKGDIR}" && createrepo . ) + rm -rf "${OSIMAGE_ROOTIMGDIR}" genimage "${OSIMAGE_NAME}" @@ -170,7 +177,7 @@ sleep 5 xdsh "${COMPUTE_NODE}" date [ "$?" -ne "0" ] && echo "Failed connect to compute node via SSH." >&2 && exit 1 -#xdsh "${COMPUTE_NODE}" 'rpm -q cuda' | grep ': cuda-' -#[ "$?" -ne "0" ] && echo "CUDA installation checking failed" >&2 && exit 1 +xdsh "${COMPUTE_NODE}" 'rpm -qa' | grep 'mlnx' +[ "$?" -ne "0" ] && echo "MLNX OFED installation checking failed" >&2 && exit 1 exit 0 From b8ce0c1c8a960d3a4f62c1e232546da708d0aa45 Mon Sep 17 00:00:00 2001 From: GONG Jie Date: Mon, 21 Aug 2017 13:27:28 +0800 Subject: [PATCH 04/10] Revise InfiniBand test case --- xCAT-test/autotest/testcase/infiniband/cases0 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xCAT-test/autotest/testcase/infiniband/cases0 b/xCAT-test/autotest/testcase/infiniband/cases0 index a4b366c1a..65fbcc04e 100644 --- a/xCAT-test/autotest/testcase/infiniband/cases0 +++ b/xCAT-test/autotest/testcase/infiniband/cases0 @@ -1,5 +1,5 @@ start:confignetwork_ib_ipoib -description: this case is to test confignetwork could configure secondarynic successfully with updatenode. +description: this case is to test confignetwork could configure the first infiniband interface, ib0, successfully with updatenode. cmd:lsdef $$CN;if [ $? -eq 0 ]; then lsdef -l $$CN -z >/tmp/CN.standa ;fi check:rc==0 cmd:xdsh $$CN "mkdir -p /tmp/backupnet/" From 4ead5aedb9e37567379fc24caeec5f0d42dc1f4a Mon Sep 17 00:00:00 2001 From: GONG Jie Date: Mon, 21 Aug 2017 13:29:42 +0800 Subject: [PATCH 05/10] Revise InfiniBand test case --- .../infiniband/rhel-infiniband-diskfull-install.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/xCAT-test/autotest/testcase/infiniband/rhel-infiniband-diskfull-install.sh b/xCAT-test/autotest/testcase/infiniband/rhel-infiniband-diskfull-install.sh index ec277a729..6cc575646 100755 --- a/xCAT-test/autotest/testcase/infiniband/rhel-infiniband-diskfull-install.sh +++ b/xCAT-test/autotest/testcase/infiniband/rhel-infiniband-diskfull-install.sh @@ -133,6 +133,11 @@ chdef "${COMPUTE_NODE}" "postscripts=syslog,remoteshell,syncfiles,mlnxofed_ib_in rm -rf "${OSIMAGE_OTHERPKGDIR}" mkdir -p "${OSIMAGE_OTHERPKGDIR}" +mkdir -p "${OSIMAGE_OTHERPKGDIR}"/dkms +[ -f "${DKMS_RPM}" ] +[ "$?" -ne "0" ] && echo "File ${DKMS_RPM} not found." >&2 && exit 1 +cp "${DKMS_RPM}" "${OSIMAGE_OTHERPKGDIR}/dkms" + makedhcp -n rinstall "${COMPUTE_NODE}" "osimage=${OSIMAGE_NAME}" @@ -154,7 +159,7 @@ sleep 5 xdsh "${COMPUTE_NODE}" date [ "$?" -ne "0" ] && echo "Failed connect to compute node via SSH." >&2 && exit 1 -#xdsh "${COMPUTE_NODE}" 'rpm -q cuda' | grep ': cuda-' -#[ "$?" -ne "0" ] && echo "CUDA installation checking failed" >&2 && exit 1 +xdsh "${COMPUTE_NODE}" 'rpm -qa' | grep 'mlnx' +[ "$?" -ne "0" ] && echo "MLNX OFED installation checking failed" >&2 && exit 1 exit 0 From 0b62a94bd5f973f4b4231fa087f10787d2329d98 Mon Sep 17 00:00:00 2001 From: GONG Jie Date: Wed, 23 Aug 2017 14:17:50 +0800 Subject: [PATCH 06/10] Make the Infiniband integration test case run with xCAT-test --- xCAT-test/autotest/testcase/infiniband/cases0 | 10 ++++++++++ .../infiniband/rhel-infiniband-diskfull-install.sh | 3 +++ .../infiniband/rhel-infiniband-diskless-install.sh | 3 +++ 3 files changed, 16 insertions(+) diff --git a/xCAT-test/autotest/testcase/infiniband/cases0 b/xCAT-test/autotest/testcase/infiniband/cases0 index 65fbcc04e..c66b66ca7 100644 --- a/xCAT-test/autotest/testcase/infiniband/cases0 +++ b/xCAT-test/autotest/testcase/infiniband/cases0 @@ -1,3 +1,13 @@ +start:rhel_infiniband_diskless_install +cmd:RHEL_ISO="$$ISO" LINUX_DISTRO="$$OS" COMPUTE_NODE="$$CN" MLNX_ISO="$$MLNX" DKMS_RPM="$$DKMS" /opt/xcat/share/xcat/tools/autotest/testcase/infiniband/rhel-infiniband-diskless-install.sh +check:rc==0 +end + +start:rhel_infiniband_diskfull_install +cmd:RHEL_ISO="$$ISO" LINUX_DISTRO="$$OS" COMPUTE_NODE="$$CN" MLNX_ISO="$$MLNX" DKMS_RPM="$$DKMS" /opt/xcat/share/xcat/tools/autotest/testcase/infiniband/rhel-infiniband-diskfull-install.sh +check:rc==0 +end + start:confignetwork_ib_ipoib description: this case is to test confignetwork could configure the first infiniband interface, ib0, successfully with updatenode. cmd:lsdef $$CN;if [ $? -eq 0 ]; then lsdef -l $$CN -z >/tmp/CN.standa ;fi diff --git a/xCAT-test/autotest/testcase/infiniband/rhel-infiniband-diskfull-install.sh b/xCAT-test/autotest/testcase/infiniband/rhel-infiniband-diskfull-install.sh index 6cc575646..66ceb74fe 100755 --- a/xCAT-test/autotest/testcase/infiniband/rhel-infiniband-diskfull-install.sh +++ b/xCAT-test/autotest/testcase/infiniband/rhel-infiniband-diskfull-install.sh @@ -3,9 +3,12 @@ ######## # Set all the variables below +[ -n "$LINUX_DISTRO" ] && LINUX_DISTRO="rhels7.4" +[ -n "$LINUX_ARCH" ] && LINUX_ARCH="ppc64le" +[ -n "$COMPUTE_NODE" ] && COMPUTE_NODE="c910f03c01p10" SOURCE_DIR="/media/xcat" diff --git a/xCAT-test/autotest/testcase/infiniband/rhel-infiniband-diskless-install.sh b/xCAT-test/autotest/testcase/infiniband/rhel-infiniband-diskless-install.sh index 18f61a70e..a5aad0e07 100755 --- a/xCAT-test/autotest/testcase/infiniband/rhel-infiniband-diskless-install.sh +++ b/xCAT-test/autotest/testcase/infiniband/rhel-infiniband-diskless-install.sh @@ -3,9 +3,12 @@ ######## # Set all the variables below +[ -n "$LINUX_DISTRO" ] && LINUX_DISTRO="rhels7.4" +[ -n "$LINUX_ARCH" ] && LINUX_ARCH="ppc64le" +[ -n "$COMPUTE_NODE" ] && COMPUTE_NODE="c910f03c01p10" SOURCE_DIR="/media/xcat" From aa068365d427ef17cf0c3edd82c0728e3048d801 Mon Sep 17 00:00:00 2001 From: GONG Jie Date: Wed, 23 Aug 2017 17:50:33 +0800 Subject: [PATCH 07/10] Revise InfiniBand test case --- xCAT-test/autotest/testcase/infiniband/cases0 | 3 +++ .../infiniband/rhel-infiniband-diskfull-install.sh | 12 +++++++----- .../infiniband/rhel-infiniband-diskless-install.sh | 4 ++-- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/xCAT-test/autotest/testcase/infiniband/cases0 b/xCAT-test/autotest/testcase/infiniband/cases0 index c66b66ca7..f634bae17 100644 --- a/xCAT-test/autotest/testcase/infiniband/cases0 +++ b/xCAT-test/autotest/testcase/infiniband/cases0 @@ -24,6 +24,9 @@ cmd:updatenode $$CN -P confignetwork check:rc==0 cmd:if grep SUSE /etc/*release;then xdsh $$CN "grep 11.1.0.100 /etc/sysconfig/network/ifcfg-ib0"; elif grep "Red Hat" /etc/*release;then xdsh $$CN "grep 11.1.0.100 /etc/sysconfig/network-scripts/ifcfg-ib0"; elif grep Ubuntu /etc/*release;then xdsh $$CN "grep 11.1.0.100 /etc/network/interfaces.d/ib0";else echo "Sorry,this is not supported os"; fi check:output=~11.1.0.100 +cmd:xdsh $$CN "ping -c 1 11.1.0.100" +check:rc==0 +# Clean up cmd:xdsh $$CN "ip addr del 11.1.0.100/16 dev ib0" cmd:rmdef -t network -o 11_1_0_0-255_255_0_0 cmd:if grep SUSE /etc/*release;then xdsh $$CN "rm -rf /etc/sysconfig/network/ifcfg-ib0"; elif grep "Red Hat" /etc/*release;then xdsh $$CN "rm -rf /etc/sysconfig/network-scripts/ifcfg-ib0"; elif grep Ubuntu /etc/*release;then xdsh $$CN "rm -rf /etc/network/interfaces.d/ib0";else echo "Sorry,this is not supported os"; fi diff --git a/xCAT-test/autotest/testcase/infiniband/rhel-infiniband-diskfull-install.sh b/xCAT-test/autotest/testcase/infiniband/rhel-infiniband-diskfull-install.sh index 66ceb74fe..ede9f8aa4 100755 --- a/xCAT-test/autotest/testcase/infiniband/rhel-infiniband-diskfull-install.sh +++ b/xCAT-test/autotest/testcase/infiniband/rhel-infiniband-diskfull-install.sh @@ -141,10 +141,12 @@ mkdir -p "${OSIMAGE_OTHERPKGDIR}"/dkms [ "$?" -ne "0" ] && echo "File ${DKMS_RPM} not found." >&2 && exit 1 cp "${DKMS_RPM}" "${OSIMAGE_OTHERPKGDIR}/dkms" +( cd "${OSIMAGE_OTHERPKGDIR}" && createrepo . ) + makedhcp -n rinstall "${COMPUTE_NODE}" "osimage=${OSIMAGE_NAME}" -NETBOOT_TIMEOUT=600 +INSTALL_TIMEOUT=1800 declare -i WAIT=0 while sleep 10 @@ -152,8 +154,8 @@ do (( WAIT += 10 )) nodestat "${COMPUTE_NODE}" | grep ': sshd$' [ "$?" -eq "0" ] && break - [ "${WAIT}" -le "${NETBOOT_TIMEOUT}" ] - [ "$?" -ne "0" ] && echo "Netboot failed" >&2 && exit 1 + [ "${WAIT}" -le "${INSTALL_TIMEOUT}" ] + [ "$?" -ne "0" ] && echo "Operating system installation failed" >&2 && exit 1 done # For workaround the GitHub issue #3549 @@ -162,7 +164,7 @@ sleep 5 xdsh "${COMPUTE_NODE}" date [ "$?" -ne "0" ] && echo "Failed connect to compute node via SSH." >&2 && exit 1 -xdsh "${COMPUTE_NODE}" 'rpm -qa' | grep 'mlnx' -[ "$?" -ne "0" ] && echo "MLNX OFED installation checking failed" >&2 && exit 1 +xdsh "${COMPUTE_NODE}" 'rpm -qa' | grep 'mlnx' +[ "$?" -ne "0" ] && echo "MLNX OFED installation checking failed." >&2 && exit 1 exit 0 diff --git a/xCAT-test/autotest/testcase/infiniband/rhel-infiniband-diskless-install.sh b/xCAT-test/autotest/testcase/infiniband/rhel-infiniband-diskless-install.sh index a5aad0e07..216ebaa63 100755 --- a/xCAT-test/autotest/testcase/infiniband/rhel-infiniband-diskless-install.sh +++ b/xCAT-test/autotest/testcase/infiniband/rhel-infiniband-diskless-install.sh @@ -180,7 +180,7 @@ sleep 5 xdsh "${COMPUTE_NODE}" date [ "$?" -ne "0" ] && echo "Failed connect to compute node via SSH." >&2 && exit 1 -xdsh "${COMPUTE_NODE}" 'rpm -qa' | grep 'mlnx' -[ "$?" -ne "0" ] && echo "MLNX OFED installation checking failed" >&2 && exit 1 +xdsh "${COMPUTE_NODE}" 'rpm -qa' | grep 'mlnx' +[ "$?" -ne "0" ] && echo "MLNX OFED installation checking failed." >&2 && exit 1 exit 0 From 32404bcc9aaec5cadf36df050f004f8048060072 Mon Sep 17 00:00:00 2001 From: GONG Jie Date: Thu, 24 Aug 2017 17:00:07 +0800 Subject: [PATCH 08/10] Revise InfiniBand test case --- xCAT-test/autotest/testcase/infiniband/cases0 | 4 ++-- .../rhel-infiniband-diskfull-install.sh | 21 +++++++++++++++---- .../rhel-infiniband-diskless-install.sh | 21 +++++++++++++++---- 3 files changed, 36 insertions(+), 10 deletions(-) diff --git a/xCAT-test/autotest/testcase/infiniband/cases0 b/xCAT-test/autotest/testcase/infiniband/cases0 index f634bae17..568dfc1b1 100644 --- a/xCAT-test/autotest/testcase/infiniband/cases0 +++ b/xCAT-test/autotest/testcase/infiniband/cases0 @@ -1,10 +1,10 @@ start:rhel_infiniband_diskless_install -cmd:RHEL_ISO="$$ISO" LINUX_DISTRO="$$OS" COMPUTE_NODE="$$CN" MLNX_ISO="$$MLNX" DKMS_RPM="$$DKMS" /opt/xcat/share/xcat/tools/autotest/testcase/infiniband/rhel-infiniband-diskless-install.sh +cmd:RHEL_ISO="$$ISO" LINUX_DISTRO="GETNODEATTR($$CN, os)" COMPUTE_NODE="$$CN" MLNX_ISO="$$MLNX" DKMS_RPM="$$DKMS" /opt/xcat/share/xcat/tools/autotest/testcase/infiniband/rhel-infiniband-diskless-install.sh check:rc==0 end start:rhel_infiniband_diskfull_install -cmd:RHEL_ISO="$$ISO" LINUX_DISTRO="$$OS" COMPUTE_NODE="$$CN" MLNX_ISO="$$MLNX" DKMS_RPM="$$DKMS" /opt/xcat/share/xcat/tools/autotest/testcase/infiniband/rhel-infiniband-diskfull-install.sh +cmd:RHEL_ISO="$$ISO" LINUX_DISTRO="GETNODEATTR($$CN, os)" COMPUTE_NODE="$$CN" MLNX_ISO="$$MLNX" DKMS_RPM="$$DKMS" /opt/xcat/share/xcat/tools/autotest/testcase/infiniband/rhel-infiniband-diskfull-install.sh check:rc==0 end diff --git a/xCAT-test/autotest/testcase/infiniband/rhel-infiniband-diskfull-install.sh b/xCAT-test/autotest/testcase/infiniband/rhel-infiniband-diskfull-install.sh index ede9f8aa4..3bc6fc7d0 100755 --- a/xCAT-test/autotest/testcase/infiniband/rhel-infiniband-diskfull-install.sh +++ b/xCAT-test/autotest/testcase/infiniband/rhel-infiniband-diskfull-install.sh @@ -1,7 +1,14 @@ #!/bin/bash ######## -# Set all the variables below +# +# For manually run this script in a standalone test environment without xCAT-test, +# do the following steps. +# +# * Set all the variables in LINE 11, 13, 16, and 17. +# * Download all the ISO files, RPMs needed, and put them in ${SOURCE_DIR} +# * If you intend to specify all the packages explicitly, set variable in LINE 61, 62, 63. +# [ -n "$LINUX_DISTRO" ] && LINUX_DISTRO="rhels7.4" @@ -9,8 +16,8 @@ LINUX_DISTRO="rhels7.4" LINUX_ARCH="ppc64le" [ -n "$COMPUTE_NODE" ] && -COMPUTE_NODE="c910f03c01p10" -SOURCE_DIR="/media/xcat" +COMPUTE_NODE="nonexistent" +SOURCE_DIR="/path/to/source" ######## @@ -155,7 +162,7 @@ do nodestat "${COMPUTE_NODE}" | grep ': sshd$' [ "$?" -eq "0" ] && break [ "${WAIT}" -le "${INSTALL_TIMEOUT}" ] - [ "$?" -ne "0" ] && echo "Operating system installation failed" >&2 && exit 1 + [ "$?" -ne "0" ] && echo "Operating system installation failed." >&2 && exit 1 done # For workaround the GitHub issue #3549 @@ -167,4 +174,10 @@ xdsh "${COMPUTE_NODE}" date xdsh "${COMPUTE_NODE}" 'rpm -qa' | grep 'mlnx' [ "$?" -ne "0" ] && echo "MLNX OFED installation checking failed." >&2 && exit 1 +xdsh "${COMPUTE_NODE}" 'lspci' +xdsh "${COMPUTE_NODE}" 'lsslot -c pci' +xdsh "${COMPUTE_NODE}" 'lsslot' +xdsh "${COMPUTE_NODE}" 'ibv_devinfo' +xdsh "${COMPUTE_NODE}" 'iblinkinfo' + exit 0 diff --git a/xCAT-test/autotest/testcase/infiniband/rhel-infiniband-diskless-install.sh b/xCAT-test/autotest/testcase/infiniband/rhel-infiniband-diskless-install.sh index 216ebaa63..014ef8082 100755 --- a/xCAT-test/autotest/testcase/infiniband/rhel-infiniband-diskless-install.sh +++ b/xCAT-test/autotest/testcase/infiniband/rhel-infiniband-diskless-install.sh @@ -1,7 +1,14 @@ #!/bin/bash ######## -# Set all the variables below +# +# For manually run this script in a standalone test environment without xCAT-test, +# do the following steps. +# +# * Set all the variables in LINE 11, 13, 16, and 17. +# * Download all the ISO files, RPMs needed, and put them in ${SOURCE_DIR} +# * If you intend to specify all the packages explicitly, set variable in LINE 61, 62, 63 +# [ -n "$LINUX_DISTRO" ] && LINUX_DISTRO="rhels7.4" @@ -9,8 +16,8 @@ LINUX_DISTRO="rhels7.4" LINUX_ARCH="ppc64le" [ -n "$COMPUTE_NODE" ] && -COMPUTE_NODE="c910f03c01p10" -SOURCE_DIR="/media/xcat" +COMPUTE_NODE="nonexistent" +SOURCE_DIR="/path/to/source" ######## @@ -171,7 +178,7 @@ do nodestat "${COMPUTE_NODE}" | grep ': sshd$' [ "$?" -eq "0" ] && break [ "${WAIT}" -le "${NETBOOT_TIMEOUT}" ] - [ "$?" -ne "0" ] && echo "Netboot failed" >&2 && exit 1 + [ "$?" -ne "0" ] && echo "Netboot failed." >&2 && exit 1 done # For workaround the GitHub issue #3549 @@ -183,4 +190,10 @@ xdsh "${COMPUTE_NODE}" date xdsh "${COMPUTE_NODE}" 'rpm -qa' | grep 'mlnx' [ "$?" -ne "0" ] && echo "MLNX OFED installation checking failed." >&2 && exit 1 +xdsh "${COMPUTE_NODE}" 'lspci' +xdsh "${COMPUTE_NODE}" 'lsslot -c pci' +xdsh "${COMPUTE_NODE}" 'lsslot' +xdsh "${COMPUTE_NODE}" 'ibv_devinfo' +xdsh "${COMPUTE_NODE}" 'iblinkinfo' + exit 0 From 89cb4ecdf1c6dd3f3fd0aa75729e8e59edd5ad74 Mon Sep 17 00:00:00 2001 From: GONG Jie Date: Fri, 25 Aug 2017 15:18:39 +0800 Subject: [PATCH 09/10] Revise InfiniBand test case --- xCAT-test/autotest/testcase/infiniband/cases0 | 2 ++ .../testcase/infiniband/rhel-infiniband-diskfull-install.sh | 2 +- .../testcase/infiniband/rhel-infiniband-diskless-install.sh | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/xCAT-test/autotest/testcase/infiniband/cases0 b/xCAT-test/autotest/testcase/infiniband/cases0 index 568dfc1b1..4f8c043a1 100644 --- a/xCAT-test/autotest/testcase/infiniband/cases0 +++ b/xCAT-test/autotest/testcase/infiniband/cases0 @@ -1,9 +1,11 @@ start:rhel_infiniband_diskless_install +description:Intigration test of RHEL 7 and MLNX OFED for diskless installation. Need to set MLNX variable in the test configuration file. cmd:RHEL_ISO="$$ISO" LINUX_DISTRO="GETNODEATTR($$CN, os)" COMPUTE_NODE="$$CN" MLNX_ISO="$$MLNX" DKMS_RPM="$$DKMS" /opt/xcat/share/xcat/tools/autotest/testcase/infiniband/rhel-infiniband-diskless-install.sh check:rc==0 end start:rhel_infiniband_diskfull_install +description:Intigration test of RHEL 7 and MLNX OFED for diskfull installation. Need to set MLNX variable in the test configuration file. cmd:RHEL_ISO="$$ISO" LINUX_DISTRO="GETNODEATTR($$CN, os)" COMPUTE_NODE="$$CN" MLNX_ISO="$$MLNX" DKMS_RPM="$$DKMS" /opt/xcat/share/xcat/tools/autotest/testcase/infiniband/rhel-infiniband-diskfull-install.sh check:rc==0 end diff --git a/xCAT-test/autotest/testcase/infiniband/rhel-infiniband-diskfull-install.sh b/xCAT-test/autotest/testcase/infiniband/rhel-infiniband-diskfull-install.sh index 3bc6fc7d0..7085a7545 100755 --- a/xCAT-test/autotest/testcase/infiniband/rhel-infiniband-diskfull-install.sh +++ b/xCAT-test/autotest/testcase/infiniband/rhel-infiniband-diskfull-install.sh @@ -145,7 +145,7 @@ mkdir -p "${OSIMAGE_OTHERPKGDIR}" mkdir -p "${OSIMAGE_OTHERPKGDIR}"/dkms [ -f "${DKMS_RPM}" ] -[ "$?" -ne "0" ] && echo "File ${DKMS_RPM} not found." >&2 && exit 1 +[ "$?" -ne "0" ] && echo "File ${DKMS_RPM} not found." >&2 cp "${DKMS_RPM}" "${OSIMAGE_OTHERPKGDIR}/dkms" ( cd "${OSIMAGE_OTHERPKGDIR}" && createrepo . ) diff --git a/xCAT-test/autotest/testcase/infiniband/rhel-infiniband-diskless-install.sh b/xCAT-test/autotest/testcase/infiniband/rhel-infiniband-diskless-install.sh index 014ef8082..f29cc891e 100755 --- a/xCAT-test/autotest/testcase/infiniband/rhel-infiniband-diskless-install.sh +++ b/xCAT-test/autotest/testcase/infiniband/rhel-infiniband-diskless-install.sh @@ -154,7 +154,7 @@ mkdir -p "${OSIMAGE_OTHERPKGDIR}" mkdir -p "${OSIMAGE_OTHERPKGDIR}"/dkms [ -f "${DKMS_RPM}" ] -[ "$?" -ne "0" ] && echo "File ${DKMS_RPM} not found." >&2 && exit 1 +[ "$?" -ne "0" ] && echo "File ${DKMS_RPM} not found." >&2 cp "${DKMS_RPM}" "${OSIMAGE_OTHERPKGDIR}/dkms" ( cd "${OSIMAGE_OTHERPKGDIR}" && createrepo . ) From f65c4e7862bb14608cad9c648f9215b1e2c6979c Mon Sep 17 00:00:00 2001 From: GONG Jie Date: Fri, 25 Aug 2017 15:51:07 +0800 Subject: [PATCH 10/10] Revise InfiniBand test case --- xCAT-test/autotest/testcase/infiniband/cases0 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xCAT-test/autotest/testcase/infiniband/cases0 b/xCAT-test/autotest/testcase/infiniband/cases0 index 4f8c043a1..2da6c8d92 100644 --- a/xCAT-test/autotest/testcase/infiniband/cases0 +++ b/xCAT-test/autotest/testcase/infiniband/cases0 @@ -1,11 +1,11 @@ start:rhel_infiniband_diskless_install -description:Intigration test of RHEL 7 and MLNX OFED for diskless installation. Need to set MLNX variable in the test configuration file. +description:Integration test of RHEL 7 and MLNX OFED for diskless installation. Following variables need to be set in the test configuration file, ISO, CN, MLNX, DKMS. Read comments in rhel-infiniband-diskless-install.sh for details. cmd:RHEL_ISO="$$ISO" LINUX_DISTRO="GETNODEATTR($$CN, os)" COMPUTE_NODE="$$CN" MLNX_ISO="$$MLNX" DKMS_RPM="$$DKMS" /opt/xcat/share/xcat/tools/autotest/testcase/infiniband/rhel-infiniband-diskless-install.sh check:rc==0 end start:rhel_infiniband_diskfull_install -description:Intigration test of RHEL 7 and MLNX OFED for diskfull installation. Need to set MLNX variable in the test configuration file. +description:Integration test of RHEL 7 and MLNX OFED for diskfull installation. Following variables need to be set in the test configuration file, ISO, CN, MLNX, DKMS. Read comments in rhel-infiniband-diskfull-install.sh for details. cmd:RHEL_ISO="$$ISO" LINUX_DISTRO="GETNODEATTR($$CN, os)" COMPUTE_NODE="$$CN" MLNX_ISO="$$MLNX" DKMS_RPM="$$DKMS" /opt/xcat/share/xcat/tools/autotest/testcase/infiniband/rhel-infiniband-diskfull-install.sh check:rc==0 end