2
0
mirror of https://github.com/xcat2/xcat-core.git synced 2025-05-29 17:23:08 +00:00

Merge pull request #3708 from neo954/testcase-infiniband

[Draft] Test cases of compute node deployment with MLNX OFED InfiniBand drivers
This commit is contained in:
Weihua Hu 2017-08-25 16:19:15 +08:00 committed by GitHub
commit 8ce8539fc7
3 changed files with 422 additions and 0 deletions

View File

@ -0,0 +1,40 @@
start:rhel_infiniband_diskless_install
description:Integration test of RHEL 7 and MLNX OFED for diskless installation. Following variables need to be set in the test configuration file, ISO, CN, MLNX, DKMS. Read comments in rhel-infiniband-diskless-install.sh for details.
cmd:RHEL_ISO="$$ISO" LINUX_DISTRO="GETNODEATTR($$CN, os)" COMPUTE_NODE="$$CN" MLNX_ISO="$$MLNX" DKMS_RPM="$$DKMS" /opt/xcat/share/xcat/tools/autotest/testcase/infiniband/rhel-infiniband-diskless-install.sh
check:rc==0
end
start:rhel_infiniband_diskfull_install
description:Integration test of RHEL 7 and MLNX OFED for diskfull installation. Following variables need to be set in the test configuration file, ISO, CN, MLNX, DKMS. Read comments in rhel-infiniband-diskfull-install.sh for details.
cmd:RHEL_ISO="$$ISO" LINUX_DISTRO="GETNODEATTR($$CN, os)" COMPUTE_NODE="$$CN" MLNX_ISO="$$MLNX" DKMS_RPM="$$DKMS" /opt/xcat/share/xcat/tools/autotest/testcase/infiniband/rhel-infiniband-diskfull-install.sh
check:rc==0
end
start:confignetwork_ib_ipoib
description: this case is to test confignetwork could configure the first infiniband interface, ib0, successfully with updatenode.
cmd:lsdef $$CN;if [ $? -eq 0 ]; then lsdef -l $$CN -z >/tmp/CN.standa ;fi
check:rc==0
cmd:xdsh $$CN "mkdir -p /tmp/backupnet/"
check:rc==0
cmd:if grep SUSE /etc/*release;then xdsh $$CN "cp -f /etc/sysconfig/network/ifcfg-* /tmp/backupnet/"; elif grep "Red Hat" /etc/*release;then xdsh $$CN "cp -f /etc/sysconfig/network-scripts/ifcfg-* /tmp/backupnet/"; elif grep Ubuntu /etc/*release;then xdsh $$CN "cp -f /etc/network/interfaces.d/* /tmp/backupnet/";else echo "Sorry,this is not supported os"; fi
check:rc==0
cmd:mkdef -t network -o 11_1_0_0-255_255_0_0 net=11.1.0.0 mask=255.255.0.0 mgtifname=ib0
check:rc==0
cmd:chdef $$CN nicips.ib0=11.1.0.100 nictypes.ib0=infiniband nicnetworks.ib0=11_1_0_0-255_255_0_0
check:rc==0
cmd:updatenode $$CN -P confignetwork
check:rc==0
cmd:if grep SUSE /etc/*release;then xdsh $$CN "grep 11.1.0.100 /etc/sysconfig/network/ifcfg-ib0"; elif grep "Red Hat" /etc/*release;then xdsh $$CN "grep 11.1.0.100 /etc/sysconfig/network-scripts/ifcfg-ib0"; elif grep Ubuntu /etc/*release;then xdsh $$CN "grep 11.1.0.100 /etc/network/interfaces.d/ib0";else echo "Sorry,this is not supported os"; fi
check:output=~11.1.0.100
cmd:xdsh $$CN "ping -c 1 11.1.0.100"
check:rc==0
# Clean up
cmd:xdsh $$CN "ip addr del 11.1.0.100/16 dev ib0"
cmd:rmdef -t network -o 11_1_0_0-255_255_0_0
cmd:if grep SUSE /etc/*release;then xdsh $$CN "rm -rf /etc/sysconfig/network/ifcfg-ib0"; elif grep "Red Hat" /etc/*release;then xdsh $$CN "rm -rf /etc/sysconfig/network-scripts/ifcfg-ib0"; elif grep Ubuntu /etc/*release;then xdsh $$CN "rm -rf /etc/network/interfaces.d/ib0";else echo "Sorry,this is not supported os"; fi
check:rc==0
cmd:if [ -e /tmp/CN.standa ]; then rmdef $$CN; cat /tmp/CN.standa | mkdef -z; rm -rf /tmp/CN.standa; fi
check:rc==0
cmd:if grep SUSE /etc/*release;then xdsh $$CN "cp -f /tmp/backupnet/* /etc/sysconfig/network/"; elif grep "Red Hat" /etc/*release;then xdsh $$CN "cp -f /tmp/backupnet/* /etc/sysconfig/network-scripts/"; elif grep Ubuntu /etc/*release;then xdsh $$CN "cp -f /tmp/backupnet/* /etc/network/interfaces.d/";else echo "Sorry,this is not supported os"; fi
cmd:xdsh $$CN "rm -rf /tmp/backupnet/"
end

View File

@ -0,0 +1,183 @@
#!/bin/bash
########
#
# For manually run this script in a standalone test environment without xCAT-test,
# do the following steps.
#
# * Set all the variables in LINE 11, 13, 16, and 17.
# * Download all the ISO files, RPMs needed, and put them in ${SOURCE_DIR}
# * If you intend to specify all the packages explicitly, set variable in LINE 61, 62, 63.
#
[ -n "$LINUX_DISTRO" ] &&
LINUX_DISTRO="rhels7.4"
[ -n "$LINUX_ARCH" ] &&
LINUX_ARCH="ppc64le"
[ -n "$COMPUTE_NODE" ] &&
COMPUTE_NODE="nonexistent"
SOURCE_DIR="/path/to/source"
########
# $SOURCE_DIR is a directory this test case will be searched for.
# Files with the name looked like the following will be searched.
#
# -rw-r--r-- 1 nobody nobody 79404 Jul 27 01:20 dkms-2.3-5.20170523git8c3065c.el7.noarch.rpm
# -rw-r--r-- 1 nobody nobody 29085696 Jul 9 09:24 mlnx-en-4.1-1.0.2.0-rhel7.3-ppc64le.iso
# -rw-r--r-- 2 nobody nobody 3188944896 Oct 31 2016 RHEL-7.3-20161019.0-Server-ppc64le-dvd1.iso
########
# Auto detect all the source packages from the ${SOURCE_DIR}
[ -d "${SOURCE_DIR}" ]
[ "$?" -ne "0" ] && echo "Directory ${SOURCE_DIR} not found." >&2 && exit 1
declare DKMS_RPM
declare RHEL_ISO
declare MLNX_ISO
for f in "${SOURCE_DIR}"/*
do
r="$(realpath "${f}")"
[ -f "${r}" ] || continue
case "${r##*/}" in
"RHEL-"*"-"*"-Server-${LINUX_ARCH}-dvd1.iso")
RHEL_ISO="${r}"
;;
"MLNX_OFED_LINUX-"*"-"*"-${LINUX_DISTRO/s/}-${LINUX_ARCH}.iso")
MLNX_ISO="${r}"
;;
"dkms-"*".el7.noarch.rpm")
DKMS_RPM="${r}"
;;
esac
done
########
# Override the auto detect results here.
#MLNX_ISO="${SOURCE_DIR}/mlnx-en-4.1-1.0.2.0-rhel7.3-ppc64le.iso"
#RHEL_ISO="${SOURCE_DIR}/RHEL-7.3-20161019.0-Server-ppc64le-dvd1.iso"
#DKMS_RPM="${SOURCE_DIR}/dkms-2.3-5.20170523git8c3065c.el7.noarch.rpm"
########
echo "Red Hat Enterprise Linux Server ISO"
echo "==================================="
echo "${RHEL_ISO}"
echo
echo "Mellanox EN Driver for Linux"
echo "============================"
echo "${MLNX_ISO}"
echo
echo "DKMS RPM"
echo "========"
echo "${DKMS_RPM}"
echo
echo "The files listed above were found and will be used for this test case"
echo "Press Ctrl-C to abort!"
for t in {5..1}
do
echo -n " ... ${t}"
sleep 1
echo -n -e "\b\b\b\b\b\b"
done
########
OSIMAGE_NAME="${LINUX_DISTRO}-${LINUX_ARCH}-install-mlnx"
OSIMAGE_OTHERPKGDIR="/install/post/otherpkgs/${LINUX_DISTRO}/${LINUX_ARCH}"
OSIMAGE_ROOTIMGDIR="/install/install/${LINUX_DISTRO}/${LINUX_ARCH}/${OSIMAGE_NAME}"
[ -f "${RHEL_ISO}" ]
[ "$?" -ne "0" ] && echo "File ${RHEL_ISO} not found." >&2 && exit 1
copycds "${RHEL_ISO}"
[ "$?" -ne "0" ] && echo "Copy CD failed." >&2 && exit 1
rmdef -t osimage "${OSIMAGE_NAME}"
mkdef -z <<-EOF
# <xCAT data object stanza file>
${OSIMAGE_NAME}:
objtype=osimage
imagetype=linux
osarch=${LINUX_ARCH}
osdistroname=${LINUX_DISTRO}-${LINUX_ARCH}
osname=Linux
osvers=${LINUX_DISTRO}
otherpkgdir="${OSIMAGE_OTHERPKGDIR}"
otherpkglist=/install/custom/install/rh/mlnx.${LINUX_DISTRO%%.*}.${LINUX_ARCH}.otherpkgs.pkglist
pkgdir=/install/${LINUX_DISTRO}/${LINUX_ARCH}
pkglist=/install/custom/install/rh/mlnx.${LINUX_DISTRO%%.*}.${LINUX_ARCH}.pkglist
profile=compute
provmethod=install
template=/opt/xcat/share/xcat/install/rh/compute.${LINUX_DISTRO%%.*}.tmpl
EOF
[ "$?" -ne "0" ] && echo "Make osimage definition failed." >&2 && exit 1
mkdir -p /install/mlnx
cp "${MLNX_ISO}" /install/mlnx
mkdir -p /install/custom/install/rh
(
cat /opt/xcat/share/xcat/install/rh/compute.${LINUX_DISTRO%%.*}.pkglist
cat <<-EOF
# For MLNX OFED support
EOF
cat /opt/xcat/share/xcat/ib/netboot/rh/ib.${LINUX_DISTRO%%.*}.${LINUX_ARCH}.pkglist
) >"/install/custom/install/rh/mlnx.${LINUX_DISTRO%%.*}.${LINUX_ARCH}.pkglist"
(
:
) >"/install/custom/install/rh/mlnx.${LINUX_DISTRO%%.*}.${LINUX_ARCH}.otherpkgs.pkglist"
cp /opt/xcat/share/xcat/ib/scripts/Mellanox/mlnxofed_ib_install.v2 \
/install/postscripts/mlnxofed_ib_install.v2
chdef "${COMPUTE_NODE}" "postscripts=syslog,remoteshell,syncfiles,mlnxofed_ib_install.v2 -p /install/mlnx/${MLNX_ISO##*/} -m --add-kernel-support"
rm -rf "${OSIMAGE_OTHERPKGDIR}"
mkdir -p "${OSIMAGE_OTHERPKGDIR}"
mkdir -p "${OSIMAGE_OTHERPKGDIR}"/dkms
[ -f "${DKMS_RPM}" ]
[ "$?" -ne "0" ] && echo "File ${DKMS_RPM} not found." >&2
cp "${DKMS_RPM}" "${OSIMAGE_OTHERPKGDIR}/dkms"
( cd "${OSIMAGE_OTHERPKGDIR}" && createrepo . )
makedhcp -n
rinstall "${COMPUTE_NODE}" "osimage=${OSIMAGE_NAME}"
INSTALL_TIMEOUT=1800
declare -i WAIT=0
while sleep 10
do
(( WAIT += 10 ))
nodestat "${COMPUTE_NODE}" | grep ': sshd$'
[ "$?" -eq "0" ] && break
[ "${WAIT}" -le "${INSTALL_TIMEOUT}" ]
[ "$?" -ne "0" ] && echo "Operating system installation failed." >&2 && exit 1
done
# For workaround the GitHub issue #3549
sleep 5
xdsh "${COMPUTE_NODE}" date
[ "$?" -ne "0" ] && echo "Failed connect to compute node via SSH." >&2 && exit 1
xdsh "${COMPUTE_NODE}" 'rpm -qa' | grep 'mlnx'
[ "$?" -ne "0" ] && echo "MLNX OFED installation checking failed." >&2 && exit 1
xdsh "${COMPUTE_NODE}" 'lspci'
xdsh "${COMPUTE_NODE}" 'lsslot -c pci'
xdsh "${COMPUTE_NODE}" 'lsslot'
xdsh "${COMPUTE_NODE}" 'ibv_devinfo'
xdsh "${COMPUTE_NODE}" 'iblinkinfo'
exit 0

View File

@ -0,0 +1,199 @@
#!/bin/bash
########
#
# For manually run this script in a standalone test environment without xCAT-test,
# do the following steps.
#
# * Set all the variables in LINE 11, 13, 16, and 17.
# * Download all the ISO files, RPMs needed, and put them in ${SOURCE_DIR}
# * If you intend to specify all the packages explicitly, set variable in LINE 61, 62, 63
#
[ -n "$LINUX_DISTRO" ] &&
LINUX_DISTRO="rhels7.4"
[ -n "$LINUX_ARCH" ] &&
LINUX_ARCH="ppc64le"
[ -n "$COMPUTE_NODE" ] &&
COMPUTE_NODE="nonexistent"
SOURCE_DIR="/path/to/source"
########
# $SOURCE_DIR is a directory this test case will be searched for.
# Files with the name looked like the following will be searched.
#
# -rw-r--r-- 1 nobody nobody 79404 Jul 27 01:20 dkms-2.3-5.20170523git8c3065c.el7.noarch.rpm
# -rw-r--r-- 1 nobody nobody 29085696 Jul 9 09:24 mlnx-en-4.1-1.0.2.0-rhel7.3-ppc64le.iso
# -rw-r--r-- 2 nobody nobody 3188944896 Oct 31 2016 RHEL-7.3-20161019.0-Server-ppc64le-dvd1.iso
########
# Auto detect all the source packages from the ${SOURCE_DIR}
[ -d "${SOURCE_DIR}" ]
[ "$?" -ne "0" ] && echo "Directory ${SOURCE_DIR} not found." >&2 && exit 1
declare DKMS_RPM
declare RHEL_ISO
declare MLNX_ISO
for f in "${SOURCE_DIR}"/*
do
r="$(realpath "${f}")"
[ -f "${r}" ] || continue
case "${r##*/}" in
"RHEL-"*"-"*"-Server-${LINUX_ARCH}-dvd1.iso")
RHEL_ISO="${r}"
;;
"MLNX_OFED_LINUX-"*"-"*"-${LINUX_DISTRO/s/}-${LINUX_ARCH}.iso")
MLNX_ISO="${r}"
;;
"dkms-"*".el7.noarch.rpm")
DKMS_RPM="${r}"
;;
esac
done
########
# Override the auto detect results here.
#MLNX_ISO="${SOURCE_DIR}/mlnx-en-4.1-1.0.2.0-rhel7.3-ppc64le.iso"
#RHEL_ISO="${SOURCE_DIR}/RHEL-7.3-20161019.0-Server-ppc64le-dvd1.iso"
#DKMS_RPM="${SOURCE_DIR}/dkms-2.3-5.20170523git8c3065c.el7.noarch.rpm"
########
echo "Red Hat Enterprise Linux Server ISO"
echo "==================================="
echo "${RHEL_ISO}"
echo
echo "Mellanox EN Driver for Linux"
echo "============================"
echo "${MLNX_ISO}"
echo
echo "DKMS RPM"
echo "========"
echo "${DKMS_RPM}"
echo
echo "The files listed above were found and will be used for this test case"
echo "Press Ctrl-C to abort!"
for t in {5..1}
do
echo -n " ... ${t}"
sleep 1
echo -n -e "\b\b\b\b\b\b"
done
########
OSIMAGE_NAME="${LINUX_DISTRO}-${LINUX_ARCH}-netboot-mlnx"
OSIMAGE_OTHERPKGDIR="/install/post/otherpkgs/${LINUX_DISTRO}/${LINUX_ARCH}"
OSIMAGE_ROOTIMGDIR="/install/netboot/${LINUX_DISTRO}/${LINUX_ARCH}/${OSIMAGE_NAME}"
[ -f "${RHEL_ISO}" ]
[ "$?" -ne "0" ] && echo "File ${RHEL_ISO} not found." >&2 && exit 1
copycds "${RHEL_ISO}"
[ "$?" -ne "0" ] && echo "Copy CD failed." >&2 && exit 1
rmdef -t osimage "${OSIMAGE_NAME}"
mkdef -z <<-EOF
# <xCAT data object stanza file>
${OSIMAGE_NAME}:
objtype=osimage
exlist=/opt/xcat/share/xcat/netboot/rh/compute.${LINUX_DISTRO%%.*}.${LINUX_ARCH}.exlist
imagetype=linux
osarch=${LINUX_ARCH}
osdistroname=${LINUX_DISTRO}-${LINUX_ARCH}
osname=Linux
osvers=${LINUX_DISTRO}
otherpkgdir="${OSIMAGE_OTHERPKGDIR}"
otherpkglist=/install/custom/netboot/rh/mlnx.${LINUX_DISTRO%%.*}.${LINUX_ARCH}.otherpkgs.pkglist
permission=755
pkgdir=/install/${LINUX_DISTRO}/${LINUX_ARCH}
pkglist=/install/custom/netboot/rh/mlnx.${LINUX_DISTRO%%.*}.${LINUX_ARCH}.pkglist
postinstall=/install/custom/netboot/rh/mlnx.${LINUX_DISTRO%%.*}.${LINUX_ARCH}.postinstall
profile=compute
provmethod=netboot
rootimgdir=${OSIMAGE_ROOTIMGDIR}
EOF
[ "$?" -ne "0" ] && echo "Make osimage definition failed." >&2 && exit 1
mkdir -p /install/mlnx
cp "${MLNX_ISO}" /install/mlnx
mkdir -p /install/custom/netboot/rh
(
cat /opt/xcat/share/xcat/netboot/rh/compute.${LINUX_DISTRO%%.*}.${LINUX_ARCH}.pkglist
cat <<-EOF
# For MLNX OFED support
EOF
cat /opt/xcat/share/xcat/ib/netboot/rh/ib.${LINUX_DISTRO%%.*}.${LINUX_ARCH}.pkglist
) >"/install/custom/netboot/rh/mlnx.${LINUX_DISTRO%%.*}.${LINUX_ARCH}.pkglist"
(
:
) >"/install/custom/netboot/rh/mlnx.${LINUX_DISTRO%%.*}.${LINUX_ARCH}.otherpkgs.pkglist"
cp /opt/xcat/share/xcat/ib/scripts/Mellanox/mlnxofed_ib_install.v2 \
/install/postscripts/mlnxofed_ib_install.v2
(
cat "/opt/xcat/share/xcat/netboot/rh/compute.${LINUX_DISTRO%%.*}.${LINUX_ARCH}.postinstall"
cat <<-EOF
NODESETSTATE=genimage IMGROOTPATH=${OSIMAGE_ROOTIMGDIR}/rootimg bash -x /install/postscripts/mlnxofed_ib_install.v2 -p /install/mlnx/${MLNX_ISO##*/} -m --add-kernel-support
EOF
) >"/install/custom/netboot/rh/mlnx.${LINUX_DISTRO%%.*}.${LINUX_ARCH}.postinstall"
chmod 0755 "/install/custom/netboot/rh/mlnx.${LINUX_DISTRO%%.*}.${LINUX_ARCH}.postinstall"
rm -rf "${OSIMAGE_OTHERPKGDIR}"
mkdir -p "${OSIMAGE_OTHERPKGDIR}"
mkdir -p "${OSIMAGE_OTHERPKGDIR}"/dkms
[ -f "${DKMS_RPM}" ]
[ "$?" -ne "0" ] && echo "File ${DKMS_RPM} not found." >&2
cp "${DKMS_RPM}" "${OSIMAGE_OTHERPKGDIR}/dkms"
( cd "${OSIMAGE_OTHERPKGDIR}" && createrepo . )
rm -rf "${OSIMAGE_ROOTIMGDIR}"
genimage "${OSIMAGE_NAME}"
[ "$?" -ne "0" ] && echo "genimage failed" >&2 && exit 1
packimage "${OSIMAGE_NAME}"
[ "$?" -ne "0" ] && echo "packimage failed" >&2 && exit 1
makedhcp -n
rinstall "${COMPUTE_NODE}" "osimage=${OSIMAGE_NAME}"
NETBOOT_TIMEOUT=600
declare -i WAIT=0
while sleep 10
do
(( WAIT += 10 ))
nodestat "${COMPUTE_NODE}" | grep ': sshd$'
[ "$?" -eq "0" ] && break
[ "${WAIT}" -le "${NETBOOT_TIMEOUT}" ]
[ "$?" -ne "0" ] && echo "Netboot failed." >&2 && exit 1
done
# For workaround the GitHub issue #3549
sleep 5
xdsh "${COMPUTE_NODE}" date
[ "$?" -ne "0" ] && echo "Failed connect to compute node via SSH." >&2 && exit 1
xdsh "${COMPUTE_NODE}" 'rpm -qa' | grep 'mlnx'
[ "$?" -ne "0" ] && echo "MLNX OFED installation checking failed." >&2 && exit 1
xdsh "${COMPUTE_NODE}" 'lspci'
xdsh "${COMPUTE_NODE}" 'lsslot -c pci'
xdsh "${COMPUTE_NODE}" 'lsslot'
xdsh "${COMPUTE_NODE}" 'ibv_devinfo'
xdsh "${COMPUTE_NODE}" 'iblinkinfo'
exit 0