mirror of
https://github.com/xcat2/xcat-core.git
synced 2025-05-30 09:36:41 +00:00
1402 lines
49 KiB
Perl
Executable File
1402 lines
49 KiB
Perl
Executable File
#!/usr/bin/env perl
|
|
|
|
$ENV{PATH} = "/opt/xcat/bin:/opt/xcat/sbin:/opt/xcat/share/xcat/tools:/usr/sbin:/usr/bin:/sbin:/bin";
|
|
|
|
use strict;
|
|
use warnings;
|
|
use Getopt::Long;
|
|
use Data::Dumper;
|
|
use Term::ANSIColor;
|
|
use Time::Local;
|
|
use IO::Select;
|
|
use File::Path;
|
|
use File::Spec;
|
|
use File::Basename;
|
|
use POSIX qw(WNOHANG setsid :errno_h);
|
|
|
|
#global arguments
|
|
my $date;
|
|
my $pro_name=basename($0);
|
|
my $homedir = dirname(File::Spec->rel2abs(__FILE__));
|
|
my $logfiledir = "";
|
|
my $logfile="";
|
|
my $globalconffile="$homedir/global.conf";
|
|
my $xcatpackagesdir="";
|
|
my $mailfile="";
|
|
my %confkeys; #global conf file
|
|
my %config; #cluster conf file
|
|
my $arch;
|
|
my $mn;
|
|
my $err_record="";
|
|
my $installlog="/var/log/xcat/autotest.log";
|
|
my $isodir="$homedir/iso";
|
|
my $orgclusterconffile="";
|
|
my $newclusterconffile="";
|
|
my $all_reg_time_consumption=0;
|
|
my $env_dply_time_consumption=0;
|
|
my $totalcase=0;
|
|
my $alltestpass=0;
|
|
my $teststopflag=0;
|
|
my $lastcase;
|
|
#my $defaultmail="huweihua\@cn.ibm.com,litingt\@cn.ibm.com,junxiaw\@cn.ibm.com,wxp\@cn.ibm.com,gongjie\@cn.ibm.com,bjcaomm\@cn.ibm.com,vhu\@us.ibm.com";
|
|
my $defaultmail="huweihua\@cn.ibm.com,litingt\@cn.ibm.com";
|
|
my $sub_process_rt=0;
|
|
my $proid;
|
|
|
|
# command line arguments
|
|
my $cluster_name;
|
|
my $os;
|
|
my $xcatcore_addr;
|
|
my $xcatdep_addr;
|
|
my $bucket;
|
|
my $mail_list = "";
|
|
my $quiet=0;
|
|
my $hold=0;
|
|
my $needhelp = 0;
|
|
my $proname="";
|
|
my $proruntime="";
|
|
|
|
|
|
#######################################
|
|
# send messages
|
|
#######################################
|
|
sub send_msg {
|
|
my $num = shift;
|
|
my $msg = shift;
|
|
my $content;
|
|
if ($num == 0) {
|
|
$content = "Fatal error:";
|
|
} elsif($num == 1) {
|
|
$content = "Warning:";
|
|
} elsif($num == 2) {
|
|
$content = "Notice:";
|
|
}
|
|
my $timestamp = `date +"%Y%m%d%H%M%S"`;
|
|
chomp($timestamp);
|
|
if ( !open (LOGFILE, ">> $logfiledir/$logfile") ) {
|
|
return 1;
|
|
}
|
|
print LOGFILE "$timestamp $$ $content $msg\n";
|
|
close LOGFILE;
|
|
|
|
if(!$quiet){
|
|
print "$timestamp $$ $content $msg\n";
|
|
}
|
|
}
|
|
|
|
#######################################
|
|
# runcmd
|
|
#######################################
|
|
sub runcmd {
|
|
my ($cmd) = @_;
|
|
my $rc = 0;
|
|
$::RUNCMD_RC = 0;
|
|
my $outref = [];
|
|
|
|
@$outref = `$cmd 2>&1`;
|
|
if ($?)
|
|
{
|
|
$rc = $? ;
|
|
$::RUNCMD_RC = $rc;
|
|
}
|
|
chomp(@$outref);
|
|
return @$outref;
|
|
}
|
|
#######################################
|
|
# parse cluser.conf file
|
|
#######################################
|
|
sub load_cluster_conf {
|
|
my $type = undef; #Script_Prev,Script_Post,Table,Object,System,Custom
|
|
my $sub_type = undef; # The string after $type_
|
|
my $name = undef;
|
|
my $attr = undef;
|
|
my $value = undef;
|
|
my $c = 0;
|
|
|
|
open(FILE, "$orgclusterconffile") or die "can't to open $orgclusterconffile";
|
|
while(my $line = <FILE>) {
|
|
$line =~ s/^\s+|#.+|\s+$//g;
|
|
next if(length($line) == 0);
|
|
|
|
#Table name can not contain "_"
|
|
if($line =~ /\[\s*(\w+)\_(\w+)\s*\]/) {
|
|
$type = $1;
|
|
$sub_type = $2;
|
|
$name = undef;
|
|
$c = 0;
|
|
}elsif($line =~ /\[\s*System|Custom\s*\]/){
|
|
$type = "Varible";
|
|
}elsif ($type eq "Table") {
|
|
##TABLE BLOCK##
|
|
if($line =~ /(\w+)\s*=\s*([\w\.\-]+)/) {
|
|
$attr = $1;
|
|
$value = $2;
|
|
if($name&&($config{table}{$sub_type}{$name}{__KEY__} ne $attr)){
|
|
$config{table}{$sub_type}{$name}{$attr}=$value;
|
|
} else {
|
|
$name = $value;
|
|
$config{table}{$sub_type}{$name}{__KEY__}=$attr;
|
|
}
|
|
}
|
|
}elsif ($type eq "Object") {
|
|
##OBJECT BLOCK##
|
|
if($line =~ /(\w+)\s*=\s*([:\w\.\-\/]+)/) {
|
|
$attr = $1;
|
|
$value = $2;
|
|
if($attr eq "Name"){
|
|
$name = $value;
|
|
} elsif(!defined($name)){
|
|
#print "Please give name for Object\n";
|
|
close FILE;
|
|
return 1;
|
|
} else {
|
|
$config{object}{$sub_type}{$name}{$attr}=$value;
|
|
}
|
|
}
|
|
}elsif ($type eq "Script") {
|
|
##SCRIPT_BLOCK##
|
|
if($sub_type eq "Prev") {
|
|
$config{script_prev}->[$c] = $line;
|
|
$c = $c + 1;
|
|
}
|
|
elsif ($sub_type eq "Post") {
|
|
$config{script_post}->[$c] = $line;
|
|
$c = $c + 1;
|
|
}
|
|
} elsif ($type eq "Varible") {
|
|
##NODE_BLOCK##
|
|
if($line =~ /(\w+)\s*=\s*([\w\.\-\+\/:]+)/) {
|
|
$config{var}{$1} = $2;
|
|
}
|
|
}
|
|
}
|
|
|
|
close FILE;
|
|
return 0;
|
|
}
|
|
#######################################
|
|
# rewrite cluster.conf file
|
|
#######################################
|
|
sub reset_cluster_conf{
|
|
if(exists $config{object}){
|
|
foreach my $type (keys %{$config{object}}){
|
|
foreach my $name (keys %{$config{object}{$type}}){
|
|
&runcmd("echo \"[Object_node]\" >> $newclusterconffile");
|
|
&runcmd("echo \"Name=$name\" >> $newclusterconffile");
|
|
foreach my $attr (keys %{$config{object}{$type}{$name}}){
|
|
&runcmd("echo \"$attr=$config{object}{$type}{$name}{$attr}\" >> $newclusterconffile");
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if(exists $config{table}){
|
|
foreach my $type (keys %{$config{table}}){
|
|
&runcmd("echo \"[Table_$type]\" >> $newclusterconffile");
|
|
foreach my $name (keys %{$config{table}{$type}}){
|
|
&runcmd("echo \"$config{table}{$type}{$name}{__KEY__} = $name\" >> $newclusterconffile");
|
|
foreach my $attr (keys %{$config{table}{$type}{$name}}){
|
|
if($attr ne '__KEY__'){
|
|
&runcmd("echo \"$attr=$config{table}{$type}{$name}{$attr}\" >> $newclusterconffile");
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if(exists $config{var}){
|
|
&runcmd("echo \"[System]\" >> $newclusterconffile");
|
|
foreach my $varname (keys %{$config{var}}){
|
|
&runcmd("echo \"$varname=$config{var}{$varname}\" >> $newclusterconffile");
|
|
}
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
#######################################
|
|
# get netboot value
|
|
#######################################
|
|
sub get_netboot_value{
|
|
my $osv=shift;
|
|
my $archv=shift;
|
|
my $mgtv=shift;
|
|
my $netbootv="none";
|
|
if($archv =~ /x86/i){
|
|
$netbootv="xnba";
|
|
}elsif($archv =~ /^ppc64$/i){
|
|
$osv =~ /(\D+)(.+)/;
|
|
my $version=$2;
|
|
if($1 =~ /rhel/i){
|
|
if($version>=7){
|
|
$netbootv="grub2";
|
|
}else{
|
|
$netbootv="yaboot";
|
|
}
|
|
}elsif($1 =~ /sles/i){
|
|
if($version<11.4){
|
|
$netbootv="yaboot";
|
|
}else{
|
|
$netbootv="grub2";
|
|
}
|
|
}
|
|
}elsif($archv =~ /ppc64le/i || $archv =~ /ppc64el/i){
|
|
if($mgtv =~ /^ipmi$/i){
|
|
$netbootv="petitboot";
|
|
}else{
|
|
$netbootv="grub2";
|
|
}
|
|
}
|
|
return $netbootv;
|
|
}
|
|
#######################################
|
|
# init
|
|
#######################################
|
|
sub init{
|
|
|
|
$proid="$proname-$proruntime";
|
|
$logfile="log.$proid";
|
|
$logfiledir="$homedir/log/$proname/$proruntime";
|
|
$xcatpackagesdir="$homedir/xcatpackages/$proname/$proruntime";
|
|
$orgclusterconffile="$homedir/$cluster_name/default.conf";
|
|
$newclusterconffile="$logfiledir/cluster.conf";
|
|
$mailfile="$logfiledir/mail.$proid";
|
|
|
|
mkpath("$logfiledir") unless(-d "$logfiledir");
|
|
mkpath("$xcatpackagesdir") unless(-d "$xcatpackagesdir");
|
|
|
|
return 0;
|
|
}
|
|
|
|
#######################################
|
|
# read global conf file
|
|
#######################################
|
|
sub read_conf{
|
|
my $myfile=undef;
|
|
my $line=undef;
|
|
|
|
if (!open($myfile, "$globalconffile")) {
|
|
send_msg(0, "Open $globalconffile failed");
|
|
return 1;
|
|
}
|
|
while ($line = <$myfile>) {
|
|
$line =~ s/\s//g;
|
|
next if($line =~ /^#/);
|
|
next if($line eq "");
|
|
my @attr=split(/=/,$line);
|
|
$confkeys{$attr[0]} = $attr[1];
|
|
}
|
|
close($myfile);
|
|
|
|
#for support both 'le' and 'el'
|
|
foreach my $k (keys %confkeys) {
|
|
if($k =~ /ppc64el/i || $k =~ /ppc64le/i){
|
|
if($k =~ /(.+)-(.+)-(.+)/){
|
|
my $newarch=$2;
|
|
my $tmp=$2;
|
|
my $os=$1;
|
|
my $flag=$3;
|
|
$newarch=~ s/ppc64el/ppc64le/g if($tmp =~ /ppc64el/i);
|
|
$newarch=~ s/ppc64le/ppc64el/g if($tmp =~ /ppc64le/i);
|
|
$confkeys{"$os-$newarch-iso"}=$confkeys{$k} if($flag=~/^iso$/);
|
|
$confkeys{"$os-$newarch-image"}=$confkeys{$k} if($flag=~/image/);
|
|
$confkeys{"$os-$newarch-miniiso"}=$confkeys{$k} if($flag=~/^miniiso$/);
|
|
}
|
|
}
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
#######################################
|
|
# usage for arguments
|
|
#######################################
|
|
sub usage{
|
|
print "Usage:$pro_name - Run xcat test cases with jenkins.\n
|
|
Explanation for the options:
|
|
--os: Required, specify the test operating system and version\n
|
|
--cluster: Required, specify the cluster where to run the test.\n
|
|
--testcase: Required, specify the test case list. value \"all\" means running daily regression test\n
|
|
--project-name: Required, specify the project name.\n
|
|
--num: Required, specifys this is how many times does this project run.\n
|
|
--xcat-core: Required, a web address, specify where to download the xcat-core package\n
|
|
--xcat-dep: Required, a web address, specify where to download the xcat-dep package\n
|
|
--email: a mail address, sepcify who should receive the test result report\n
|
|
--quiet: don't output to screen, just keep output in log file. output to screen by default\n
|
|
--hold: hold environment for 24 hours if some case stop and need to hold environment\n\n";
|
|
|
|
print " $pro_name [-?|-h]\n";
|
|
print " $pro_name --os <os> --cluster <clustername> --testcase <all|caselist> --project-name <projectname> --num <projectruntime> --xcat-core <downloadaddr> --xcat-dep <downloadaddr> --email <mailaddr>\n";
|
|
print "\n";
|
|
return;
|
|
}
|
|
|
|
#######################################
|
|
# get build from build server
|
|
#######################################
|
|
sub get_build{
|
|
send_msg(2, "start to get xcat build........");
|
|
|
|
&runcmd("cd $xcatpackagesdir >/dev/null 2>&1 && wget $xcatcore_addr >/dev/null 2>&1");
|
|
if($?){
|
|
send_msg(0, "[get_build] Can't download xcat-core from $xcatcore_addr");
|
|
return 1;
|
|
}
|
|
|
|
&runcmd("cd $xcatpackagesdir >/dev/null 2>&1 && wget $xcatdep_addr >/dev/null 2>&1");
|
|
if($?){
|
|
send_msg(0, "[get_build] Can't download xcat-dep from $xcatdep_addr");
|
|
return 1;
|
|
}
|
|
|
|
&runcmd("cd $xcatpackagesdir >/dev/null 2>&1 && wget http://xcat.org/files/go-xcat >/dev/null 2>&1");
|
|
if($?){
|
|
send_msg(0, "[get_build] Can't download go-xcat from xcat.org");
|
|
return 1;
|
|
}
|
|
|
|
send_msg(2, "get xcat build........[done]");
|
|
return 0;
|
|
}
|
|
|
|
#######################################
|
|
# do clean uo job when exit
|
|
#######################################
|
|
sub cleanup{
|
|
unlink("/tmp/totalcase.$proid") if(-e "/tmp/totalcase.$proid");
|
|
unlink("/tmp/failed_case_detail.$proid") if(-e "/tmp/failed_case_detail.$proid");
|
|
#rmdir("$xcatpackagesdir") if(-d "$xcatpackagesdir");
|
|
}
|
|
|
|
#######################################
|
|
# exit
|
|
#######################################
|
|
sub exit_test{
|
|
my $rst=shift;
|
|
&cleanup;
|
|
send_msg(2, "project $proid exit $rst");
|
|
exit $rst;
|
|
}
|
|
|
|
#######################################
|
|
# environment check
|
|
#######################################
|
|
sub env_check {
|
|
send_msg(2, "start to environment checking..........");
|
|
|
|
if(! -d "$homedir/$cluster_name"){
|
|
send_msg(0, "can't find $cluster_name under $homedir");
|
|
return 1;
|
|
}
|
|
|
|
if(! -e "$orgclusterconffile"){
|
|
send_msg(0, "can't find $orgclusterconffile");
|
|
return 1;
|
|
}
|
|
|
|
if($cluster_name =~ /(.+)-(.+)/){
|
|
$mn=$1;
|
|
#$arch=$2;
|
|
}else{
|
|
send_msg(0, "the name of $cluster_name is invalid");
|
|
return 1;
|
|
}
|
|
|
|
|
|
&load_cluster_conf;
|
|
if(!exists($config{object}{node}{$config{var}{CN}}{arch})){
|
|
send_msg(0, "can't fine arch of CN in $orgclusterconffile");
|
|
return 1;
|
|
}
|
|
|
|
$arch=$config{object}{node}{$config{var}{CN}}{arch};
|
|
|
|
if($arch !~ /^x86/i &&
|
|
$arch !~ /^ppc64$/i &&
|
|
$arch !~ /^ppc64le$/i &&
|
|
$arch !~ /^ppc64el$/i){
|
|
send_msg(0, "the arch $arch is invalid");
|
|
return 1;
|
|
}
|
|
|
|
send_msg(2, "environment checking..........[done]");
|
|
return 0;
|
|
}
|
|
|
|
#######################################
|
|
# install mn
|
|
######################################
|
|
sub mn_install {
|
|
send_msg(2, "[mn_install] start to install mn $mn.......");
|
|
|
|
&runcmd("lsdef|grep $mn");
|
|
if($?){
|
|
send_msg(0, "[mn_install] Can't find definition of $mn in current control node");
|
|
return 1;
|
|
}
|
|
|
|
if(!exists($confkeys{"$os-$arch-image"}) || !defined($confkeys{"$os-$arch-image"})){
|
|
send_msg(0, "[mn_install] Can't find image definition for $mn in global conf file");
|
|
return 1;
|
|
}
|
|
|
|
my $osimage= $confkeys{"$os-$arch-image"};
|
|
&runcmd("lsdef -t osimage|grep $osimage");
|
|
if($?){
|
|
send_msg(0, "[mn_install] Can't find definition of $osimage in current control node");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[mn_install] plan to install $osimage on mn $mn................");
|
|
|
|
my $mgt = `lsdef $mn |grep mgt|awk -F'=' '{print \$2}'`;
|
|
chomp($mgt);
|
|
my $netboot=get_netboot_value("$os","$arch","$mgt");
|
|
send_msg(2, "[mn_install] plan to set netboot of $mn [mgt=$mgt] to $netboot......");
|
|
&runcmd("chdef $mn netboot=$netboot");
|
|
if($?){
|
|
send_msg(0, "[mn_install] set netboot of $mn to $netboot failed");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[mn_install] set netboot of $mn to $netboot..[done]");
|
|
|
|
&runcmd("lsdef $mn -i postscripts|grep setupntp");
|
|
if($?){
|
|
send_msg(2, "[mn_install] plan to set setupntp as $mn 's postscript......");
|
|
&runcmd("chdef $mn -p postscripts=setupntp");
|
|
if($?){
|
|
send_msg(0, "[mn_install] set setupntp as $mn 's postscript failed");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[mn_install] set setupntp as $mn 's postscript...[done]");
|
|
}
|
|
|
|
if($os =~ /ubuntu/i){
|
|
if($arch =~ /le/i || $arch =~ /el/i){
|
|
my $miniisopath=$logfiledir;
|
|
my $downloadpath=$confkeys{"$os-$arch-miniiso"};
|
|
&runcmd("cd $miniisopath >/dev/null 2>&1 && wget $downloadpath >/dev/null 2>&1");
|
|
if($?){
|
|
send_msg(0, "[mn_install] Can't download mini.iso from $downloadpath");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[mn_install] download mini.iso from $downloadpath ....[done]");
|
|
mkpath("/install/$os/ppc64el/install/netboot") if(! -e "/install/$os/ppc64el/install/netboot");
|
|
rename("/install/$os/ppc64el/install/netboot/initrd.gz", "/install/$os/ppc64el/install/netboot/initrd.gz.bakupby$proid") if(-e "/install/$os/ppc64el/install/netboot/initrd.gz");
|
|
mkpath("$miniisopath/mountpoint");
|
|
&runcmd("mount -o loop $miniisopath/mini.iso $miniisopath/mountpoint");
|
|
&runcmd("cp $miniisopath/mountpoint/install/initrd.gz /install/$os/ppc64el/install/netboot/");
|
|
&runcmd("umount $miniisopath/mountpoint");
|
|
unlink("$miniisopath/mountpoint");
|
|
}
|
|
}
|
|
|
|
&runcmd("nodeset $mn osimage=$osimage");
|
|
if($?){
|
|
send_msg(0, "[mn_install] running nodeset error, please checking your configuration");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[mn_install] nodeset $mn osimage=$osimage...[done]");
|
|
|
|
if( $arch !~ /^ppc64$/i){
|
|
&runcmd("rpower $mn boot");
|
|
if($?){
|
|
send_msg(0, "[mn_install] rpower $mn boot error, please checking your configuration");
|
|
return 1;
|
|
}
|
|
}else{
|
|
&runcmd("rnetboot $mn");
|
|
if($?){
|
|
send_msg(0, "[mn_install] rnetboot $mn error, please checking your configuration");
|
|
return 1;
|
|
}
|
|
}
|
|
send_msg(2, "[mn_install] reboot $mn ...[done]");
|
|
|
|
#sleep while for installation.
|
|
&runcmd("sleep 360");
|
|
&runcmd("a=0;while ! `lsdef -l $mn|grep status|grep booted >/dev/null`; do sleep 10;((a++));if [ \$a -gt 400 ];then break;fi done");
|
|
|
|
&runcmd("lsdef -l $mn|grep status|grep booted >/dev/null");
|
|
if($?){
|
|
send_msg(0, "[mn_install] install $osimage on mn $mn failed");
|
|
return 1;
|
|
}
|
|
|
|
&runcmd("ping -c 2 $mn > /dev/null");
|
|
if($?){
|
|
send_msg(0, "[mn_install] ping $mn failed after installation");
|
|
return 1;
|
|
}
|
|
|
|
&runcmd("xdsh $mn date > /dev/null");
|
|
if($?){
|
|
send_msg(0, "[mn_install] xdsh $mn date failed after installation");
|
|
return 1;
|
|
}
|
|
|
|
send_msg(2, "install mn $mn .....[done]");
|
|
return 0;
|
|
}
|
|
|
|
|
|
#######################################
|
|
# prepare mn
|
|
# copy all necessary file to MN
|
|
######################################
|
|
sub prepare_mn {
|
|
send_msg(2, "[prepare_mn] start to prepare mn $mn.......");
|
|
|
|
if($os =~ /rhels7/i){
|
|
&runcmd("xdsh $mn \"yum -y install bzip2 \" >/dev/null 2>&1");
|
|
}
|
|
|
|
my $xcatcore=`ls -l /$xcatpackagesdir |grep core |awk '{print \$9}'`;
|
|
my $xcatdep=`ls -l /$xcatpackagesdir |grep dep|awk '{print \$9}'`;
|
|
chomp($xcatcore);
|
|
chomp($xcatdep);
|
|
if($xcatcore eq ""){
|
|
send_msg(0, "[prepare_mn] can't find xcat core under /$xcatpackagesdir");
|
|
return 1;
|
|
}
|
|
if($xcatdep eq ""){
|
|
send_msg(0, "[prepare_mn] can't find xcat dep under /$xcatpackagesdir");
|
|
return 1;
|
|
}
|
|
|
|
send_msg(2, "[prepare_mn] starting to copy $xcatcore and $xcatdep to $mn");
|
|
&runcmd("scp /$xcatpackagesdir/$xcatcore /$xcatpackagesdir/$xcatdep root\@$mn:/ >/dev/null");
|
|
if($?){
|
|
send_msg(0, "[prepare_mn] copy $xcatcore and $xcatdep to $mn failed");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[prepare_mn] copy $xcatcore and $xcatdep to $mn...[done]");
|
|
|
|
send_msg(2, "[prepare_mn] starting to copy go-xcat to $mn");
|
|
&runcmd("scp /$xcatpackagesdir/go-xcat root\@$mn:/ >/dev/null");
|
|
if($?){
|
|
send_msg(0, "[prepare_mn] copy go-xcat to $mn failed");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[prepare_mn] copy go-xcat to $mn...[done]");
|
|
|
|
send_msg(2, "[prepare_mn] starting to decompress xcat packages.....");
|
|
&runcmd("xdsh $mn 'cd / && tar xvf /$xcatcore' >/dev/null 2>&1");
|
|
if($?){
|
|
send_msg(0, "[prepare_mn] decompress $xcatcore on $mn failed");
|
|
return 1;
|
|
}
|
|
&runcmd("xdsh $mn 'cd / && tar xvf /$xcatdep' >/dev/null 2>&1");
|
|
if($?){
|
|
send_msg(0, "[prepare_mn] decompress $xcatdep on $mn failed");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[prepare_mn] decompress xcat packages....[done]");
|
|
|
|
#prepare /etc/hosts file on MN
|
|
&runcmd("scp /etc/hosts root\@$mn:/etc >/dev/null");
|
|
if ($?){
|
|
send_msg(0, "[prepare_mn] generate /etc/hosts on $mn failed");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[prepare_mn] generate /etc/hosts on $mn...[done]");
|
|
|
|
#prepare specific files on mn for each platform
|
|
if($os =~ /rhel/i){
|
|
if($arch =~ /ppc64le/i || $arch =~ /ppc64el/i){
|
|
&runcmd("scp /install/$os/ppc64le/RPM-GPG-KEY-redhat-release root\@$mn:/ >/dev/null");
|
|
}else{
|
|
&runcmd("scp /install/$os/$arch/RPM-GPG-KEY-redhat-release root\@$mn:/ >/dev/null");
|
|
}
|
|
if($?){
|
|
send_msg(0, "[prepare_mn] copy RPM-GPG-KEY-redhat-release to $mn failed");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[prepare_mn] copy RPM-GPG-KEY-redhat-release to $mn...[done]");
|
|
}
|
|
|
|
my $iso= $confkeys{"$os-$arch-iso"};
|
|
if ($iso eq ""){
|
|
send_msg(0, "[prepare_mn] can't find iso for $mn in golbal conf file");
|
|
return 1;
|
|
}
|
|
if(! -e "$isodir/$iso"){
|
|
send_msg(0, "[prepare_mn] can't find $iso under $isodir");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[prepare_mn] find $iso for os=$os and arch=$arch");
|
|
|
|
&runcmd("scp $isodir/$iso root\@$mn:/ >/dev/null");
|
|
if($?){
|
|
send_msg(0, "[prepare_mn] copy $iso to $mn failed");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[prepare_mn] copy $iso to $mn...[done]");
|
|
|
|
if($os =~ /ubuntu/i){
|
|
if($arch =~ /le/i || $arch =~ /el/i){
|
|
my $miniisopath=$logfiledir;
|
|
&runcmd("scp $miniisopath/mini.iso root\@$mn:/ >/dev/null 2>&1");
|
|
if($?){
|
|
send_msg(0, "[prepare_mn] copy $miniisopath/mini.iso to $mn failed");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[prepare_mn] copy $miniisopath/mini.iso to $mn...[done]");
|
|
}
|
|
}
|
|
|
|
&load_cluster_conf;
|
|
if(exists ($config{var}{SN})){
|
|
if(!exists($config{object}{node}{$config{var}{SN}}{arch})){
|
|
send_msg(0, "[prepare_mn] SN without arch attribute in $orgclusterconffile");
|
|
return 1;
|
|
}
|
|
if(!exists($config{object}{node}{$config{var}{SN}}{mgt})){
|
|
send_msg(0, "[prepare_mn] SN without mgt attribute in $orgclusterconffile");
|
|
return 1;
|
|
}
|
|
$config{object}{node}{$config{var}{SN}}{netboot}=get_netboot_value("$os","$config{object}{node}{$config{var}{SN}}{arch}","$config{object}{node}{$config{var}{SN}}{mgt}");
|
|
if($config{object}{node}{$config{var}{SN}}{netboot} eq "none"){
|
|
send_msg(0, "[prepare_mn] can't find appropriate netboot value for $os+$config{object}{node}{$config{var}{SN}}{arch}+$config{object}{node}{$config{var}{SN}}{mgt} scenario");
|
|
return 1;
|
|
}
|
|
$config{object}{node}{$config{var}{SN}}{os}=$os;
|
|
|
|
if($arch =~ /ppc64le/i || $arch =~ /ppc64el/i){
|
|
if($os =~ /ubuntu/i){
|
|
$config{object}{node}{$config{var}{SN}}{arch}="ppc64el";
|
|
}else{
|
|
$config{object}{node}{$config{var}{SN}}{arch}="ppc64le";
|
|
}
|
|
}
|
|
}
|
|
|
|
if(exists ($config{var}{CN})){
|
|
if(!exists($config{object}{node}{$config{var}{CN}}{arch})){
|
|
send_msg(0, "[prepare_mn] CN without arch attribute in $orgclusterconffile");
|
|
return 1;
|
|
}
|
|
if(!exists($config{object}{node}{$config{var}{CN}}{mgt})){
|
|
send_msg(0, "[prepare_mn] CN without mgt attribute in $orgclusterconffile");
|
|
return 1;
|
|
}
|
|
$config{object}{node}{$config{var}{CN}}{netboot}=get_netboot_value("$os","$config{object}{node}{$config{var}{CN}}{arch}","$config{object}{node}{$config{var}{CN}}{mgt}");
|
|
if($config{object}{node}{$config{var}{CN}}{netboot} eq "none"){
|
|
send_msg(0, "[prepare_mn] can't find appropriate netboot value for $os+$config{object}{node}{$config{var}{CN}}{arch}+$config{object}{node}{$config{var}{CN}}{mgt} scenario");
|
|
return 1;
|
|
}
|
|
$config{object}{node}{$config{var}{CN}}{os}=$os;
|
|
|
|
if($arch =~ /ppc64le/i || $arch =~ /ppc64el/i){
|
|
if($os =~ /ubuntu/i){
|
|
$config{object}{node}{$config{var}{CN}}{arch}="ppc64el";
|
|
}else{
|
|
$config{object}{node}{$config{var}{CN}}{arch}="ppc64le";
|
|
}
|
|
}
|
|
}
|
|
|
|
if(exists ($config{var}{ISO})){
|
|
$config{var}{ISO}="/$iso";
|
|
}
|
|
|
|
if(exists ($config{var}{OS})){
|
|
if($os =~ /(\D+)\d.*/){
|
|
$config{var}{OS}="$1";
|
|
}
|
|
}
|
|
&reset_cluster_conf;
|
|
|
|
send_msg(2, "prepare mn $mn .....[done]");
|
|
return 0;
|
|
}
|
|
|
|
|
|
############################################################
|
|
# install xcat-test package and other dependency packages
|
|
############################################################
|
|
sub install_xcattest {
|
|
send_msg(2, "[install_xcattest] starting to install xcat-test in $mn");
|
|
if($os =~ /rhel/i){
|
|
$os =~ /(\D+)(\d+)\.?(\d?)/;
|
|
&runcmd("xdsh $mn \"cd /xcat-core && ./mklocalrepo.sh\" >/dev/null 2>&1");
|
|
my $version=$2;
|
|
if($arch =~ /ppc64le/i || $arch =~ /ppc64el/i){
|
|
&runcmd("xdsh $mn \"cd /xcat-dep/rh$version/ppc64le && ./mklocalrepo.sh\" >/dev/null 2>&1");
|
|
}else{
|
|
&runcmd("xdsh $mn \"cd /xcat-dep/rh$version/$arch && ./mklocalrepo.sh\" >/dev/null 2>&1");
|
|
}
|
|
&runcmd("xdsh $mn \"rpm --import /RPM-GPG-KEY-redhat-release\" >/dev/null 2>&1");
|
|
&runcmd("xdsh $mn \"rpm -ivh /xcat-core/xCAT-test-*.rpm --nodeps > $installlog 2>&1 \" >/dev/null 2>&1");
|
|
&runcmd("xdsh $mn \"yum -y install createrepo expect\" >/dev/null 2>&1");
|
|
|
|
if($os =~ /rhels7/i){
|
|
&runcmd("xdsh $mn \"yum -y install mariadb-devel mariadb-libs mariadb-server mariadb-bench mariadb perl-DBD-MySQL mysql-connector-odbc unixODBC\">/dev/null 2>&1");
|
|
}elsif($os =~ /rhels6/i){
|
|
&runcmd("xdsh $mn \"yum -y install mysql-server mysql mysql-bench mysql-devel mysql-connector-odbc\" >/dev/null 2>&1");
|
|
}
|
|
|
|
if($arch =~ /x86/i){
|
|
&runcmd("xdsh $mn \"yum install -y perl-Sys-Virt\" >/dev/null 2>&1");
|
|
}
|
|
}elsif($os =~ /sles/i){
|
|
$os =~ /(\D+)(\d+)\.?(\d?)/;
|
|
my $version=$2;
|
|
if($arch =~ /ppc64le/i || $arch =~ /ppc64el/i){
|
|
&runcmd("xdsh $mn \"zypper ar file:///xcat-dep/sles$version/ppc64le xCAT-dep\" >/dev/null 2>&1");
|
|
}else{
|
|
&runcmd("xdsh $mn \"zypper ar file:///xcat-dep/sles$version/$arch xCAT-dep\" >/dev/null 2>&1");
|
|
}
|
|
&runcmd("xdsh $mn \"zypper ar file:///xcat-core xCAT-core\" >/dev/null 2>&1");
|
|
&runcmd("xdsh $mn \"zypper sl -U\" >/dev/null 2>&1");
|
|
&runcmd("xdsh $mn \"zypper --gpg-auto-import-keys search --match-exact -s screen\" >/dev/null 2>&1");
|
|
&runcmd("xdsh $mn \"rpm -ivh /xcat-core/xCAT-test-*.rpm --nodeps > $installlog 2>&1 \" >/dev/null 2>&1");
|
|
&runcmd("xdsh $mn \"zypper -n install createrepo expect\" >/dev/null 2>&1");
|
|
&runcmd("xdsh $mn \"zypper -n install mysql-client libmysqlclient_r15 libqt4-sql-mysql libmysqlclient15 perl-DBD-mysql mysql unixODBC\" >/dev/null 2>&1");
|
|
|
|
if($arch =~ /le/i || $arch =~ /el/i){
|
|
&runcmd ("xdsh $mn \"zypper -n install perl-Net-DNS-0.80-1.ppc64le\">/dev/null 2>&1");
|
|
}
|
|
if($arch =~ /x86/i){
|
|
&runcmd("xdsh $mn \"zypper -n install perl-Sys-Virt\" >/dev/null 2>&1");
|
|
}
|
|
}elsif($os =~ /ubuntu/i){
|
|
&runcmd("xdsh $mn \"echo \"nameserver 9.0.2.1\" >> /etc/resolv.conf\" >/dev/null 2>&1");
|
|
&runcmd("xdsh $mn 'apt-get -y install software-properties-common' >/dev/null 2>&1");
|
|
if($?){
|
|
send_msg(0, "[install_xcattest] apt-get -y install software-properties-common in $mn failed");
|
|
return 1;
|
|
}
|
|
|
|
if($arch =~ /x86/i){
|
|
&runcmd("xdsh $mn 'add-apt-repository \"deb http://archive.ubuntu.com/ubuntu \$(lsb_release -sc) main\"' >/dev/null 2>&1");
|
|
&runcmd("xdsh $mn 'add-apt-repository \"deb http://archive.ubuntu.com/ubuntu \$(lsb_release -sc)-updates main\"' >/dev/null 2>&1");
|
|
&runcmd("xdsh $mn 'add-apt-repository \"deb http://archive.ubuntu.com/ubuntu \$(lsb_release -sc) universe\"' >/dev/null 2>&1");
|
|
&runcmd("xdsh $mn 'add-apt-repository \"deb http://archive.ubuntu.com/ubuntu \$(lsb_release -sc)-updates universe\"' >/dev/null 2>&1");
|
|
}elsif($arch =~ /le/i || $arch =~ /el/i){
|
|
&runcmd("xdsh $mn 'add-apt-repository \"deb http://ports.ubuntu.com/ubuntu-ports \$(lsb_release -sc) main\"' >/dev/null 2>&1");
|
|
if($?){
|
|
send_msg(0, "[install_xcattest] add-apt-repository \"deb http://ports.ubuntu.com/ubuntu-ports \$(lsb_release -sc) main\" in $mn failed");
|
|
return 1;
|
|
}
|
|
&runcmd("xdsh $mn 'add-apt-repository \"deb http://ports.ubuntu.com/ubuntu-ports \$(lsb_release -sc)-updates main\"' >/dev/null 2>&1");
|
|
if($?){
|
|
send_msg(0, "[install_xcattest] add-apt-repository \"deb http://ports.ubuntu.com/ubuntu-ports \$(lsb_release -sc)-updates main\" in $mn failed");
|
|
return 1;
|
|
}
|
|
&runcmd("xdsh $mn 'add-apt-repository \"deb http://ports.ubuntu.com/ubuntu-ports \$(lsb_release -sc) universe\"' >/dev/null 2>&1");
|
|
if($?){
|
|
send_msg(0, "[install_xcattest] add-apt-repository \"deb http://ports.ubuntu.com/ubuntu-ports \$(lsb_release -sc) universe\" in $mn failed");
|
|
return 1;
|
|
}
|
|
&runcmd("xdsh $mn 'add-apt-repository \"deb http://ports.ubuntu.com/ubuntu-ports \$(lsb_release -sc)-updates universe\"' >/dev/null 2>&1");
|
|
if($?){
|
|
send_msg(0, "[install_xcattest] add-apt-repository \"deb http://ports.ubuntu.com/ubuntu-ports \$(lsb_release -sc)-updates universe\" in $mn failed");
|
|
return 1;
|
|
}
|
|
}
|
|
|
|
&runcmd("xdsh $mn \"wget -O - \"http://sourceforge.net/projects/xcat/files/ubuntu/apt.key/download\" | apt-key add -\" >/dev/null 2>&1");
|
|
if($?){
|
|
send_msg(0, "[install_xcattest] \"wget -O - \"http://sourceforge.net/projects/xcat/files/ubuntu/apt.key/download\" | apt-key add -\" in $mn failed");
|
|
return 1;
|
|
}
|
|
&runcmd("xdsh $mn 'apt-get clean all' >/dev/null 2>&1");
|
|
if($?){
|
|
send_msg(0, "[install_xcattest] apt-get clean all in $mn failed");
|
|
return 1;
|
|
}
|
|
&runcmd("xdsh $mn 'apt-get update' >/dev/null 2>&1");
|
|
&runcmd("xdsh $mn 'apt-get -y install build-essential dpkg-dev dh-make debhelper fakeroot gnupg lintian pbuilder quilt reprepro libsoap-lite-perl libdbi-perl' >/dev/null 2>&1");
|
|
if($?){
|
|
send_msg(0, "[install_xcattest] apt-get -y install build-essential dpkg-dev dh-make debhelper fakeroot gnupg lintian pbuilder quilt reprepro libsoap-lite-perl libdbi-perl in $mn failed");
|
|
return 1;
|
|
}
|
|
|
|
if($arch=~ /x86/i){
|
|
&runcmd("xdsh $mn \"/xcat-core/mklocalrepo.sh\" >/dev/null 2>&1");
|
|
&runcmd("xdsh $mn \"/xcat-dep/mklocalrepo.sh\" >/dev/null 2>&1");
|
|
&runcmd("xdsh $mn 'apt-get clean all' >/dev/null 2>&1");
|
|
&runcmd("xdsh $mn 'apt-get update' >/dev/null 2>&1");
|
|
&runcmd("xdsh $mn \"dpkg -i /xcat-core/pool/main/x/xcat-test/xcat-test_*.deb > $installlog 2>&1\" >/dev/null 2>&1");
|
|
}elsif($arch =~ /le/i || $arch =~ /el/i){
|
|
&runcmd("xdsh $mn \"/xcat-core/mklocalrepo.sh\" >/dev/null 2>&1");
|
|
if($?){
|
|
send_msg(0, "[install_xcattest] /xcat-core/mklocalrepo.sh in $mn failed");
|
|
return 1;
|
|
}
|
|
&runcmd("xdsh $mn \"/xcat-dep/mklocalrepo.sh\" >/dev/null 2>&1");
|
|
if($?){
|
|
send_msg(0, "[install_xcattest] /xcat-dep/mklocalrepo.sh in $mn failed");
|
|
return 1;
|
|
}
|
|
&runcmd("xdsh $mn 'apt-get clean all' >/dev/null 2>&1");
|
|
if($?){
|
|
send_msg(0, "[install_xcattest] apt-get clean all in $mn failed");
|
|
return 1;
|
|
}
|
|
&runcmd("xdsh $mn 'apt-get update' >/dev/null 2>&1");
|
|
&runcmd("xdsh $mn \"dpkg -i /xcat-core/pool/main/x/xcat-test/xcat-test_*.deb > $installlog 2>&1\" >/dev/null 2>&1");
|
|
if($?){
|
|
send_msg(0, "[install_xcattest] apt-get -y install xcat-test in $mn failed");
|
|
}else{
|
|
&runcmd("xdsh $mn 'mkdir -p /install/$os/%arch/install/netboot '");
|
|
&runcmd("scp /install/$os/%arch/install/netboot/initrd.gz $mn:/install/%os/%arch/install/netboot ");
|
|
}
|
|
}
|
|
}
|
|
|
|
&runcmd("scp $mn:/$installlog $logfiledir/new_xcattest_installation.log >/dev/null 2>&1");
|
|
|
|
#check if xcat-test is installed successfully on mn
|
|
if($os =~ /rhel/i || $os =~ /sles/i){
|
|
&runcmd("xdsh $mn \"rpm -qa |grep -i xCAT-test\" >/dev/null 2>&1");
|
|
}else{
|
|
&runcmd("xdsh $mn 'dpkg -l |grep -i xcat-test' >/dev/null 2>&1");
|
|
}
|
|
|
|
if($?){
|
|
send_msg(0, "[install_xcattest] install xcat-test in $mn failed");
|
|
return 1;
|
|
}
|
|
|
|
#export some environment variables
|
|
&runcmd("xdsh $mn \"touch /etc/profile.d/xcatjk.sh; echo -e \\\"XCATROOT=/opt/xcat\nPATH=\$XCATROOT/bin:\$XCATROOT/sbin:\$XCATROOT/share/xcat/tools:\$PATH\nMANPATH=\$XCATROOT/share/man:\$MANPATH\nexport XCATROOT PATH MANPATH\nexport PERL_BADLANG=0\\\" >> /etc/profile.d/xcatjk.sh;source /etc/profile.d/xcatjk.sh\" >/dev/null 2>&1");
|
|
if($?){
|
|
send_msg(2, "[install_xcattest] set environment variables failed");
|
|
return 1;
|
|
}else{
|
|
send_msg(2, "[install_xcattest] set environment variables succeeded");
|
|
}
|
|
|
|
send_msg(2, "[install_xcattest] install xcat-test in $mn successfully");
|
|
return 0;
|
|
|
|
}
|
|
|
|
#######################################
|
|
# do test
|
|
#######################################
|
|
sub do_test {
|
|
my $casestop=0;
|
|
my $installcasestop=0;
|
|
|
|
send_msg(2, "[do_test] starting to run regression test in $mn");
|
|
|
|
&runcmd("scp $newclusterconffile root\@$mn:/opt/xcat/share/xcat/tools/autotest/default.conf >/dev/null");
|
|
if ($?){
|
|
send_msg(0, "[do_test] copy $newclusterconffile to $mn failed");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[do_test] copy $newclusterconffile to $mn successfully");
|
|
|
|
#this is a workaround in ubuntu environment
|
|
if($os =~ /ubuntu/i){
|
|
send_msg(2, "[do_test] change /bin/sh setting in ubuntu environment");
|
|
runcmd("xdsh $mn \"rm -rf /bin/sh\" >/dev/null 2>&1");
|
|
runcmd("xdsh $mn \"ln -s /bin/bash /bin/sh\"");
|
|
}
|
|
|
|
if($bucket =~ /^all$/i){
|
|
my $bundle=$os."_".$arch.".bundle";
|
|
|
|
&runcmd("xdsh $mn 'ls -l /opt/xcat/share/xcat/tools/autotest/bundle/|grep $bundle' >/dev/null 2>&1");
|
|
if($?){
|
|
send_msg(0, "[do_test] can't find $bundle under $mn:/opt/xcat/share/xcat/tools/autotest/bundle/ ");
|
|
return 1;
|
|
}
|
|
|
|
&runcmd("scp -r $mn:/opt/xcat/share/xcat/tools/autotest/bundle/$bundle $logfiledir >/dev/null 2>&1");
|
|
if($?){
|
|
send_msg(0, "[do_test] can't copy $mn:/opt/xcat/share/xcat/tools/autotest/bundle/$bundle back");
|
|
return 1;
|
|
}
|
|
|
|
if(!open(FILE, "<$logfiledir/$bundle")){
|
|
send_msg(0, "[do_test] can't open $logfiledir/$bundle");
|
|
return 1;
|
|
}
|
|
|
|
my $casecnt=0;
|
|
my $line="";
|
|
while($line=<FILE>){
|
|
$line =~ s/^\s+|#.+|\s+$//g;
|
|
next if(length($line) == 0);
|
|
$casecnt++;
|
|
}
|
|
close(FILE);
|
|
&runcmd("echo $casecnt > /tmp/totalcase.$proid");
|
|
|
|
#run the special install_xcat case first
|
|
my $rst1=0;
|
|
my @output1;
|
|
send_msg(2, "[do_test] doing test for case install_xcat in $mn.....");
|
|
if($os !~ /ubuntu/i){
|
|
@output1=runcmd("xdsh $mn \"xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf:System -t install_xCAT_on_rhels_sles >/dev/null\"");
|
|
$rst1=$?;
|
|
}else{
|
|
@output1=runcmd("ssh -t $mn 'exec bash -l -i -c \"xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf:System -t install_xCAT_on_ubuntu >/dev/null\"'");
|
|
$rst1=$?;
|
|
}
|
|
|
|
if($rst1){
|
|
$installcasestop=1;
|
|
my $tmpoutput1=join(' ', @output1);
|
|
send_msg(2, "[do_test] install_xcat output of trigger xcattest: $tmpoutput1");
|
|
send_msg(2, "[do_test] the install_xcat case on $mn was STOPPED for some reason");
|
|
}else{
|
|
send_msg(2, "[do_test] run install_xcat in $mn finished");
|
|
}
|
|
|
|
if($installcasestop)
|
|
{
|
|
&runcmd("scp -r $mn:/opt/xcat/share/xcat/tools/autotest/result/* $logfiledir >/dev/null 2>&1");
|
|
if ($?){
|
|
send_msg(0, "[do_test] copy install_case result to $logfiledir failed");
|
|
}
|
|
return 2;
|
|
}
|
|
|
|
#then run other cases with the bundle file
|
|
|
|
my $rst=0;
|
|
my @output;
|
|
send_msg(2, "[do_test] doing test [$bundle] in $mn.....");
|
|
if($os !~ /ubuntu/i){
|
|
@output=runcmd("xdsh $mn \"xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf -b $bundle >/dev/null\"");
|
|
$rst=$?;
|
|
}else{
|
|
@output=runcmd("ssh -t $mn 'exec bash -l -i -c \"xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf -b $bundle >/dev/null\"'");
|
|
$rst=$?;
|
|
}
|
|
my $tmpoutput=join(' ', @output);
|
|
send_msg(2, "[do_test] output of trigger xcattest: $tmpoutput");
|
|
if($rst){
|
|
$casestop=1;
|
|
send_msg(2, "[do_test] the regression job in $mn was STOPPED for some one case");
|
|
}else{
|
|
send_msg(2, "[do_test] run the whole regression test in $mn finished");
|
|
}
|
|
|
|
&runcmd("scp -r $mn:/opt/xcat/share/xcat/tools/autotest/result/* $logfiledir >/dev/null 2>&1");
|
|
if ($?){
|
|
send_msg(0, "[do_test] copy regression result to $logfiledir failed");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[do_test] copy regression result to $logfiledir successfully");
|
|
}else{
|
|
#to do part
|
|
}
|
|
|
|
|
|
|
|
return 2 if($casestop);
|
|
return 0;
|
|
}
|
|
|
|
#######################################
|
|
# creat report
|
|
#######################################
|
|
sub create_report{
|
|
send_msg(2, "[create_report] start to create test report");
|
|
|
|
my $mailreport .= "======================================\n";
|
|
$mailreport .= " Test Result for Project $proid\n";
|
|
$mailreport .= "======================================\n\n";
|
|
$mailreport .= "[$os+$arch]\n\n";
|
|
|
|
if($sub_process_rt){
|
|
my $h=int($all_reg_time_consumption/3600);
|
|
my $m=int(($all_reg_time_consumption - $h*3600)/60);
|
|
my $s=($all_reg_time_consumption - $h*3600)%60;
|
|
$mailreport .= "\tAll time consumption: $h hours $m minutes $s seconds\n\n";
|
|
|
|
if($env_dply_time_consumption){
|
|
$h=int($env_dply_time_consumption/3600);
|
|
$m=int(($env_dply_time_consumption - $h*3600)/60);
|
|
$s=($env_dply_time_consumption- $h*3600)%60;
|
|
$mailreport .= "\tDeploy test environment time consumption: $h hours $m minutes $s seconds\n\n";
|
|
}
|
|
|
|
if($err_record ne ""){
|
|
$mailreport .= "\tDeploy MN result: $err_record\n\n";
|
|
}else{
|
|
if(-e "/tmp/totalcase.$proid"){
|
|
my $tmp = `cat /tmp/totalcase.$proid`;
|
|
chomp($tmp);
|
|
$totalcase=$tmp;
|
|
}
|
|
|
|
my $totalcnt=0;
|
|
my $failcnt=0;
|
|
my $faillist="";
|
|
opendir(DIR, "$logfiledir");
|
|
foreach my $file (readdir DIR){
|
|
next if($file !~ /xcattest.log/);
|
|
my $cnt = `cat $logfiledir/$file |grep -- "------END::"|wc -l`;
|
|
$totalcnt+=int($cnt);
|
|
for(my $i=1;$i<$cnt+1;$i++){
|
|
my $line=`cat $logfiledir/$file |grep -- "------END::"|sed -n ${i}p`;
|
|
chomp($line);
|
|
if($line =~ /------END::([a-zA-Z0-9_-]+)::([a-zA-Z0-9_-]+)::Time.+/){
|
|
my $failedcase=$1;
|
|
if($2 =~ /Failed/){
|
|
$failcnt++;
|
|
$faillist.=$failedcase.", ";
|
|
}
|
|
}
|
|
}
|
|
}
|
|
closedir(DIR);
|
|
$mailreport .= "\tTotalCase $totalcase TotalRun $totalcnt Failed $failcnt\n\n\tFailed cases: $faillist\n\n";
|
|
|
|
if($teststopflag){
|
|
my $lastfile = `ls -l $logfiledir|grep "xcattest.log"|tail -1|awk '{print \$9}'`;
|
|
chomp($lastfile);
|
|
my $lastline=`cat $logfiledir/$lastfile |grep -- "------END::"|tail -1`;
|
|
chomp($lastline);
|
|
$lastcase=$1 if($lastline =~ /------END::([a-zA-Z0-9_-]+)::([a-zA-Z0-9_-]+)::Time.+/);
|
|
$mailreport .= "\tFinal environment stop at $lastcase\n";
|
|
}
|
|
|
|
$alltestpass=1 if($totalcase!=0 && $failcnt==0 && $totalcnt==$totalcase);
|
|
}
|
|
}else{
|
|
$mailreport .= "\tTime consumption: Out of 8 hours!!!!!!\n";
|
|
}
|
|
|
|
$mailreport .= "\n-------------------------------------------\nCluster Information:\n";
|
|
$mailreport .= "\t MN: $mn\n";
|
|
$mailreport .= "\t SN: $config{var}{SN}\n";
|
|
$mailreport .= "\t CN: $config{var}{CN}\n";
|
|
$mailreport .= "\t Test result are saved under $mn:/opt/xcat/share/xcat/tools/autotest/result/\n" if($teststopflag && $hold);
|
|
|
|
&runcmd("touch $mailfile && echo \"$mailreport\" > $mailfile");
|
|
send_msg(2, "[create_report] created test report.....[done]");
|
|
return 0;
|
|
}
|
|
|
|
sub send_mail{
|
|
my $totalcnt=0;
|
|
my $failcnt=0;
|
|
my $subject="";
|
|
my $attachfile="";
|
|
|
|
if($err_record ne ""){
|
|
$subject = "[JK][$proid][FAILED] $err_record";
|
|
}else{
|
|
my $output = `cat $mailfile |grep "Total"|grep "Failed"`;
|
|
chomp($output);
|
|
if($output =~ /TotalCase (\d+) TotalRun (\d+) Failed (\d+)/){
|
|
$totalcnt=$2;
|
|
$failcnt=$3;
|
|
}
|
|
|
|
if($failcnt > 0){
|
|
$attachfile="/tmp/failed_case_detail.$proid";
|
|
opendir(DIR, "$logfiledir");
|
|
foreach my $file (readdir DIR){
|
|
next if($file !~ /^failedcases/);
|
|
next if(-z "$logfiledir/$file");
|
|
&runcmd("cat $logfiledir/$file >> $attachfile");
|
|
}
|
|
close(DIR);
|
|
}
|
|
|
|
if($teststopflag){
|
|
$subject = "[JK][$proid][STOP for $lastcase] TotalCase:$totalcase TotalRun:$totalcnt Fail:$failcnt";
|
|
}else{
|
|
if($alltestpass){
|
|
$subject = "[JK][$proid][SUCCESS] TotalCase:$totalcase TotalRun:$totalcnt Fail:$failcnt";
|
|
}else{
|
|
$subject = "[JK][$proid][FAILED] TotalCase:$totalcase TotalRun:$totalcnt Fail:$failcnt";
|
|
}
|
|
}
|
|
}
|
|
|
|
my $sendto = "$mail_list $defaultmail";
|
|
if($failcnt > 0){
|
|
&runcmd("cat $mailfile | /bin/mail -s \"$subject\" -a $attachfile \"$sendto\"");
|
|
}else{
|
|
&runcmd("cat $mailfile | /bin/mail -s \"$subject\" \"$sendto\"");
|
|
}
|
|
unlink($attachfile);
|
|
}
|
|
###############################################################
|
|
# Mainfunction
|
|
###############################################################
|
|
if (
|
|
!GetOptions("--help|h|?" => \$needhelp,
|
|
"--quiet" => \$quiet,
|
|
"--hold" => \$hold,
|
|
"os=s" => \$os,
|
|
"cluster=s" => \$cluster_name,
|
|
"testcase=s" => \$bucket,
|
|
"project-name=s" => \$proname,
|
|
"num=s" => \$proruntime,
|
|
"xcat-core=s" => \$xcatcore_addr,
|
|
"xcat-dep=s" => \$xcatdep_addr,
|
|
"email=s" => \$mail_list)
|
|
){
|
|
&usage;
|
|
send_msg(0, "step 0, PARSE ARGUMENTS returns error, exit");
|
|
exit 1;
|
|
}
|
|
|
|
if ($needhelp)
|
|
{
|
|
&usage;
|
|
exit 0;
|
|
}
|
|
|
|
unless(defined($os) && defined($proname) && defined($proruntime) && defined($bucket) && defined($cluster_name) && defined($xcatcore_addr) && defined($xcatdep_addr)){
|
|
&usage;
|
|
exit 1;
|
|
}
|
|
|
|
&init;
|
|
|
|
my $rst=0;
|
|
my $progname=\$0;;
|
|
$$progname="$pro_name ($proid): main";
|
|
|
|
send_msg(2,"........................");
|
|
send_msg(2,"........................");
|
|
send_msg(2,".....ooooO..............");
|
|
send_msg(2,"....(....)....Ooooo.....");
|
|
send_msg(2,".....\\..(.....(....)....");
|
|
send_msg(2,"......\\__).....)../.....");
|
|
send_msg(2,"..............(_ /......");
|
|
send_msg(2,"........................");
|
|
send_msg(2,".........START .........");
|
|
send_msg(2,"........................");
|
|
send_msg(2,"........................");
|
|
send_msg(2,"project $proid description:");
|
|
send_msg(2,"cluster => $cluster_name");
|
|
send_msg(2,"os => $os");
|
|
send_msg(2,"case => $bucket");
|
|
send_msg(2,"xcatcore => $xcatcore_addr");
|
|
send_msg(2,"xcatdep => $xcatdep_addr");
|
|
send_msg(2,"user => $mail_list");
|
|
if($hold){
|
|
send_msg(2,"hold => ON");
|
|
}else{
|
|
send_msg(2,"hold => OFF");
|
|
}
|
|
send_msg(2,"........................");
|
|
#######################################
|
|
|
|
$rst = env_check();
|
|
if($rst) {
|
|
send_msg(0, "environment check failed...........exit");
|
|
exit_test 1;
|
|
}
|
|
|
|
$rst = read_conf();
|
|
if($rst) {
|
|
send_msg(0, "read global configuration file failed...........exit");
|
|
exit_test 1;
|
|
}
|
|
|
|
=pod
|
|
foreach my $k (keys %confkeys) {
|
|
print "$k = $confkeys{$k}\n";
|
|
}
|
|
=cut
|
|
|
|
$rst = get_build();
|
|
if($rst) {
|
|
send_msg(0, "get xcat build failed...........exit");
|
|
exit_test 1;
|
|
}
|
|
|
|
pipe CONTROLREAD,MNWRITE;
|
|
my $pid = fork();
|
|
if ( !defined($pid) ) {
|
|
send_msg(0, "fork process for auto test error");
|
|
exit_test 1;
|
|
} elsif ( $pid == 0 ) { # child process
|
|
$$progname="$pro_name ($proid): test on $cluster_name";
|
|
|
|
$SIG{INT} = sub {
|
|
send_msg(2, "sub proc $$ recrive INT signal to exit");
|
|
exit 0;
|
|
};
|
|
|
|
send_msg(2, "..........fork process[pid=$$] for doing test..........");
|
|
close CONTROLREAD;
|
|
my $res;
|
|
|
|
#install mn
|
|
send_msg(2, "[$$]:Running mn_install...............");
|
|
$res = mn_install();
|
|
if ($res) {
|
|
syswrite MNWRITE,"[$$]:install $mn failed\n";
|
|
exit 1;
|
|
}
|
|
send_msg(2, "[$$]:Run mn_install...............[OK]");
|
|
|
|
#prepare for installing xcat on mn
|
|
send_msg(2, "[$$]:Running prepare_mn...............");
|
|
$res = prepare_mn();
|
|
if ($res) {
|
|
syswrite MNWRITE,"[$$]:prepare $mn failed\n";
|
|
exit 1;
|
|
}
|
|
send_msg(2, "[$$]:Run prepare_mn...............[OK]");
|
|
|
|
#install xcat-test on mn
|
|
send_msg(2, "[$$]:Running install_xcattest.............");
|
|
$res = install_xcattest();
|
|
if ($res) {
|
|
syswrite MNWRITE,"[$$]:install xcat-test on $mn failed\n";
|
|
exit 1;
|
|
}
|
|
send_msg(2, "[$$]:Run install_xcattest..............[OK]");
|
|
|
|
my $deployenvtime=time();
|
|
|
|
send_msg(2, "[$$] Running do_test...............");
|
|
$res = do_test();
|
|
if ($res) {
|
|
if($res == 2){
|
|
syswrite MNWRITE,"[$$][$deployenvtime]:do test stopped for some case\n";
|
|
}else{
|
|
syswrite MNWRITE,"[$$][$deployenvtime]:do test error\n";
|
|
}
|
|
exit 1;
|
|
}
|
|
send_msg(2, "[$$] Run do_test...............[OK]");
|
|
|
|
#below code has nothing to do with xCAT autotest main job
|
|
#just in order to collect logs and data to do xCAT product further analyse
|
|
#----------start----------------
|
|
&runcmd("scp -r $mn:/var/log/messages $mn:/var/log/xcat/computes.log $mn:/var/log/xcat/cluster.log $mn:/var/log/xcat/commands.log $logfiledir >/dev/null 2>&1");
|
|
#-----------end-----------------
|
|
|
|
send_msg(2, "[$$]:whole regression test on $cluster_name finished");
|
|
syswrite MNWRITE,"[$$][$deployenvtime]:whole regression test are successful\n";
|
|
close MNWRITE;
|
|
exit 0;
|
|
}
|
|
|
|
$SIG{TERM} = $SIG{INT} = sub {
|
|
if($pid) {
|
|
my $try=0;
|
|
kill 'INT', $pid;
|
|
while(waitpid($pid, WNOHANG)==0){
|
|
++$try;
|
|
#try INT up to 4 times if need
|
|
if($try < 5){
|
|
kill 'INT', $pid;
|
|
#try TERM up to 4 times if need
|
|
}elsif($try < 9){
|
|
kill 'TERM', $pid;
|
|
#force to kill finally
|
|
}elsif($try < 100){
|
|
kill 'KILL', $pid;
|
|
}else{
|
|
last;
|
|
}
|
|
}
|
|
}
|
|
|
|
&cleanup;
|
|
exit 0;
|
|
};
|
|
|
|
close MNWRITE;
|
|
my $regstarttime = time();
|
|
my $select = new IO::Select;
|
|
$select->add(\*CONTROLREAD);
|
|
while(! $sub_process_rt) {
|
|
my @hdls;
|
|
if (@hdls = $select->can_read(0)) {
|
|
my $hdl;
|
|
foreach $hdl (@hdls) {
|
|
if ($hdl == \*CONTROLREAD) {
|
|
my $line="";
|
|
chomp($line=<CONTROLREAD>);
|
|
if ($line){
|
|
my $tmp=$line;
|
|
if($line =~ /successful/){
|
|
send_msg(2, "[[main]]: $line");
|
|
}
|
|
if($line =~ /failed/){
|
|
$line =~ s/(.+):(.+)/$2/g;
|
|
send_msg(0, "[[main]]: $line");
|
|
$err_record = $line;
|
|
}
|
|
if($tmp =~ /\[(.+)\]\[(.+)\]:(.+)/){
|
|
$env_dply_time_consumption= $2 - $regstarttime;
|
|
$teststopflag=1 if($3 =~ /do test stopped for some case/);
|
|
}
|
|
$sub_process_rt=1;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if(time() - $regstarttime > 36000) {
|
|
send_msg(1, "[timing] 10 hours is expired");
|
|
last;
|
|
}
|
|
sleep 1;
|
|
}
|
|
|
|
close CONTROLREAD;
|
|
|
|
if($sub_process_rt) {
|
|
send_msg(2, "[[main]]: regression test return on time");
|
|
}else{
|
|
send_msg(0, "[[main]]: regression test return out of time");
|
|
if($pid) {
|
|
my $try=0;
|
|
kill 'INT', $pid;
|
|
while(waitpid($pid, WNOHANG)==0){
|
|
++$try;
|
|
#try INT up to 4 times if need
|
|
if($try < 5){
|
|
kill 'INT', $pid;
|
|
send_msg(2, "send INT to subprocess $pid...[$try]");
|
|
#try TERM up to 4 times if need
|
|
}elsif($try < 9){
|
|
kill 'TERM', $pid;
|
|
send_msg(2, "send TERM to subprocess $pid...[$try]");
|
|
#force to kill finally
|
|
}elsif($try < 100){
|
|
kill 'KILL', $pid;
|
|
send_msg(2, "send KILL to subprocess $pid...[$try]");
|
|
}else{
|
|
send_msg(2, "Can't stop pid $pid");
|
|
last;
|
|
}
|
|
sleep 1;
|
|
}
|
|
}
|
|
}
|
|
|
|
$all_reg_time_consumption=time()-$regstarttime;
|
|
|
|
send_msg(2, "[[main]]: creat test result report...........");
|
|
$rst = create_report();
|
|
if($rst){
|
|
send_msg(0, "[[main]]: creat test result report failed");
|
|
exit_test 1;
|
|
}else{
|
|
send_msg(2, "[[main]]: creat test result report....[done]");
|
|
send_msg(2, "[[main]]: send test result by mail.......");
|
|
send_mail();
|
|
send_msg(2, "[[main]]: send test result........[done]");
|
|
}
|
|
|
|
#used to hold the stop environment for developer to debug
|
|
if($teststopflag && $hold){
|
|
send_msg(2, "[[main]]: hold environment for 24 hours..........");
|
|
sleep(86400);
|
|
send_msg(2, "[[main]]: hold environment for 24 hours..........[done]");
|
|
}
|
|
|
|
if($alltestpass){
|
|
exit_test 0;
|
|
}else{
|
|
exit_test 1;
|
|
}
|