mirror of
https://github.com/xcat2/xcat-core.git
synced 2025-10-31 19:32:31 +00:00
1580 lines
63 KiB
Perl
Executable File
1580 lines
63 KiB
Perl
Executable File
#!/usr/bin/env perl
|
|
# IBM(c) 2010 EPL license http://www.eclipse.org/legal/epl-v10.html
|
|
#
|
|
#############################################################################################################################################
|
|
#
|
|
# This script will be used to run xCAT daily regression
|
|
#
|
|
# Usage:xCATreg - Run xCAT daily regression
|
|
# xCATreg [-?|-h]
|
|
# xCATreg [-b branch][-m MN][-f configure file][-a][-V]
|
|
#
|
|
# -?/-h: help information for the xCATreg usage.
|
|
# -b: when -b is specified, the regression will be done against the specified branch. You can use 2.7, 2.8,master.The default value is master.
|
|
# -m: when -m is specified, the regression will be done the specified mn. The default value is all the mn in configuration file.
|
|
# -f :specify the configuration file,default value is /xCATreg/regresson.conf
|
|
# -a: when -a is specified, run the regression even there is no code updates
|
|
# -V: log and message in verbose mode
|
|
#
|
|
# Example usage:
|
|
# ./xCATreg Run regressoins on master branch and on all platforms in configuration file
|
|
# ./xCATreg -b 2.8 -m slesmn -f /root/regressoin.conf -a Run regression on 2.8 branch on slesmn even when there is no code updates.
|
|
#
|
|
#############################################################################################################################################
|
|
|
|
$ENV{PATH} = "/opt/xcat/bin:/opt/xcat/sbin:/opt/xcat/share/xcat/tools:/usr/sbin:/usr/bin:/sbin:/bin";
|
|
|
|
use strict;
|
|
use warnings;
|
|
use Getopt::Long;
|
|
use Data::Dumper;
|
|
use Term::ANSIColor;
|
|
use Time::Local;
|
|
use IO::Select;
|
|
|
|
BEGIN
|
|
{
|
|
$::XCATROOT = $ENV{'XCATROOT'} ? $ENV{'XCATROOT'} : -d '/opt/xcat' ? '/opt/xcat' : '/usr';
|
|
}
|
|
use lib "$::XCATROOT/lib/perl";
|
|
|
|
my $date;
|
|
my $regrootdir="/home/autotest_code";
|
|
my $mailtitle="PPC64LE Cluster";
|
|
#my $regrootdir="/xCATreg";
|
|
#my $mailtitle="X Cluster";
|
|
my $logfiledir = "$regrootdir/log";
|
|
my $logfile="";
|
|
my $configfiledir = "$regrootdir/conf";
|
|
my $configfile = "$regrootdir/conf/xCATreg.conf";
|
|
my $xcatcoredowndir = "$regrootdir/xcatsource/xcat-core-build-down";
|
|
my $xcatcoredir = "$regrootdir/xcatsource/xcat-core";
|
|
my $xcatdepdir = "$regrootdir/xcatsource/xcat-dep";
|
|
my $osisodir = "$regrootdir/iso";
|
|
my $regresultdir = "$regrootdir/regresult";
|
|
my $todayregresultdir="";
|
|
my $maildir="$regrootdir/mail";
|
|
my $mailfile="";
|
|
my $rootdir = "$::XCATROOT/share/xcat/tools/autotest";
|
|
my %confkeys;
|
|
my $mail_list = 0;
|
|
my $needhelp = 0;
|
|
my $branch = "master";
|
|
my $management_node = undef;
|
|
#my $verbose_mode = 0;
|
|
my $updates_regression = 0;
|
|
my %mns;
|
|
my %pidrecord;
|
|
my %piderrrecord;
|
|
my %consumptionrecord;
|
|
my $commitinfofile="/tmp/commitinfo";
|
|
my $gitupdateflag = 0;
|
|
my $forceregwithoutupdate=0;
|
|
my $depupdateflag = 0;
|
|
|
|
#######################################
|
|
# runcmd
|
|
#######################################
|
|
sub runcmd {
|
|
my ($cmd) = @_;
|
|
my $rc = 0;
|
|
$::RUNCMD_RC = 0;
|
|
my $outref = [];
|
|
@$outref = `$cmd 2>&1`;
|
|
if ($?)
|
|
{
|
|
$rc = $? ;
|
|
$::RUNCMD_RC = $rc;
|
|
}
|
|
chomp(@$outref);
|
|
return @$outref;
|
|
}
|
|
|
|
#######################################
|
|
# clean_outdated_file
|
|
#######################################
|
|
sub is5daysago{
|
|
my $targetdate= shift;
|
|
my $targetdatey="";
|
|
my $targetdatem="";
|
|
my $targetdated="";
|
|
my $datey="";
|
|
my $datem="";
|
|
my $dated="";
|
|
if($targetdate =~ /(\d{4})(\d{2})(\d{2})/){
|
|
$targetdatey=$1;
|
|
$targetdatem=$2-1;
|
|
$targetdated=$3;
|
|
}
|
|
if($date =~ /(\d{4})(\d{2})(\d{2})/){
|
|
$datey=$1;
|
|
$datem=$2-1;
|
|
$dated=$3;
|
|
}
|
|
my $todays=timelocal(0,0,0,$dated,$datem,$datey);
|
|
my $targets=timelocal(0,0,0,$targetdated,$targetdatem,$targetdatey);
|
|
if($todays - $targets > 432000){
|
|
return 1;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
#######################################
|
|
# clean_outdated_file
|
|
#######################################
|
|
sub clean_outdated_file{
|
|
opendir(DIR, "$logfiledir");
|
|
foreach my $file (readdir DIR){
|
|
if($file =~ /xCATreg(\d+).log/){
|
|
my $res = is5daysago($1);
|
|
if($res){
|
|
&runcmd("rm -f $logfiledir/$file");
|
|
}
|
|
}
|
|
}
|
|
closedir(DIR);
|
|
|
|
opendir(DIR, "$maildir");
|
|
foreach my $file (readdir DIR){
|
|
if($file =~ /mailreport.(\d+)/){
|
|
my $res = is5daysago($1);
|
|
if($res){
|
|
&runcmd("rm -f $maildir/$file");
|
|
}
|
|
}
|
|
}
|
|
closedir(DIR);
|
|
|
|
opendir(DIR, "$regresultdir");
|
|
foreach my $file (readdir DIR){
|
|
if($file =~ /(\d+)/){
|
|
my $res = is5daysago($1);
|
|
if($res){
|
|
&runcmd("rm -rf $regresultdir/$file");
|
|
}
|
|
}
|
|
}
|
|
closedir(DIR);
|
|
|
|
return 0;
|
|
}
|
|
|
|
#######################################
|
|
# xCATreg_init
|
|
#######################################
|
|
sub xCATreg_init{
|
|
|
|
#init important attributes
|
|
|
|
$date = `date +"%Y%m%d%H%M%S"`;
|
|
chomp($date);
|
|
|
|
&runcmd("mkdir -p $logfiledir") if(! -e $logfiledir);
|
|
&runcmd("mkdir -p $maildir") if(! -e $maildir);
|
|
&runcmd("mkdir -p $regresultdir") if(! -e $regresultdir);
|
|
&runcmd("mkdir -p $xcatcoredir") if(! -e $xcatcoredir);
|
|
|
|
$logfile="xCATreg".$date.".log";
|
|
$todayregresultdir=$regresultdir."/".$date;
|
|
&runcmd("mkdir -p $todayregresultdir");
|
|
$mailfile=$maildir."/mailreport.".$date;
|
|
|
|
my $cnt=`ps aux|grep xCATreg |grep -v grep|awk '{print \$2}'|wc -l`;
|
|
my $out=`ps aux|grep xCATreg |grep -v grep|awk '{print \$2}'`;
|
|
for(my $i=1;$i<$cnt+1;$i++){
|
|
my $pid=`echo "$out"|sed -n ${i}p`;
|
|
chomp($pid);
|
|
next if($pid == $$);
|
|
&runcmd("kill $pid");
|
|
}
|
|
|
|
clean_outdated_file();
|
|
return 0;
|
|
}
|
|
|
|
#######################################
|
|
# usage for arguments
|
|
#######################################
|
|
sub usage{
|
|
print "Usage:xCATreg - Run xcat autotest test cases.\n
|
|
Explanation for the options:
|
|
-b: when -b is specified, the regression will be done against the specified branch. The default value is master.\n
|
|
-m: when -m is specified, run regression on the specified mn. The default value is all the mn in configuration file.\n
|
|
-f: specify the configuration file location.\n
|
|
-a: when -a is specified, run the regression even there is no code updates.\n
|
|
-e: <email_addr> send the test result to email_addr\n\n";
|
|
print " xCATreg [-?|-h]\n";
|
|
print " xCATreg [-f configure file] [-b branch] [-m mangement node] [-e mail list] [-a] Even code not updates, there will be regression.\n";
|
|
print " xCATreg [-f configure file] [-b branch] [-m mangement node] [-e mail list] if code updates there will be regression.\n";
|
|
print "\n";
|
|
return;
|
|
}
|
|
|
|
#######################################
|
|
# send messages
|
|
#######################################
|
|
sub send_msg {
|
|
my $num = shift;
|
|
my $msg = shift;
|
|
my $content;
|
|
if ($num == 0) {
|
|
$content = "Fatal error:";
|
|
} elsif($num == 1) {
|
|
$content = "Warning:";
|
|
} elsif($num == 2) {
|
|
$content = "Notice:";
|
|
}
|
|
my $timestamp = `date +"%Y%m%d%H%M%S"`;
|
|
chomp($timestamp);
|
|
if ( !open (LOGFILE, ">> $logfiledir/$logfile") ) {
|
|
return 1;
|
|
}
|
|
print LOGFILE "$content $timestamp $$: $msg.\n";
|
|
close LOGFILE;
|
|
}
|
|
|
|
|
|
#######################################
|
|
# read_conf
|
|
#######################################
|
|
sub read_conf{
|
|
my $myfile=undef;
|
|
my $line=undef;
|
|
|
|
if (!open($myfile, "$configfile")) {
|
|
send_msg(0, "Open $configfile failed");
|
|
return 1;
|
|
}
|
|
while ($line = <$myfile>) {
|
|
$line =~ s/\s//g;
|
|
next if($line =~ /^#/);
|
|
next if($line eq "");
|
|
my @attr=split(/=/,$line);
|
|
$confkeys{$attr[0]} = $attr[1];
|
|
}
|
|
close($myfile);
|
|
return 0;
|
|
}
|
|
|
|
#######################################
|
|
# mn_install to install all mn
|
|
#######################################
|
|
sub mn_install {
|
|
my $mn=shift;
|
|
|
|
&runcmd("lsdef|grep $confkeys{$mn}");
|
|
if($?){
|
|
send_msg(0, "[$mn->$confkeys{$mn}][mn_install] Didn't defined $confkeys{$mn} in current control node");
|
|
return 1;
|
|
}
|
|
|
|
my $os = $confkeys{$mn."os"};
|
|
my $arch = "ppc64";
|
|
if($mn !~ /aix/ && $mn =~ /x/){
|
|
$arch="x86_64";
|
|
}elsif($mn !~ /aix/ && $mn =~ /LE/){
|
|
$arch="ppc64le";
|
|
} elsif($mn !~ /aix/ && $mn =~ /EL/){
|
|
$arch="ppc64el";
|
|
}
|
|
my $osimage="";
|
|
|
|
if($mn !~ /aix/){
|
|
$osimage=$os."-".$arch."-install-compute";
|
|
}else{
|
|
$osimage="aix"; #????????need to test for aix
|
|
}
|
|
|
|
#print "[$mn] : os= $os arch=$arch osimage=$osimage\n";
|
|
|
|
&runcmd("lsdef -t osimage|grep $osimage");
|
|
if($?){
|
|
send_msg(0, "[$mn->$confkeys{$mn}][mn_install] Didn't defined $osimage in current control node");
|
|
return 1;
|
|
}
|
|
|
|
send_msg(2, "[$mn->$confkeys{$mn}][mn_install] installing $osimage for node $confkeys{$mn}................");
|
|
|
|
if( $arch =~ /x86_64/ || $arch =~ /ppc64le/ || $arch =~ /ppc64el/){
|
|
&runcmd("rmvm $confkeys{$mn} -f -p ");
|
|
&runcmd("chdef $confkeys{$mn} mac= ");
|
|
&runcmd("mkvm $confkeys{$mn} -s 40G");
|
|
}
|
|
|
|
&runcmd("makedhcp -n");
|
|
&runcmd("makedns -n");
|
|
&runcmd("chdef $confkeys{$mn} status=");
|
|
if($?){
|
|
send_msg(0, "[$mn->$confkeys{$mn}][mn_install] Failed to change MN status in current control node");
|
|
return 1;
|
|
}
|
|
|
|
|
|
&runcmd("nodeset $confkeys{$mn} osimage=$osimage");
|
|
if( $arch =~ /x86_64/ || $arch =~ /ppc64le/ || $arch =~ /ppc64el/){
|
|
&runcmd("rpower $confkeys{$mn} boot");
|
|
}else{
|
|
&runcmd("rnetboot $confkeys{$mn}");
|
|
}
|
|
|
|
&runcmd("sleep 600");
|
|
&runcmd("a=0;while ! `lsdef -l $confkeys{$mn}|grep status|grep booted >/dev/null`; do sleep 30;((a++));if [ \$a -gt 100 ];then break;fi done");
|
|
|
|
# if($mn !~ /ubuntu/){
|
|
# &runcmd("sleep 300");
|
|
# &runcmd("a=0;while ! `lsdef -l $confkeys{$mn}|grep status|grep booted >/dev/null`; do sleep 10;((a++));if [ \$a -gt 200 ];then break;fi done");}
|
|
# else{
|
|
# &runcmd("sleep 1800");
|
|
# }
|
|
|
|
&runcmd("lsdef -l $confkeys{$mn}|grep status|grep booted >/dev/null");
|
|
if($?){
|
|
send_msg(0, "[$mn->$confkeys{$mn}][mn_install] install $osimage for node $confkeys{$mn} failed");
|
|
return 1;
|
|
}
|
|
|
|
if($mn =~ /ubuntu/){
|
|
system("xdsh $confkeys{$mn} \"rm -rf /bin/sh\" >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} \"ln -s /bin/bash /bin/sh\"");
|
|
}
|
|
# if($mn =~ /ubuntu/){
|
|
# system("updatenode $confkeys{$mn} -P \"confignics -s\" >/dev/null 2>&1");
|
|
# system("xdsh $confkeys{$mn} \"iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE\" >/dev/null 2>&1");
|
|
# system("xdsh $confkeys{$mn} \"iptables -A FORWARD -i eth1 -j ACCEPT\" >/dev/null 2>&1");
|
|
# system("xdsh $confkeys{$mn} \"iptables -A FORWARD -i eth1 -o eth0 -m state --state RELATED,ESTABLISHED -j ACCEPT\" >/dev/null 2>&1");
|
|
# }
|
|
if($os =~ /rhels7/){
|
|
system("xdsh $confkeys{$mn} \"yum -y install bzip2 > /var/log/xcat/install_xcat.log 2>&1\" >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} \"yum -y install createrepo > /var/log/xcat/install_xcat.log 2>&1\" >/dev/null 2>&1");
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
#######################################
|
|
# down dep
|
|
#######################################
|
|
sub down_dep{
|
|
my $mn=shift;
|
|
send_msg(2, "[$mn->$confkeys{$mn}][down xcatdep] starting to down the latest xcat dep");
|
|
my $downxcatdepdir="$regrootdir/curl/$mn";
|
|
`mkdir -p $downxcatdepdir`;
|
|
my $curllink = $confkeys{$mn."curllink"};
|
|
my $deplink;
|
|
if (! -f "$downxcatdepdir/curlhtml1"){
|
|
send_msg(2, "it is first time to down xcatdep");
|
|
` curl "$curllink/" |grep "xcat-dep-" > "$downxcatdepdir/curlhtml1" `;
|
|
` tail -1 "$downxcatdepdir/curlhtml1" > "$downxcatdepdir/curlfile"`;
|
|
$deplink = `cat "$downxcatdepdir"/"curlfile" |awk '{print \$6}' |sed 's/"/ /g' |awk '{print \$2}' `;
|
|
`mkdir -p $xcatdepdir/$mn`;
|
|
` wget -P "$xcatdepdir/$mn" -c $curllink/$deplink `;
|
|
if($?){
|
|
send_msg(0, "download xcatdep failed");
|
|
return 1;
|
|
}
|
|
$depupdateflag=1;
|
|
}else{
|
|
`curl "$curllink/" |grep "xcat-dep-" > "$downxcatdepdir/curlhtml2" `;
|
|
` diff "$downxcatdepdir"/"curlhtml1" "$downxcatdepdir"/"curlhtml2"`;
|
|
if ($?==0) {
|
|
send_msg(2, "no update for xcatdep ");
|
|
$depupdateflag=0;
|
|
}else{
|
|
` tail -1 "$downxcatdepdir"/"curlhtml2"> "$downxcatdepdir"/"curlfile"`;
|
|
$deplink = `cat "$downxcatdepdir"/"curlfile" |awk '{print \$6}' |sed 's/"/ /g' |awk '{print \$2}' `;
|
|
` rm -rf $xcatdepdir/$mn`;
|
|
`mkdir -p $xcatdepdir/$mn`;
|
|
` wget -P "$xcatdepdir/$mn" -c $curllink/$deplink `;
|
|
if($?){
|
|
send_msg(2, "download xcatdep failed");
|
|
return 1;
|
|
}
|
|
`mv "$downxcatdepdir"/"curlhtml2" "$downxcatdepdir"/"curlhtml1" -f `;
|
|
$depupdateflag=2;
|
|
}
|
|
}
|
|
return ($depupdateflag);
|
|
}
|
|
|
|
#########################################
|
|
# down xcat-core build
|
|
###########################################
|
|
sub down_xcat_core_build {
|
|
my $mn=shift;
|
|
send_msg(2, "[$mn->$confkeys{$mn}][down xcatcore] starting to down the latest xcat core");
|
|
my $downxcatcoredowndir="$xcatcoredowndir/$mn";
|
|
if (! -f "$xcatcoredowndir/$mn"){
|
|
`mkdir -p $xcatcoredowndir/$mn`;
|
|
}
|
|
my $downxcatcoreip = $confkeys{$mn."downxcatcoreip"};
|
|
my $downxcatcoredir = $confkeys{$mn."downxcatcoredir"};
|
|
my $latestxcatcoredir = `ssh $downxcatcoreip ls -rt $downxcatcoredir / |tail -1 |awk '{printf \$0}'`;
|
|
my $xcatcorename = $confkeys{$mn."xcatcorename"};
|
|
`scp -r $downxcatcoreip\:$downxcatcoredir/$latestxcatcoredir/$xcatcorename $downxcatcoredowndir `;
|
|
if($? != 0){
|
|
send_msg(0,"[$mn->$confkeys{$mn}][down_xcat_core_build] failed");
|
|
return 1;
|
|
}
|
|
}
|
|
|
|
#######################################
|
|
# copy code
|
|
#######################################
|
|
sub copy_code {
|
|
my $mn=shift;
|
|
|
|
send_msg(2, "[$mn->$confkeys{$mn}][copy_code] starting to copy the latest xcat code and dep");
|
|
my $dnsserver=`cat /etc/hosts |grep $confkeys{$mn}|awk '{print \$1}'|head -n 1`;
|
|
chomp($dnsserver);
|
|
my $res = system("xdsh $confkeys{$mn} \"sed -i 's:nameserver .*:nameserver $dnsserver:g' /etc/resolv.conf\" >/dev/null 2>&1");
|
|
if($res != 0){
|
|
send_msg(0, "[$mn->$confkeys{$mn}][install_xcat] modify resolv.conf in $mn failed");
|
|
return 1;
|
|
}
|
|
|
|
#copy xcat-dep tarball to MN:/
|
|
my $depball="";
|
|
if($mn =~ /aix/){
|
|
$depball = `ls -l $xcatdepdir/$mn|awk '{print \$9}'|grep aix|tail -n 1`;
|
|
}elsif($mn =~ /ubuntu/){
|
|
$depball = `ls -l $xcatdepdir/$mn|awk '{print \$9}'|grep ubuntu|tail -n 1`;
|
|
}else{
|
|
$depball = `ls -l $xcatdepdir/$mn|awk '{print \$9}'|grep 'xcat-dep'|tail -n 1`;
|
|
}
|
|
chomp($depball);
|
|
print "depball is $depball\n";
|
|
my $coreball = `ls -l $xcatcoredowndir/$mn|awk '{print \$9}'|tail -n 1`;
|
|
chomp($coreball);
|
|
if ($depball eq ""){
|
|
send_msg(0, "[$mn->$confkeys{$mn}][copy_code] can't find dep tarball for $mn under dir $xcatdepdir");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[$mn->$confkeys{$mn}][copy_code] find dep tarball $depball for $mn");
|
|
if ($coreball eq ""){
|
|
send_msg(0, "[$mn->$confkeys{$mn}][copy_code] can't find core tarball for $mn under dir $coreball");
|
|
return 1;
|
|
}
|
|
|
|
send_msg(2, "[$mn->$confkeys{$mn}][copy_code] find core tarball $coreball for $mn");
|
|
|
|
$res = system("scp $xcatdepdir/$mn/$depball root\@$confkeys{$mn}:/ >/dev/null");
|
|
if ($res){
|
|
send_msg(0, "[$mn->$confkeys{$mn}][copy_code] copy $depball to $mn failed");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[$mn->$confkeys{$mn}][copy_code] copy $depball to $mn successfully");
|
|
|
|
$res = system("scp $xcatcoredowndir/$mn/$coreball root\@$confkeys{$mn}:/ >/dev/null");
|
|
if ($res){
|
|
send_msg(0, "[$mn->$confkeys{$mn}][copy_code] copy $coreball to $mn failed");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[$mn->$confkeys{$mn}][copy_code] copy $coreball to $mn successfully");
|
|
|
|
$res = system("xdsh $confkeys{$mn} 'cd / && tar xvf /$depball' >/dev/null 2>&1");
|
|
if ($?){
|
|
send_msg(0, "[$mn->$confkeys{$mn}][copy_code] untar $depball in $mn failed");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[$mn->$confkeys{$mn}][copy_code] untar $depball in $mn successfully");
|
|
$res = system("xdsh $confkeys{$mn} 'cd / && tar xvf /$coreball' >/dev/null 2>&1");
|
|
if ($?){
|
|
send_msg(0, "[$mn->$confkeys{$mn}][copy_code] untar $coreball in $mn failed");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[$mn->$confkeys{$mn}][copy_code] untar $coreball in $mn successfully");
|
|
|
|
#copy xcat-core source code to MN:/
|
|
if($mn =~ /ubuntux/){
|
|
#defect 107 Automation ENV replaces the link file to real file before running xcat build
|
|
$res = system("xdsh $confkeys{$mn} 'mkdir -p /xcatbuild' && rsync -avz $xcatcoredir root\@$confkeys{$mn}:/xcatbuild/ >/dev/null");
|
|
}else{
|
|
#defect 107 Automation ENV replaces the link file to real file before running xcat build
|
|
$res = system("rsync -avz $xcatcoredir root\@$confkeys{$mn}:/ >/dev/null");
|
|
}
|
|
if ($res){
|
|
send_msg(0, "[$mn->$confkeys{$mn}][copy_code] copy the latest xcat code to $mn failed");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[$mn->$confkeys{$mn}][copy_code] copy the latest xcat code to $mn successfully");
|
|
|
|
#copy /etc/hosts file to MN
|
|
$res = system("scp /etc/hosts root\@$confkeys{$mn}:/etc >/dev/null");
|
|
if ($?){
|
|
send_msg(0, "[$mn->$confkeys{$mn}][copy_code] copy /etc/hosts to $mn failed");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[$mn->$confkeys{$mn}][copy_code] copy /etc/hosts to $mn successfully");
|
|
|
|
my $os = $confkeys{$mn."os"};
|
|
my $regconf=$mn.".conf";
|
|
system("sed -i 's|os=\\S*|os=$os|g' $configfiledir/$regconf");
|
|
|
|
my $arch = "ppc64";
|
|
if($mn !~ /aix/ && $mn =~ /x/){
|
|
$arch="x86_64";
|
|
}elsif($mn !~ /aix/ && $mn =~ /LE/){
|
|
$arch="ppc64le";
|
|
}elsif($mn !~ /aix/ && $mn =~ /EL/){
|
|
$arch="ppc64el";
|
|
}
|
|
my $iso;
|
|
|
|
#copy specific files to MN for each platform
|
|
if($mn =~ /rh/){
|
|
$res = system("scp /install/$os/$arch/RPM-GPG-KEY-redhat-release root\@$confkeys{$mn}:/ >/dev/null");
|
|
if ($?){
|
|
send_msg(0, "[$mn->$confkeys{$mn}][copy_code] copy RPM-GPG-KEY-redhat-release to $mn failed");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[$mn->$confkeys{$mn}][copy_code] copy RPM-GPG-KEY-redhat-release to $mn successfully");
|
|
|
|
$os =~ /(\D+)(\d+\.\d+)/;
|
|
$iso=`ls -l $osisodir|awk '{print \$9}'|grep RHEL`;
|
|
chomp($iso);
|
|
#print "[$mn->$confkeys{$mn}][copy_code] iso=$iso\n";
|
|
if ($iso eq ""){
|
|
send_msg(0, "[$mn->$confkeys{$mn}][copy_code] can't find iso for $os and $arch under dir $osisodir");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[$mn->$confkeys{$mn}][copy_code] find $iso for os=$os and arch=$arch");
|
|
|
|
$res = system("scp $osisodir/$iso root\@$confkeys{$mn}:/ >/dev/null");
|
|
if ($?){
|
|
send_msg(0, "[$mn->$confkeys{$mn}][copy_code] copy ISO $iso to $mn failed");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[$mn->$confkeys{$mn}][copy_code] copy ISO $iso to $mn successfully");
|
|
system ("sed -i 's|^ISO=/\\S*|ISO=/$iso|g' $configfiledir/$regconf");
|
|
}
|
|
|
|
if($mn =~ /sles/){
|
|
$os =~ /(\D+)(\d+)\.?(\d?)/;
|
|
if($3){
|
|
$iso=`ls -l $osisodir| awk '{print \$9}'|grep SLE|grep $arch |grep -- "-$2-SP$3-" |tail -n 1`;
|
|
}else{
|
|
$iso=`ls -l $osisodir| awk '{print \$9}'|grep SLE|grep $arch |grep -v "SP"|grep -- "-$2-"|tail -n 1`;
|
|
}
|
|
chomp($iso);
|
|
#print "[$mn->$confkeys{$mn}][copy_code] iso=$iso\n";
|
|
if ($iso eq ""){
|
|
send_msg(0, "[$mn->$confkeys{$mn}][copy_code] can't find ISO for os=$os and arch=$arch under dir $osisodir");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[$mn->$confkeys{$mn}][copy_code] find $iso for os=$os and arch=$arch");
|
|
|
|
$res = system("scp $osisodir/$iso root\@$confkeys{$mn}:/ >/dev/null");
|
|
if ($?){
|
|
send_msg(0, "[$mn->$confkeys{$mn}][copy_code] copy ISO $iso to $mn failed");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[$mn->$confkeys{$mn}][copy_code] copy ISO $iso to $mn successfully");
|
|
system ("sed -i 's|^ISO=/\\S*|ISO=/$iso|g' $configfiledir/$regconf");
|
|
|
|
if($os =~ /sles11/){
|
|
$res = system("xdsh $confkeys{$mn} \"mv /usr/lib/rpm/find-requires /usr/lib/rpm/find-requires.org\" && scp $osisodir/find-requires $confkeys{$mn}:/usr/lib/rpm/ > /dev/null");
|
|
if ($?){
|
|
send_msg(0, "[$mn->$confkeys{$mn}][copy_code] For sles11, copy find-requires to $mn failed");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[$mn->$confkeys{$mn}][copy_code] For sles11, copy find-requires to $mn successfully");
|
|
}
|
|
}
|
|
if($mn =~ /ubuntu/){
|
|
$os =~ /(\D+)(\d+\.\d+\.?\d?)/;
|
|
my $osversion = $2;
|
|
if ($mn =~ /ubuntux/){
|
|
$iso=`ls -l $osisodir| awk '{print \$9}'|grep ubuntu|grep amd64 |grep $osversion |tail -n 1`;
|
|
}elsif($mn =~ /ubuntuEL/){
|
|
$iso=`ls -l $osisodir| awk '{print \$9}'|grep ubuntu|grep ppc64el |grep $osversion |tail -n 1`;
|
|
}
|
|
chomp($iso);
|
|
#print "[$mn->$confkeys{$mn}][copy_code] iso=$iso\n";
|
|
if ($iso eq ""){
|
|
send_msg(0, "[$mn->$confkeys{$mn}][copy_code] can't find ISO for os=$os and arch=$arch under dir $osisodir");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[$mn->$confkeys{$mn}][copy_code] find $iso for os=$os and arch=$arch");
|
|
|
|
$res = system("scp $osisodir/$iso root\@$confkeys{$mn}:/ >/dev/null");
|
|
if ($?){
|
|
send_msg(0, "[$mn->$confkeys{$mn}][copy_code] copy ISO $iso to $mn failed");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[$mn->$confkeys{$mn}][copy_code] copy ISO $iso to $mn successfully");
|
|
system ("sed -i 's|^ISO=/\\S*|ISO=/$iso|g' $configfiledir/$regconf");
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
#######################################
|
|
# config mn
|
|
#######################################
|
|
sub install_xcat {
|
|
my $mn = shift;
|
|
send_msg(2, "[$mn->$confkeys{$mn}][install_xcat] starting to install xcat in $confkeys{$mn}");
|
|
|
|
my $os = $confkeys{$mn."os"};
|
|
my $arch = "ppc64";
|
|
if($mn !~ /aix/ && $mn =~ /x/){
|
|
$arch="x86_64";
|
|
}elsif($mn !~ /aix/ && $mn =~ /LE/){
|
|
$arch="ppc64le";
|
|
}elsif($mn !~ /aix/ && $mn =~ /EL/){
|
|
$arch="ppc64el";
|
|
}
|
|
if($mn =~ /ubuntu/){
|
|
system("xdsh $confkeys{$mn} \"echo \"nameserver 9.0.2.1\" >> /etc/resolv.conf\" >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} 'apt-get -y install software-properties-common' >/dev/null 2>&1");
|
|
if($mn =~ /ubuntux/){
|
|
system("xdsh $confkeys{$mn} 'add-apt-repository \"deb http://archive.ubuntu.com/ubuntu \$(lsb_release -sc) main\"' >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} 'add-apt-repository \"deb http://archive.ubuntu.com/ubuntu \$(lsb_release -sc)-updates main\"' >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} 'add-apt-repository \"deb http://archive.ubuntu.com/ubuntu \$(lsb_release -sc) universe\"' >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} 'add-apt-repository \"deb http://archive.ubuntu.com/ubuntu \$(lsb_release -sc)-updates universe\"' >/dev/null 2>&1");
|
|
}elsif($mn =~ /ubuntuEL/){
|
|
system("xdsh $confkeys{$mn} 'add-apt-repository \"deb http://ports.ubuntu.com/ubuntu-ports \$(lsb_release -sc) main\"' >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} 'add-apt-repository \"deb http://ports.ubuntu.com/ubuntu-ports \$(lsb_release -sc)-updates main\"' >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} 'add-apt-repository \"deb http://ports.ubuntu.com/ubuntu-ports \$(lsb_release -sc) universe\"' >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} 'add-apt-repository \"deb http://ports.ubuntu.com/ubuntu-ports \$(lsb_release -sc)-updates universe\"' >/dev/null 2>&1");
|
|
}
|
|
|
|
system("xdsh $confkeys{$mn} \"wget -O - \"http://sourceforge.net/projects/xcat/files/ubuntu/apt.key/download\" | apt-key add -\" >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} 'apt-get clean all' >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} 'apt-get update' >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} 'apt-get -y install build-essential dpkg-dev dh-make debhelper fakeroot gnupg lintian pbuilder quilt reprepro libsoap-lite-perl libdbi-perl' >/dev/null 2>&1");
|
|
}
|
|
|
|
if($mn =~ /rh/){
|
|
$os =~ /(\D+)(\d+)\.?(\d?)/;
|
|
system("xdsh $confkeys{$mn} \"cd /xcat-core && ./mklocalrepo.sh\" >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} \"cd /xcat-dep/rh$2/$arch && ./mklocalrepo.sh\" >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} \"rpm --import /RPM-GPG-KEY-redhat-release\" >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} \"yum -y install xCAT xCAT-test > /var/log/xcat/install_xcat.log 2>&1\" >/dev/null 2>&1");
|
|
if($os =~ /rhels7/){
|
|
system("xdsh $confkeys{$mn} \"yum -y install mariadb-devel mariadb-libs mariadb-server mariadb-bench mariadb perl-DBD-MySQL mysql-connector-odbc unixODBC\" >/dev/null 2>&1");
|
|
}elsif($os =~ /rhels6/){
|
|
system("xdsh $confkeys{$mn} \"yum -y install mysql-server mysql mysql-bench mysql-devel mysql-connector-odbc\" >/dev/null 2>&1");
|
|
}
|
|
system("xdsh $confkeys{$mn} \"source /etc/profile.d/xcat.sh\" >/dev/null 2>&1");
|
|
if($mn =~ /rhx/){
|
|
system("xdsh $confkeys{$mn} \"yum install -y perl-Sys-Virt\" >/dev/null 2>&1");
|
|
}
|
|
}elsif($mn =~ /sles/){
|
|
$os =~ /(\D+)(\d+)\.?(\d?)/;
|
|
system("xdsh $confkeys{$mn} \"zypper ar file:///xcat-dep/sles$2/$arch xCAT-dep\" >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} \"zypper ar file:///xcat-core xCAT-core\" >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} \"zypper sl -U\" >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} \"zypper --gpg-auto-import-keys search --match-exact -s screen\" >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} \"zypper -n install xCAT xCAT-test > /var/log/xcat/install_xcat.log 2>&1\" >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} \"zypper -n install createrepo\" >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} \"zypper -n install mysql-client libmysqlclient_r15 libqt4-sql-mysql libmysqlclient15 perl-DBD-mysql mysql unixODBC\" >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} \"source /etc/profile.d/xcat.sh\" >/dev/null 2>&1");
|
|
if($mn =~ /slesLE/){
|
|
system ("xdsh $confkeys{$mn} \"zypper -n install perl-Net-DNS-0.80-1.ppc64le\">/dev/null 2>&1");
|
|
}
|
|
if($mn =~ /slesx/){
|
|
system("xdsh $confkeys{$mn} \"zypper -n install perl-Sys-Virt\" >/dev/null 2>&1");
|
|
}
|
|
}elsif($mn =~ /ubuntux/){
|
|
system("xdsh $confkeys{$mn} \"/xcat-core/mklocalrepo.sh\" >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} \"/xcat-dep/mklocalrepo.sh\" >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} 'apt-get clean all' >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} 'apt-get update' >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} 'apt-get -y install xcat xCAT-test > /var/log/xcat/install_xcat.log 2>&1' >/dev/null 2>&1");
|
|
}elsif($mn =~ /ubuntuEL/){
|
|
system("xdsh $confkeys{$mn} \"/xcat-core/mklocalrepo.sh\" >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} \"/xcat-dep/mklocalrepo.sh\" >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} 'apt-get clean all' >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} 'apt-get update' >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} 'apt-get -y install xcat xCAT-test > /var/log/xcat/install_xcat.log 2>&1' >/dev/null 2>&1");
|
|
system("scp $osisodir/mini.iso $confkeys{$mn}:/ ");
|
|
}
|
|
|
|
#check if MN is installed successful
|
|
my $lsdefout = `xdsh $confkeys{$mn} '/opt/xcat/bin/lsdef -v'`;
|
|
chomp($lsdefout);
|
|
if($lsdefout !~ /lsdef - Version/ ){
|
|
send_msg(0, "[$mn->$confkeys{$mn}][install_xcat] install xcat in $mn failed");
|
|
return 1;
|
|
}
|
|
|
|
send_msg(2, "[$mn->$confkeys{$mn}][install_xcat] install xcat in $mn successfully");
|
|
return 0;
|
|
}
|
|
|
|
#######################################
|
|
# do test
|
|
#######################################
|
|
sub do_test {
|
|
my $mn = shift;
|
|
my $os = $confkeys{$mn."os"};
|
|
my $casestop=1;
|
|
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] starting to run regression test in $confkeys{$mn}");
|
|
|
|
my $regconf=$mn.".conf";
|
|
my $res = system("scp $configfiledir/$regconf root\@$confkeys{$mn}:/opt/xcat/share/xcat/tools/autotest/default.conf >/dev/null");
|
|
if ($res != 0){
|
|
send_msg(0, "[$mn->$confkeys{$mn}][do_test] copy $configfiledir/$regconf to $mn failed");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] copy $configfiledir/$regconf to $mn successfully");
|
|
if($os =~ /rhels7/){
|
|
system("scp /opt/xcat/share/xcat/install/rh/service.rhels7.ppc64.otherpkgs.pkglist root\@$confkeys{$mn}:/opt/xcat/share/xcat/install/rh >/dev/null");
|
|
}
|
|
my $diskfull_installation_flat_testcase="";
|
|
my $diskless_installation_flat_testcase="";
|
|
my $statelite_installation_flat_testcase="";
|
|
my $sn_installation_hierarchy_testcase="";
|
|
my $diskfull_installation_hierarchy_testcase="";
|
|
my $diskless_installation_hierarchy_testcase="";
|
|
my $statelite_installation_hierarchy_by_ramdisk_testcase="";
|
|
my $statelite_installation_hierarchy_by_nfs_testcase="";
|
|
my $setup_dockerhost_testcase="";
|
|
my $batbundle="";
|
|
if((($mn =~ /sles/) || ($mn =~ /rh/)) && ($mn !~ /E/)){
|
|
$diskfull_installation_flat_testcase="reg_linux_diskfull_installation_flat";
|
|
$diskless_installation_flat_testcase="reg_linux_diskless_installation_flat";
|
|
$statelite_installation_flat_testcase="reg_linux_statelite_installation_flat";
|
|
$sn_installation_hierarchy_testcase="reg_linux_SN_installation_hierarchy";
|
|
$diskfull_installation_hierarchy_testcase="reg_linux_diskfull_installation_hierarchy";
|
|
$diskless_installation_hierarchy_testcase="reg_linux_diskless_installation_hierarchy";
|
|
$statelite_installation_hierarchy_by_ramdisk_testcase="reg_linux_statelite_installation_hierarchy_by_ramdisk";
|
|
$statelite_installation_hierarchy_by_nfs_testcase="reg_linux_statelite_installation_hierarchy_by_nfs";
|
|
$batbundle="p_rhels_cmd.bundle" if($mn =~ /rhp/);
|
|
$batbundle="x_rhels_cmd.bundle" if($mn =~ /rhx/);
|
|
$batbundle="p_sles_cmd.bundle" if($mn =~ /slesp/);
|
|
$batbundle="x_sles_cmd.bundle" if($mn =~ /slesx/);
|
|
}elsif($mn =~ /ubuntux/){
|
|
$diskfull_installation_flat_testcase="Ubuntu_full_installation_flat_x86_vm";
|
|
$diskless_installation_flat_testcase="Ubuntu_diskless_installation_flat_x86_vm";
|
|
$setup_dockerhost_testcase="Full_installation_flat_docker";
|
|
$batbundle="x_ubuntu_cmd.bundle";
|
|
}elsif($mn =~ /E/){
|
|
$diskfull_installation_flat_testcase="Full_installation_flat_p8_le";
|
|
$diskless_installation_flat_testcase="Diskless_installation_flat_p8_le";
|
|
$batbundle="le_rhels_cmd.bundle" if($mn =~ /rhLE/);
|
|
$batbundle="le_sles_cmd.bundle" if($mn =~ /slesLE/);
|
|
$batbundle="le_ubuntu_cmd.bundle" if($mn =~ /ubuntuEL/);
|
|
}
|
|
if(($mn !~ /ubuntu/) && ($mn !~ /E/)){
|
|
{
|
|
my $rc;
|
|
|
|
#diskfull installation in flat mode test
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] doing diskfull installation test in flat mode in $mn.....");
|
|
$rc=system("xdsh $confkeys{$mn} \"xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf -t $diskfull_installation_flat_testcase\" >/dev/null 2>&1");
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] do diskfull installation test in flat mode in $mn.....[done]");
|
|
last if($rc != 0);
|
|
|
|
#commands bundle in flat mode test
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] doing all commands test in flat mode in $mn.....");
|
|
$rc=system("xdsh $confkeys{$mn} \"xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf -b $batbundle\" >/dev/null 2>&1");
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] do all commands test in flat mode in $mn.....[done]");
|
|
last if($rc != 0);
|
|
|
|
#diskless installation in flat mode test
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] doing diskless installation test in flat mode in $mn.....");
|
|
$rc=system("xdsh $confkeys{$mn} \"xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf -t $diskless_installation_flat_testcase\" >/dev/null 2>&1");
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] do diskless installation test in flat mode in $mn.....[done]");
|
|
last if($rc != 0);
|
|
|
|
#statelite installation in flat mode test
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] doing statelite installation test in flat mode in $mn.....");
|
|
$rc=system("xdsh $confkeys{$mn} \"xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf -t $statelite_installation_flat_testcase\" >/dev/null 2>&1");
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] do statelite installation test in flat mode in $mn.....[done]");
|
|
last if($rc != 0);
|
|
|
|
#deploy SN
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] setting up mysql for installing SN in $mn.....");
|
|
$res = system("ssh -t $confkeys{$mn} 'exec bash -l -i -c \"XCATMYSQLADMIN_PW=12345 XCATMYSQLROOT_PW=12345 /opt/xcat/bin/mysqlsetup -i\"' >/dev/null 2>&1");
|
|
if ($res != 0){
|
|
send_msg(0, "[$mn->$confkeys{$mn}][do_test] set up mysql for installing SN in $mn failed");
|
|
return 1;
|
|
}
|
|
$res = system ("xdsh $confkeys{$mn} \"echo \\\"GRANT ALL on xcatdb.* TO xcatadmin@\'%\' IDENTIFIED BY \'12345\'\;\\\" | mysql -u root -p12345\" >/dev/null 2>&1");
|
|
if ($res != 0){
|
|
send_msg(0, "[$mn->$confkeys{$mn}][do_test] grant right to mysql for installing SN in $mn failed");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] set up mysql for installing SN in $mn.....[done]");
|
|
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] preparing xcat ball for installing SN in $mn.....");
|
|
my $os = $confkeys{$mn."os"};
|
|
my $arch = "ppc64";
|
|
if(($mn !~ /aix/) && ($mn =~ /x/)){
|
|
$arch="x86_64";
|
|
}elsif($mn !~ /aix/ && $mn =~ /LE/){
|
|
$arch="ppc64le";
|
|
}elsif($mn !~ /aix/ && $mn =~ /EL/){
|
|
$arch="ppc64el";
|
|
}
|
|
my $xcatballpath = "/install/post/otherpkgs/$os/$arch/xcat";
|
|
$res = system("xdsh $confkeys{$mn} \"mkdir -p $xcatballpath/xcat-core && cp -r /xcat-core/* $xcatballpath/xcat-core && cp -r /xcat-dep $xcatballpath\"");
|
|
if ($res != 0){
|
|
send_msg(0, "[$mn->$confkeys{$mn}][do_test] prepare xcat ball for installing SN in $mn failed");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] prepare xcat ball for installing SN in $mn.....[done]");
|
|
|
|
if($mn =~ /rhp/){
|
|
my $ping_sn_on_rhp_pid = fork();
|
|
if( !defined($ping_sn_on_rhp_pid)){
|
|
send_msg(0, "fork process for trigger random number generator on redhat+ppc environment to install SN error");
|
|
exit;
|
|
}elsif($ping_sn_on_rhp_pid ==0){
|
|
send_msg(2, "....fork 1 process[pid=$$] for trigger random number generator on redhat+ppc environment to install SN..........");
|
|
my $snnodename = `cat $configfiledir/$regconf |grep SN|grep =|awk -F'=' '{print \$2}'`;
|
|
chomp($snnodename);
|
|
&runcmd("sleep 900");
|
|
my $timecnt=0;
|
|
while($timecnt < 180){
|
|
my $booted = system("xdsh $confkeys{$mn} \"lsdef -l $snnodename|grep status|grep booted\"");
|
|
last if($booted==0);
|
|
|
|
my $booting = system("xdsh $confkeys{$mn} \"lsdef -l $snnodename|grep status|grep booting\"");
|
|
system("xdsh $confkeys{$mn} \"xdsh $snnodename \"echo 1\"\"") if($booting==0);
|
|
|
|
&runcmd("sleep 10");
|
|
$timecnt++;
|
|
}
|
|
send_msg(2, "....process[pid=$$] for trigger random number generator on redhat+ppc environment to install SN....[exit]");
|
|
exit 0;
|
|
}
|
|
}
|
|
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] installing SN.....");
|
|
$rc=system("xdsh $confkeys{$mn} \"xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf -t $sn_installation_hierarchy_testcase\" >/dev/null 2>&1");
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] install SN.....[done]");
|
|
last if($rc != 0);
|
|
|
|
#diskfull installation in hierarchy mode test
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] doing diskfull installation test in hierarchy mode in $mn.....");
|
|
$rc=system("xdsh $confkeys{$mn} \"xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf -t $diskfull_installation_hierarchy_testcase\" >/dev/null 2>&1");
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] do diskfull installation test in hierarchy mode in $mn.....[done]");
|
|
last if($rc != 0);
|
|
|
|
#diskless installation in hierarchy mode test
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] doing diskless installation test in hierarchy mode in $mn.....");
|
|
$rc=system("xdsh $confkeys{$mn} \"xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf -t $diskless_installation_hierarchy_testcase\" >/dev/null 2>&1");
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] do diskless installation test in hierarchy mode in $mn.....[done]");
|
|
last if($rc != 0);
|
|
|
|
#statelite installation by ramdisk in hierarchy mode test
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] doing statelite installation by ramdisk test in hierarchy mode in $mn.....");
|
|
$rc=system("xdsh $confkeys{$mn} \"xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf -t $statelite_installation_hierarchy_by_ramdisk_testcase\" >/dev/null 2>&1");
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] do statelite installation by ramdisk test in hierarchy mode in $mn.....[done]");
|
|
last if($rc != 0);
|
|
|
|
#statelite installation by nfs in hierarchy mode test
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] doing statelite installation by nfs test in hierarchy mode in $mn.....");
|
|
$rc=system("xdsh $confkeys{$mn} \"xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf -t $statelite_installation_hierarchy_by_nfs_testcase\" >/dev/null 2>&1");
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] do statelite installation by nfs test in hierarchy mode in $mn.....[done]");
|
|
last if($rc != 0);
|
|
|
|
$casestop=0;
|
|
}
|
|
}else{
|
|
{
|
|
my $rc=0;
|
|
|
|
#diskless installation in flat mode test
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] doing diskless installation test in flat mode in $mn.....");
|
|
$rc=system("ssh -t $confkeys{$mn} 'exec bash -l -i -c \"xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf -t $diskless_installation_flat_testcase\"' >/dev/null 2>&1");
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] do diskless installation test in flat mode in $mn.....[done]");
|
|
last if($rc != 0);
|
|
|
|
#diskfull installation in flat mode test
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] doing diskfull installation test in flat mode in $mn.....");
|
|
$rc=system("ssh -t $confkeys{$mn} 'exec bash -l -i -c \"xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf -t $diskfull_installation_flat_testcase\"' >/dev/null 2>&1");
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] do diskfull installation test in flat mode in $mn.....[done]");
|
|
last if($rc != 0);
|
|
|
|
if($mn =~ /ubuntux/){
|
|
#set up dockerhosts
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] setupdockerhost $mn.....");
|
|
$rc=system("ssh -t $confkeys{$mn} 'exec bash -l -i -c \"xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf -t $setup_dockerhost_testcase\"' >/dev/null 2>&1");
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] setup dockerhosts $mn.....[done]");
|
|
last if($rc != 0);
|
|
}
|
|
|
|
#commands bundle in flat mode test
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] doing all commands test in flat mode in $mn.....");
|
|
$rc=system("ssh -t $confkeys{$mn} 'exec bash -l -i -c \"xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf -b $batbundle\"' >/dev/null 2>&1");
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] do all commands test in flat mode in $mn.....[done]");
|
|
$casestop=0;
|
|
}
|
|
}
|
|
|
|
if($casestop){
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] the regression job in $mn was broken by one case.");
|
|
}else{
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] run the whole regression test in $mn finished");
|
|
}
|
|
|
|
send_msg(2, "[$mn->$confkeys{$mn}][install_xcat] run regression test in $mn finished");
|
|
|
|
$res = system("scp -r $confkeys{$mn}:/opt/xcat/share/xcat/tools/autotest/result/* $todayregresultdir/$mn/ >/dev/null 2>&1");
|
|
if ($res != 0){
|
|
send_msg(0, "[$mn->$confkeys{$mn}][do_test] copy regression result to $todayregresultdir/$mn/ failed");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] copy regression result to $todayregresultdir/$mn/ successfully");
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
#######################################
|
|
# creat report
|
|
#######################################
|
|
sub creat_report{
|
|
|
|
my $mailreport .= "======================================\n";
|
|
$mailreport .= " Regression Result Summary\n";
|
|
$mailreport .= "======================================\n\n";
|
|
# $gitupdateflag = 0 git update out of time
|
|
# $gitupdateflag = 1 there isn't new code checkin in last one day
|
|
# $gitupdateflag = 2 git update failed
|
|
if($gitupdateflag==0){
|
|
$mailreport .= "Get the latest xcat code from git out of time, stop regression test today!!!\n\n\n";
|
|
&runcmd("touch $mailfile && echo \"$mailreport\" > $mailfile");
|
|
&runcmd("mv $commitinfofile $commitinfofile.old && echo \"\" > $commitinfofile");
|
|
return 0;
|
|
}elsif($gitupdateflag==1 && ! $forceregwithoutupdate){
|
|
$mailreport .= "There isn't new code checkin last one day, stop regression test today!!!\n\n\n";
|
|
&runcmd("touch $mailfile && echo \"$mailreport\" > $mailfile");
|
|
&runcmd("mv $commitinfofile $commitinfofile.old && echo \"\" > $commitinfofile");
|
|
return 0;
|
|
}elsif($gitupdateflag==1 && $forceregwithoutupdate){
|
|
$mailreport .= "There isn't new code checkin last one day, But force to run regresson today!!!\n\n\n";
|
|
&runcmd("mv $commitinfofile $commitinfofile.old && echo \"\" > $commitinfofile");
|
|
}elsif($gitupdateflag==2){
|
|
$mailreport .= "Get the latest xcat source code from git error, stop regression test today!!!\n\n\n";
|
|
&runcmd("touch $mailfile && echo \"$mailreport\" > $mailfile");
|
|
&runcmd("mv $commitinfofile $commitinfofile.old && echo \"\" > $commitinfofile");
|
|
return 0;
|
|
}
|
|
|
|
while(my($k,$v) = each %pidrecord){
|
|
my $scenario="[".$confkeys{$k."os"};
|
|
if($k=~/p/){
|
|
$scenario.="+PPC64] ";
|
|
}elsif($k=~ /E/){
|
|
$scenario.="+PPC64LE] ";
|
|
}elsif($k=~/x/){
|
|
$scenario.="+x86_64] ";
|
|
}
|
|
$mailreport .= "$scenario\n";
|
|
my $batbundle="";
|
|
my $totalcase=0;
|
|
|
|
if($k=~ /slesp/ || $k =~ /rhx/ || $k=~ /slesx/ || $k =~ /rhp/){
|
|
$batbundle="$xcatcoredir/xCAT-test/autotest/bundle/p_rhels_cmd.bundle" if($k =~ /rhp/);
|
|
$batbundle="$xcatcoredir/xCAT-test/autotest/bundle/x_rhels_cmd.bundle" if($k =~ /rhx/);
|
|
$batbundle="$xcatcoredir/xCAT-test/autotest/bundle/p_sles_cmd.bundle" if($k =~ /slesp/);
|
|
$batbundle="$xcatcoredir/xCAT-test/autotest/bundle/x_sles_cmd.bundle" if($k =~ /slesx/);
|
|
$totalcase=`cat $batbundle|wc -l`;
|
|
$totalcase+=8;
|
|
}elsif($k =~ /ubuntux/){
|
|
$batbundle="/$xcatcoredir/xCAT-test/autotest/bundle/x_ubuntu_cmd.bundle";
|
|
$totalcase=`cat $batbundle|wc -l`;
|
|
$totalcase+=3;
|
|
}elsif($k =~ /E/){
|
|
$batbundle="$xcatcoredir/xCAT-test/autotest/bundle/le_rhels_cmd.bundle" if($k =~ /rhLE/);
|
|
$batbundle="$xcatcoredir/xCAT-test/autotest/bundle/le_sles_cmd.bundle" if($k =~ /slesLE/);
|
|
$batbundle="$xcatcoredir/xCAT-test/autotest/bundle/le_ubuntu_cmd.bundle" if($k =~ /ubuntuEL/);
|
|
$totalcase=`cat $batbundle|wc -l`;
|
|
$totalcase+=2;
|
|
}
|
|
|
|
|
|
if ($v == 0){ # whole regression test finished
|
|
my $h=int($consumptionrecord{$k}/3600);
|
|
my $m=int(($consumptionrecord{$k} - $h*3600)/60);
|
|
my $s=($consumptionrecord{$k} - $h*3600)%60;
|
|
my $consumption="$h hours $m minutes $s seconds";
|
|
$mailreport .= "\tTime consumption: $consumption\n";
|
|
|
|
if($piderrrecord{$k} ne ""){
|
|
$mailreport .= "\tDeploy MN result: $piderrrecord{$k}\n";
|
|
$mailreport .= "\tTotalCase $totalcase TotalRun 0 Failed 0\n";
|
|
next;
|
|
}
|
|
|
|
my $totalcnt=0;
|
|
my $failcnt=0;
|
|
my $faillist="";
|
|
opendir(DIR, "$todayregresultdir/$k");
|
|
foreach my $file (readdir DIR)
|
|
{
|
|
next if($file !~ /xcattest.log/);
|
|
my $cnt = `cat $todayregresultdir/$k/$file |grep -- "------END::"|wc -l`;
|
|
$totalcnt+=int($cnt);
|
|
for(my $i=1;$i<$cnt+1;$i++){
|
|
my $line=`cat $todayregresultdir/$k/$file |grep -- "------END::"|sed -n ${i}p`;
|
|
chomp($line);
|
|
if($line =~ /------END::([a-zA-Z0-9_-]+)::([a-zA-Z0-9_-]+)::Time.+/){
|
|
my $failedcase=$1;
|
|
if($2 =~ /Failed/){
|
|
$failcnt++;
|
|
$faillist.=$failedcase.", ";
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
closedir(DIR);
|
|
|
|
my $lastfile = `ls -lrt $todayregresultdir/$k|grep "xcattest.log"|tail -1|awk '{print \$9}'`;
|
|
chomp($lastfile);
|
|
my $lastline=`cat $todayregresultdir/$k/$lastfile |grep -- "------END::"|tail -1`;
|
|
chomp($lastline);
|
|
my $lastcase;
|
|
$lastcase=$1 if($lastline =~ /------END::([a-zA-Z0-9_-]+)::([a-zA-Z0-9_-]+)::Time.+/);
|
|
|
|
$mailreport .= "\tTotalCase $totalcase TotalRun $totalcnt Failed $failcnt\n\tFailed cases: $faillist\n\tFinal environment stop at $lastcase\n";
|
|
}else{ # regression test out of time
|
|
$mailreport .= "\tTime consumption: Out of 9 hours!!!!!!\n";
|
|
}
|
|
}
|
|
|
|
&runcmd("touch $mailfile && echo \"$mailreport\" > $mailfile");
|
|
return 0;
|
|
}
|
|
|
|
#######################################
|
|
# send email
|
|
#######################################
|
|
sub send_email{
|
|
my $checkinlist="";
|
|
my $cnt =`cat $commitinfofile |grep '20[0-9][0-9] [-+]0[0-9]00'|awk '{print \$1}' |grep -v root|sort | uniq|wc -l`;
|
|
my $out =`cat $commitinfofile |grep '20[0-9][0-9] [-+]0[0-9]00'|awk '{print \$1}' |grep -v root|sort | uniq`;
|
|
for(my $i=1;$i<$cnt+1;$i++){
|
|
my $tmpline=`echo "$out"|sed -n ${i}p`;
|
|
chomp($tmpline);
|
|
$checkinlist.= "$tmpline ";
|
|
}
|
|
|
|
my $totalcnt=0;
|
|
my $failcnt=0;
|
|
my $totalcase=0;
|
|
$cnt =`cat $mailfile |grep "Total"|grep "Failed"|wc -l`;
|
|
$out =`cat $mailfile |grep "Total"|grep "Failed"`;
|
|
for(my $i=1;$i<$cnt+1;$i++){
|
|
my $tmpline=`echo "$out"|sed -n ${i}p`;
|
|
chomp($tmpline);
|
|
if($tmpline =~ /TotalCase (\d+) TotalRun (\d+) Failed (\d+)/){
|
|
$totalcase+=$1;
|
|
$totalcnt+=$2;
|
|
$failcnt+=$3;
|
|
}
|
|
}
|
|
|
|
# $gitupdateflag = 0 git update out of time
|
|
# $gitupdateflag = 1 there isn't new code checkin in last one day
|
|
# $gitupdateflag = 2 git update failed
|
|
$depupdateflag = `cat $logfiledir/$logfile |grep "updated for xcat dep" `;
|
|
my $subject="";
|
|
if($gitupdateflag==0 || $gitupdateflag==2){
|
|
$subject = "[$mailtitle] Git update failed, stop regression test today!!!";
|
|
}elsif($gitupdateflag==1 && ! $forceregwithoutupdate){
|
|
$subject = "[$mailtitle] No code checkin, stop regression test today!!!";
|
|
}elsif($gitupdateflag==1 && $forceregwithoutupdate){
|
|
`cat $logfiledir/$logfile |grep "updated for xcat dep" `;
|
|
if($?==0){
|
|
$subject = "[$mailtitle] TotalCase:".$totalcase.", TotalRun:".$totalcnt.", Fail:".$failcnt.", Branch:".$branch.", xcatdep updated, Checkin: NO ONE !!";
|
|
}else{
|
|
$subject = "[$mailtitle] TotalCase:".$totalcase.", TotalRun:".$totalcnt.", Fail:".$failcnt.", Branch:".$branch.", xcatdep no updated, Checkin: NO ONE !!";
|
|
}
|
|
}else{
|
|
`cat $logfiledir/$logfile |grep "updated for xcat dep" `;
|
|
if($?==0){
|
|
$subject = "[$mailtitle] TotalCase:".$totalcase.", TotalRun:".$totalcnt.", Fail:".$failcnt.", Branch:".$branch.", xcatdep updated, Checkin: ".$checkinlist;
|
|
}else{
|
|
$subject = "[$mailtitle] TotalCase:".$totalcase.", TotalRun:".$totalcnt.", Fail:".$failcnt.", Branch:".$branch.", xcatdep no updated, Checkin: ".$checkinlist;
|
|
}
|
|
}
|
|
my $mailreport .= "\n\n======================================\n";
|
|
$mailreport .= " Commit Codes Last Day\n";
|
|
$mailreport .= "======================================\n\n";
|
|
|
|
&runcmd("echo \"$mailreport\">> $mailfile");
|
|
&runcmd("cat $commitinfofile >> $mailfile");
|
|
|
|
my $attachfile="/tmp/failed_case_detail.txt";
|
|
&runcmd("echo \"\" > $attachfile");
|
|
if($failcnt > 0){
|
|
opendir(DIR, "$todayregresultdir");
|
|
foreach my $subdir (readdir DIR){
|
|
next if($subdir =~ /^\./);
|
|
#print "subdir-> $subdir\n";
|
|
|
|
my $scenario="[".$confkeys{$subdir."os"};
|
|
if($subdir =~ /p/){
|
|
$scenario.="+PPC64] ";
|
|
}elsif($subdir =~ /x/){
|
|
$scenario.="+x86_64] ";
|
|
}elsif($subdir =~ /E/){
|
|
$scenario.="+ppc64le] ";
|
|
}
|
|
|
|
my $separator .= "\n###############################################################\n";
|
|
$separator .= " $scenario\n";
|
|
$separator .= "###############################################################\n\n";
|
|
&runcmd("echo \"$separator\" >> $attachfile");
|
|
opendir(SUBDIR, "$todayregresultdir/$subdir");
|
|
foreach my $file (readdir SUBDIR){
|
|
next if($file !~ /^failedcases/);
|
|
next if(-z "$todayregresultdir/$subdir/$file");
|
|
#print "\t$file\n";
|
|
system("cat $todayregresultdir/$subdir/$file >> $attachfile");
|
|
}
|
|
}
|
|
}
|
|
|
|
if($mail_list){
|
|
if($failcnt > 0){
|
|
system("cat $mailfile"." | /bin/mail -s \"$subject\" -a $attachfile \"$mail_list\"");
|
|
}else{
|
|
system("cat $mailfile"." | /bin/mail -s \"$subject\" \"$mail_list\"");
|
|
}
|
|
}else{
|
|
if($failcnt > 0){
|
|
system("cat $mailfile"." | /bin/mail -s \"$subject\" -a $attachfile \"$confkeys{mailgroup}\"");
|
|
}else{
|
|
system("cat $mailfile"." | /bin/mail -s \"$subject\" \"$confkeys{mailgroup}\"");
|
|
}
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
|
|
#######################################
|
|
#git update
|
|
#return value:
|
|
#0 successful
|
|
#1 code is already at latest version
|
|
#2 someting error
|
|
#######################################
|
|
sub git_update{
|
|
send_msg(2, "[git update] starting to update xcat source code from git");
|
|
|
|
my $res = system("cd $xcatcoredir && git checkout $branch > /dev/nul 2>&1");
|
|
if ($res != 0){
|
|
send_msg(0, "[git update] change to branch $branch....[failed]");
|
|
return 2;
|
|
}
|
|
send_msg(2, "[git update] change to branch $branch....[ok]");
|
|
|
|
my $gitpulloutput = "/tmp/gitpulloutput";
|
|
my $trytime=1;
|
|
while($trytime < 4){
|
|
$res = system("cd $xcatcoredir && git pull > $gitpulloutput 2>&1");
|
|
if ($res != 0){
|
|
send_msg(0, "[git update] pull the latest code to control node....[$trytime failed]");
|
|
$trytime++;
|
|
&runcmd("sleep 120");
|
|
}else{
|
|
last;
|
|
}
|
|
}
|
|
|
|
if($trytime == 4){
|
|
send_msg(0, "[git update] pull the latest code to control node....[finally failed]");
|
|
return 2;
|
|
}
|
|
|
|
$res = system("grep 'Already up-to-date' $gitpulloutput > /dev/nul 2>&1");
|
|
if ($res == 0 && ! $forceregwithoutupdate) {
|
|
send_msg(2, "[git update] code is already at latest version. exit regresson");
|
|
return 1;
|
|
}elsif($res == 0 && $forceregwithoutupdate){
|
|
send_msg(2, "[git update] code is already at latest version. But force to run regresson");
|
|
return 0;
|
|
}
|
|
send_msg(2, "[git update] pull the latest code to control node....[ok]");
|
|
|
|
my $lastcommitfile="/tmp/lastcommitfile";
|
|
my $lastcommitinfo;
|
|
if (-f $lastcommitfile){
|
|
$lastcommitinfo = `cat $lastcommitfile`;
|
|
chomp($lastcommitinfo);
|
|
}else{
|
|
$lastcommitinfo = "";
|
|
}
|
|
#print "lastcommitinfo=$lastcommitinfo\n";
|
|
|
|
if($lastcommitinfo eq ""){
|
|
$res = system("cd $xcatcoredir && git log --since=\"1 days ago\" --pretty=format:\"%cn %ad %s\" --name-status > $commitinfofile");
|
|
if ($res != 0 || (-f $commitinfofile and -z _)){
|
|
send_msg(0, "[git update] get commit information....[failed]");
|
|
return 2;
|
|
}
|
|
send_msg(0, "[git update] get commit information....[ok]");
|
|
}else{
|
|
$res = system("cd $xcatcoredir && git log $lastcommitinfo..HEAD --pretty=format:\"%cn %ad %s\" --name-status > $commitinfofile");
|
|
if ($res != 0 || (-f $commitinfofile and -z _)){
|
|
send_msg(0, "[git update] get commit information....[failed]");
|
|
return 2;
|
|
}
|
|
send_msg(2, "[git update] get commit information....[ok]");
|
|
}
|
|
|
|
$res = system("cd $xcatcoredir && git log -n 1|awk 'NR==1'|awk '{print \$2}' > $lastcommitfile");
|
|
if ($res != 0){
|
|
send_msg(0, "[git update] get latest commit number failed");
|
|
exit;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
#######################################
|
|
# step 0. Parse input arguments
|
|
#######################################
|
|
send_msg(2, "step 1, Parse input arguments............");
|
|
=pod
|
|
if (
|
|
!GetOptions("--help|h|?" => \$needhelp,
|
|
"f=s" => \$configfile,
|
|
"b=s" => \$branch,
|
|
"m=s" => \$management_node,
|
|
"V" => \$verbose_mode,
|
|
"U" => \$updates_regression,
|
|
"e=s" => \$send_msg)
|
|
)
|
|
=cut
|
|
if (
|
|
!GetOptions("--help|h|?" => \$needhelp,
|
|
"f=s" => \$configfile,
|
|
"b=s" => \$branch,
|
|
"m=s" => \$management_node,
|
|
"a" => \$forceregwithoutupdate,
|
|
"e=s" => \$mail_list)
|
|
)
|
|
{
|
|
&usage;
|
|
send_msg(0, "step 0, PARSE ARGUMENTS returns error, exit");
|
|
exit;
|
|
}
|
|
if ($needhelp)
|
|
{
|
|
&usage;
|
|
exit 0;
|
|
}
|
|
|
|
|
|
xCATreg_init();
|
|
|
|
###############################################################
|
|
# Mainfunction
|
|
###############################################################
|
|
send_msg(2,"........................");
|
|
send_msg(2,"........................");
|
|
send_msg(2,".....ooooO..............");
|
|
send_msg(2,"....(....)....Ooooo.....");
|
|
send_msg(2,".....\\..(.....(....)....");
|
|
send_msg(2,"......\\__).....)../.....");
|
|
send_msg(2,"..............(_ /......");
|
|
send_msg(2,"........................");
|
|
send_msg(2,".........START .........");
|
|
send_msg(2,"........................");
|
|
#######################################
|
|
# step 1. Read configuration files
|
|
#######################################
|
|
send_msg(2, "step 2, Reading xCATreg conf............");
|
|
my $rst = read_conf();
|
|
unless (! $rst) {
|
|
send_msg(0, "read xCATreg configuration failed...........exit");
|
|
exit;
|
|
}
|
|
|
|
foreach my $k (keys %confkeys) {
|
|
print "$k = $confkeys{$k}\n";
|
|
}
|
|
|
|
# begin to find which mn should be used
|
|
if ($management_node) {
|
|
my @all;
|
|
if ($management_node =~ /,/) {
|
|
@all = split /,/, $management_node;
|
|
} else {
|
|
push @all, $management_node;
|
|
}
|
|
foreach my $mn (@all) {
|
|
foreach my $k (keys %confkeys) {
|
|
if ($confkeys{$k} eq $mn) {
|
|
$mns{$k} = $mn;
|
|
send_msg(2, "specify mn :$mn");
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
pipe PREAD,CWRITE;
|
|
my $gitpid = fork();
|
|
if( !defined($gitpid)){
|
|
send_msg(0, "fork process for git update error");
|
|
exit;
|
|
}elsif($gitpid ==0){
|
|
close PREAD;
|
|
send_msg(2, "..........fork 1 process[pid=$$] for git update..........");
|
|
my $res = git_update();
|
|
if($res > 1){
|
|
send_msg(0, "git_update......[failed]");
|
|
syswrite CWRITE,"git update failed\n";
|
|
close CWRITE;
|
|
exit 1;
|
|
}
|
|
if($res == 1){
|
|
send_msg(2, "git_update......[not find new code]");
|
|
syswrite CWRITE,"git update got nothing\n";
|
|
close CWRITE;
|
|
exit 1;
|
|
}
|
|
send_msg(2, "git_update......[ok]");
|
|
syswrite CWRITE,"git update successful\n";
|
|
close CWRITE;
|
|
exit 0;
|
|
}
|
|
|
|
{#main process
|
|
close CWRITE;
|
|
my $gitstarttime = time();
|
|
my $gitprocess=1;
|
|
my $gitselect = new IO::Select;
|
|
$gitselect->add(\*PREAD);
|
|
while($gitprocess){
|
|
my @hdls;
|
|
if (@hdls = $gitselect->can_read(0)) {
|
|
my $hdl;
|
|
foreach $hdl (@hdls) {
|
|
if ($hdl == \*PREAD) {
|
|
my $line;
|
|
chomp($line=<PREAD>);
|
|
if ($line){
|
|
if($line =~ /successful/){
|
|
send_msg(2, "[[main]]: $line");
|
|
$gitupdateflag=3;
|
|
}
|
|
if($line =~ /failed/){
|
|
send_msg(0, "[[main]]: $line");
|
|
$gitupdateflag=2;
|
|
}
|
|
if($line =~ /nothing/){
|
|
send_msg(2, "[[main]]: there isn't new code checkin during last one day.");
|
|
$gitupdateflag=1;
|
|
}
|
|
$gitprocess = 0;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if(time() - $gitstarttime > 7200) {
|
|
#if(time() - $gitstarttime > 2) {
|
|
send_msg(0, "[[main]] 2 hours is expired. git update out of time");
|
|
last;
|
|
}
|
|
&runcmd("sleep 2");
|
|
}
|
|
close PREAD;
|
|
|
|
&runcmd("kill $gitpid") if($gitprocess==1);
|
|
send_msg(2, "[[main]]........git process[$gitpid] exit successful..............");
|
|
|
|
# $gitupdateflag = 0 git update out of time
|
|
# $gitupdateflag = 1 there isn't new code checkin in last one day
|
|
# $gitupdateflag = 2 there is something wrong during git update
|
|
if($gitupdateflag<3){
|
|
send_msg(2, "[[main]]: creat regression result report...........");
|
|
$rst = creat_report();
|
|
if ($rst) {
|
|
send_msg(0, "[[main]]: creat regression result report...........[FAILED]");
|
|
}else{
|
|
send_msg(2, "[[main]]: creat regression result report...........[OK]");
|
|
|
|
send_msg(2, "[[main]]: send report email...........");
|
|
$rst = send_email();
|
|
if ($rst) {
|
|
send_msg(0, "[[main]]: send report email...........[FAILED]");
|
|
}else{
|
|
send_msg(2, "[[main]]: send report email...........[OK]");
|
|
}
|
|
}
|
|
exit 1;
|
|
}
|
|
send_msg(2, "[[main]]: git update successful, start regression today........");
|
|
|
|
#exit 0
|
|
}
|
|
|
|
|
|
my $platformfork = 0;
|
|
my $timingork = 0;
|
|
# begin child process
|
|
pipe CONTROLREAD,MNWRITE;
|
|
foreach my $m (keys %mns) {
|
|
$platformfork += 1;
|
|
my $mn = $mns{$m};
|
|
my $pid = fork();
|
|
if ( !defined($pid) ) {
|
|
send_msg(0, "fork process for $m error");
|
|
exit;
|
|
} elsif ( $pid == 0 ) { # child process
|
|
send_msg(2, "..........fork process[pid=$$] for $m [$confkeys{$m}]..........");
|
|
close CONTROLREAD;
|
|
my $res;
|
|
|
|
#######################################
|
|
# step 3. download xcat core
|
|
########################################
|
|
send_msg(2, "[$m]step 3, Running down_xcat_core...............");
|
|
$res = down_xcat_core_build($m);
|
|
if ($res) {
|
|
syswrite MNWRITE,"REPORTFROM:$m->$mn: download xcat core for $mn failed\n";
|
|
exit 1;
|
|
}
|
|
send_msg(2, "[$m]step 3, Run down_xcat_core...............[OK]");
|
|
|
|
#######################################
|
|
# step 4. Install MNs,
|
|
#######################################
|
|
send_msg(2, "[$m]step 4, Running mn_install...............");
|
|
$res = mn_install($m);
|
|
if ($res) {
|
|
syswrite MNWRITE,"REPORTFROM:$m->$mn: install $mn failed\n";
|
|
exit 1;
|
|
}
|
|
send_msg(2, "[$m]step 4, Run mn_install...............[OK]");
|
|
#######################################
|
|
# step 5. Down xcatdep to MNs
|
|
#######################################
|
|
send_msg(2, "[$m]step 5, Down xcatdep...............");
|
|
$depupdateflag = down_dep($m);
|
|
if ($depupdateflag ==2){
|
|
send_msg(2, "[$m]step 5, updated for xcat dep");
|
|
}else{
|
|
send_msg(2, "[$m]step 5, no update for xcat dep");
|
|
}
|
|
#######################################
|
|
# step 6. Copy code to MNs
|
|
#######################################
|
|
send_msg(2, "[$m]step 6, Running copy_code...............");
|
|
$res = copy_code($m);
|
|
if ($res) {
|
|
syswrite MNWRITE,"REPORTFROM:$m->$mn: copy xcat source code to $mn failed\n";
|
|
exit 1;
|
|
}
|
|
send_msg(2, "[$m]step 6, Run copy_code...............[OK]");
|
|
|
|
########################################
|
|
# step 7. Read xCAT MN's configuration
|
|
#######################################
|
|
send_msg(2, "[$m]step 7, Running install_xcat...............");
|
|
$res = install_xcat($m);
|
|
if ($res) {
|
|
syswrite MNWRITE,"REPORTFROM:$m->$mn: install new xcat in $mn failed\n";
|
|
exit 1;
|
|
}
|
|
send_msg(2, "[$m]step 7, Run install_xcat...............[OK]");
|
|
|
|
#######################################
|
|
# step 8. Genrate local configuration file for xcattest
|
|
# Do test
|
|
# Write log
|
|
#######################################
|
|
send_msg(2, "[$m]step 8, Running do_test...............");
|
|
$res = do_test($m);
|
|
if ($res) {
|
|
syswrite MNWRITE,"REPORTFROM:$m->$mn: run regression test cases in $mn failed\n";
|
|
exit 1;
|
|
}
|
|
send_msg(2, "[$m]step 8, Running do_test...............[OK]");
|
|
|
|
send_msg(2, "whole regression test for $mn finished");
|
|
syswrite MNWRITE,"REPORTFROM:$m->$mn: whole regression test are successful\n";
|
|
exit 0;
|
|
} # end of child process
|
|
&runcmd("mkdir -p $todayregresultdir/$m");
|
|
$pidrecord{$m} = $pid;
|
|
$piderrrecord{$m} = "";
|
|
} # end of foreach
|
|
|
|
|
|
{ # parent process
|
|
close MNWRITE;
|
|
my $rst;
|
|
my $regstarttime = time();
|
|
|
|
my $select = new IO::Select;
|
|
$select->add(\*CONTROLREAD);
|
|
while ($platformfork) {
|
|
my @hdls;
|
|
if (@hdls = $select->can_read(0)) {
|
|
my $hdl;
|
|
foreach $hdl (@hdls) {
|
|
if ($hdl == \*CONTROLREAD) {
|
|
my $line;
|
|
chomp($line=<CONTROLREAD>);
|
|
if ($line){
|
|
if($line =~ /successful/){
|
|
send_msg(2, "[[main]]: $line");
|
|
$line =~ s/^REPORTFROM:(.+)->.+/$1/g;
|
|
$pidrecord{$line} = 0;
|
|
$consumptionrecord{$line} = time() - $regstarttime;
|
|
}
|
|
if($line =~ /failed/){
|
|
send_msg(0, "[[main]]: $line");
|
|
my $tmp = $line;
|
|
$line =~ s/^REPORTFROM:(.+)->.+/$1/g;
|
|
$pidrecord{$line} = 0;
|
|
$consumptionrecord{$line} = time() - $regstarttime;
|
|
$tmp =~ s/REPORTFROM:.+->.+:(.+)/$1/g;
|
|
$piderrrecord{$line} = $tmp;
|
|
}
|
|
$platformfork -= 1;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if(time() - $regstarttime > 36000) {
|
|
#if($curtime - $regstarttime > 10) {
|
|
send_msg(1, "[timing] 8 hours is expired");
|
|
last;
|
|
}
|
|
&runcmd("sleep 1");
|
|
}
|
|
|
|
close CONTROLREAD;
|
|
if($platformfork){
|
|
send_msg(0, "[[main]]: there are $platformfork platform's regression test return out of time");
|
|
while(my($k,$v) = each %pidrecord){
|
|
&runcmd("kill $v") if ($v != 0);
|
|
}
|
|
}else{
|
|
send_msg(2, "[[main]]: all platform's regression test return on time");
|
|
}
|
|
|
|
send_msg(2, "[[main]]: creat regression result report...........");
|
|
$rst = creat_report();
|
|
if ($rst) {
|
|
send_msg(0, "[[main]]: creat regression result report...........[FAILED]");
|
|
}else{
|
|
send_msg(2, "[[main]]: creat regression result report...........[OK]");
|
|
|
|
send_msg(2, "[[main]]: send report email...........");
|
|
$rst = send_email();
|
|
if ($rst) {
|
|
send_msg(0, "[[main]]: send report email...........[FAILED]");
|
|
}else{
|
|
send_msg(2, "[[main]]: send report email...........[OK]");
|
|
}
|
|
}
|
|
send_msg(2, "[[main]]: main process exit");
|
|
exit 0;
|
|
}
|
|
|
|
|