1350 lines
51 KiB
Perl
1350 lines
51 KiB
Perl
#!/usr/bin/env perl
|
|
# IBM(c) 2010 EPL license http://www.eclipse.org/legal/epl-v10.html
|
|
#
|
|
#############################################################################################################################################
|
|
#
|
|
# This script will be used to run xCAT daily regression
|
|
#
|
|
# Usage:xCATreg - Run xCAT daily regression
|
|
# xCATreg [-?|-h]
|
|
# xCATreg [-b branch][-m MN][-f configure file][-a][-V]
|
|
#
|
|
# -?/-h: help information for the xCATreg usage.
|
|
# -b: when -b is specified, the regression will be done against the specified branch. You can use 2.7, 2.8,master.The default value is master.
|
|
# -m: when -m is specified, the regression will be done the specified mn. The default value is all the mn in configuration file.
|
|
# -f :specify the configuration file,default value is /xCATreg/regresson.conf
|
|
# -a: when -a is specified, run the regression even there is no code updates
|
|
# -V: log and message in verbose mode
|
|
#
|
|
# Example usage:
|
|
# ./xCATreg Run regressoins on master branch and on all platforms in configuration file
|
|
# ./xCATreg -b 2.8 -m slesmn -f /root/regressoin.conf -a Run regression on 2.8 branch on slesmn even when there is no code updates.
|
|
#
|
|
#############################################################################################################################################
|
|
|
|
$ENV{PATH} = "/opt/xcat/bin:/opt/xcat/sbin:/opt/xcat/share/xcat/tools:/usr/sbin:/usr/bin:/sbin:/bin";
|
|
|
|
use strict;
|
|
use warnings;
|
|
use Getopt::Long;
|
|
use Data::Dumper;
|
|
use Term::ANSIColor;
|
|
use Time::Local;
|
|
use IO::Select;
|
|
|
|
BEGIN
|
|
{
|
|
$::XCATROOT = $ENV{'XCATROOT'} ? $ENV{'XCATROOT'} : -d '/opt/xcat' ? '/opt/xcat' : '/usr';
|
|
}
|
|
use lib "$::XCATROOT/lib/perl";
|
|
|
|
my $date;
|
|
#my $regrootdir="/home/autotest_code";
|
|
#my $mailtitle="P Cluster";
|
|
my $regrootdir="/xCATreg";
|
|
my $mailtitle="X Cluster";
|
|
my $logfiledir = "$regrootdir/log";
|
|
my $logfile="";
|
|
my $configfiledir = "$regrootdir/conf";
|
|
my $configfile = "$regrootdir/conf/xCATreg.conf";
|
|
my $xcatcoredir = "$regrootdir/xcatsource/xcat-core";
|
|
my $xcatdepdir = "$regrootdir/xcatsource/xcat-dep";
|
|
my $osisodir = "$regrootdir/iso";
|
|
my $regresultdir = "$regrootdir/regresult";
|
|
my $todayregresultdir="";
|
|
my $maildir="$regrootdir/mail";
|
|
my $mailfile="";
|
|
my $rootdir = "$::XCATROOT/share/xcat/tools/autotest";
|
|
my %confkeys;
|
|
my $mail_list = 0;
|
|
my $needhelp = 0;
|
|
my $branch = "master";
|
|
my $management_node = undef;
|
|
#my $verbose_mode = 0;
|
|
my $updates_regression = 0;
|
|
my %mns;
|
|
my %pidrecord;
|
|
my %piderrrecord;
|
|
my %consumptionrecord;
|
|
my $commitinfofile="/tmp/commitinfo";
|
|
my $gitupdateflag = 0;
|
|
my $forceregwithoutupdate=0;
|
|
|
|
|
|
#######################################
|
|
# runcmd
|
|
#######################################
|
|
sub runcmd {
|
|
my ($cmd) = @_;
|
|
my $rc = 0;
|
|
$::RUNCMD_RC = 0;
|
|
my $outref = [];
|
|
@$outref = `$cmd 2>&1`;
|
|
if ($?)
|
|
{
|
|
$rc = $? ;
|
|
$::RUNCMD_RC = $rc;
|
|
}
|
|
chomp(@$outref);
|
|
return @$outref;
|
|
}
|
|
|
|
#######################################
|
|
# clean_outdated_file
|
|
#######################################
|
|
sub is5daysago{
|
|
my $targetdate= shift;
|
|
my $targetdatey="";
|
|
my $targetdatem="";
|
|
my $targetdated="";
|
|
my $datey="";
|
|
my $datem="";
|
|
my $dated="";
|
|
if($targetdate =~ /(\d{4})(\d{2})(\d{2})/){
|
|
$targetdatey=$1;
|
|
$targetdatem=$2-1;
|
|
$targetdated=$3;
|
|
}
|
|
if($date =~ /(\d{4})(\d{2})(\d{2})/){
|
|
$datey=$1;
|
|
$datem=$2-1;
|
|
$dated=$3;
|
|
}
|
|
my $todays=timelocal(0,0,0,$dated,$datem,$datey);
|
|
my $targets=timelocal(0,0,0,$targetdated,$targetdatem,$targetdatey);
|
|
if($todays - $targets > 432000){
|
|
return 1;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
#######################################
|
|
# clean_outdated_file
|
|
#######################################
|
|
sub clean_outdated_file{
|
|
opendir(DIR, "$logfiledir");
|
|
foreach my $file (readdir DIR){
|
|
if($file =~ /xCATreg(\d+).log/){
|
|
my $res = is5daysago($1);
|
|
if($res){
|
|
&runcmd("rm -f $logfiledir/$file");
|
|
}
|
|
}
|
|
}
|
|
closedir(DIR);
|
|
|
|
opendir(DIR, "$maildir");
|
|
foreach my $file (readdir DIR){
|
|
if($file =~ /mailreport.(\d+)/){
|
|
my $res = is5daysago($1);
|
|
if($res){
|
|
&runcmd("rm -f $maildir/$file");
|
|
}
|
|
}
|
|
}
|
|
closedir(DIR);
|
|
|
|
opendir(DIR, "$regresultdir");
|
|
foreach my $file (readdir DIR){
|
|
if($file =~ /(\d+)/){
|
|
my $res = is5daysago($1);
|
|
if($res){
|
|
&runcmd("rm -rf $regresultdir/$file");
|
|
}
|
|
}
|
|
}
|
|
closedir(DIR);
|
|
|
|
return 0;
|
|
}
|
|
|
|
#######################################
|
|
# xCATreg_init
|
|
#######################################
|
|
sub xCATreg_init{
|
|
|
|
#init important attributes
|
|
|
|
$date = `date +"%Y%m%d"`;
|
|
chomp($date);
|
|
|
|
&runcmd("mkdir -p $logfiledir") if(! -e $logfiledir);
|
|
&runcmd("mkdir -p $maildir") if(! -e $maildir);
|
|
&runcmd("mkdir -p $regresultdir") if(! -e $regresultdir);
|
|
&runcmd("mkdir -p $xcatcoredir") if(! -e $xcatcoredir);
|
|
|
|
$logfile="xCATreg".$date.".log";
|
|
$todayregresultdir=$regresultdir."/".$date;
|
|
&runcmd("mkdir -p $todayregresultdir");
|
|
$mailfile=$maildir."/mailreport.".$date;
|
|
|
|
my $cnt=`ps aux|grep xCATreg |grep -v grep|awk '{print \$2}'|wc -l`;
|
|
my $out=`ps aux|grep xCATreg |grep -v grep|awk '{print \$2}'`;
|
|
for(my $i=1;$i<$cnt+1;$i++){
|
|
my $pid=`echo "$out"|sed -n ${i}p`;
|
|
chomp($pid);
|
|
next if($pid == $$);
|
|
&runcmd("kill $pid");
|
|
}
|
|
|
|
clean_outdated_file();
|
|
return 0;
|
|
}
|
|
|
|
#######################################
|
|
# usage for arguments
|
|
#######################################
|
|
sub usage{
|
|
print "Usage:xCATreg - Run xcat autotest test cases.\n
|
|
Explanation for the options:
|
|
-b: when -b is specified, the regression will be done against the specified branch. The default value is master.\n
|
|
-m: when -m is specified, run regression on the specified mn. The default value is all the mn in configuration file.\n
|
|
-f: specify the configuration file location.\n
|
|
-a: when -a is specified, run the regression even there is no code updates.\n
|
|
-e: <email_addr> send the test result to email_addr\n\n";
|
|
print " xCATreg [-?|-h]\n";
|
|
print " xCATreg [-f configure file] [-b branch] [-m mangement node] [-e mail list] [-a] Even code not updates, there will be regression.\n";
|
|
print " xCATreg [-f configure file] [-b branch] [-m mangement node] [-e mail list] if code updates there will be regression.\n";
|
|
print "\n";
|
|
return;
|
|
}
|
|
|
|
#######################################
|
|
# send messages
|
|
#######################################
|
|
sub send_msg {
|
|
my $num = shift;
|
|
my $msg = shift;
|
|
my $content;
|
|
if ($num == 0) {
|
|
$content = "Fatal error:";
|
|
} elsif($num == 1) {
|
|
$content = "Warning:";
|
|
} elsif($num == 2) {
|
|
$content = "Notice:";
|
|
}
|
|
my $timestamp = `date +"%Y%m%d%H%M%S"`;
|
|
chomp($timestamp);
|
|
if ( !open (LOGFILE, ">> $logfiledir/$logfile") ) {
|
|
return 1;
|
|
}
|
|
print LOGFILE "$content $timestamp $$: $msg.\n";
|
|
close LOGFILE;
|
|
}
|
|
|
|
|
|
#######################################
|
|
# read_conf
|
|
#######################################
|
|
sub read_conf{
|
|
my $myfile=undef;
|
|
my $line=undef;
|
|
|
|
if (!open($myfile, "$configfile")) {
|
|
send_msg(0, "Open $configfile failed");
|
|
return 1;
|
|
}
|
|
while ($line = <$myfile>) {
|
|
$line =~ s/\s//g;
|
|
next if($line =~ /^#/);
|
|
next if($line eq "");
|
|
my @attr=split(/=/,$line);
|
|
$confkeys{$attr[0]} = $attr[1];
|
|
}
|
|
close($myfile);
|
|
return 0;
|
|
}
|
|
|
|
#######################################
|
|
# mn_install to install all mn
|
|
#######################################
|
|
sub mn_install {
|
|
my $mn=shift;
|
|
|
|
&runcmd("lsdef|grep $confkeys{$mn}");
|
|
if($?){
|
|
send_msg(0, "[$mn->$confkeys{$mn}][mn_install] Didn't defined $confkeys{$mn} in current control node");
|
|
return 1;
|
|
}
|
|
|
|
my $os = $confkeys{$mn."os"};
|
|
my $arch = "ppc64";
|
|
$arch="x86_64" if($mn !~ /aix/ && $mn =~ /x/);
|
|
my $osimage="";
|
|
|
|
if($mn !~ /aix/){
|
|
$osimage=$os."-".$arch."-install-compute";
|
|
}else{
|
|
$osimage="aix"; #????????need to test for aix
|
|
}
|
|
|
|
#print "[$mn] : os= $os arch=$arch osimage=$osimage\n";
|
|
|
|
&runcmd("lsdef -t osimage|grep $osimage");
|
|
if($?){
|
|
send_msg(0, "[$mn->$confkeys{$mn}][mn_install] Didn't defined $osimage in current control node");
|
|
return 1;
|
|
}
|
|
|
|
send_msg(2, "[$mn->$confkeys{$mn}][mn_install] installing $osimage for node $confkeys{$mn}................");
|
|
|
|
&runcmd("nodeset $confkeys{$mn} osimage=$osimage");
|
|
if( $arch =~ /x86_64/){
|
|
&runcmd("rpower $confkeys{$mn} boot");
|
|
}else{
|
|
&runcmd("rnetboot $confkeys{$mn}");
|
|
}
|
|
|
|
if($mn !~ /ubuntu/){
|
|
&runcmd("sleep 300");
|
|
&runcmd("a=0;while ! `lsdef -l $confkeys{$mn}|grep status|grep booted >/dev/null`; do sleep 10;((a++));if [ \$a -gt 200 ];then break;fi done");}
|
|
else{
|
|
&runcmd("sleep 1800");
|
|
}
|
|
|
|
&runcmd("lsdef -l $confkeys{$mn}|grep status|grep booted >/dev/null");
|
|
if($?){
|
|
send_msg(0, "[$mn->$confkeys{$mn}][mn_install] install $osimage for node $confkeys{$mn} failed");
|
|
return 1;
|
|
}
|
|
|
|
if($mn =~ /ubuntu/){
|
|
system("updatenode $confkeys{$mn} -P \"confignics -s\" >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} \"iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE\" >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} \"iptables -A FORWARD -i eth1 -j ACCEPT\" >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} \"iptables -A FORWARD -i eth1 -o eth0 -m state --state RELATED,ESTABLISHED -j ACCEPT\" >/dev/null 2>&1");
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
|
|
#######################################
|
|
# copy code
|
|
#######################################
|
|
sub copy_code {
|
|
my $mn=shift;
|
|
|
|
send_msg(2, "[$mn->$confkeys{$mn}][copy_code] starting to copy the latest xcat code and dep");
|
|
|
|
#copy xcat-dep tarball to MN:/
|
|
my $depball="";
|
|
if($mn =~ /aix/){
|
|
$depball = `ls -l $xcatdepdir|awk '{print \$9}'|grep aix|tail -n 1`;
|
|
}elsif($mn =~ /ubuntu/){
|
|
$depball = `ls -l $xcatdepdir|awk '{print \$9}'|grep ubuntu|tail -n 1`;
|
|
}else{
|
|
$depball = `ls -l $xcatdepdir|awk '{print \$9}'|grep 'xcat-dep-[0-9]\\{12\\}.tar.bz'|tail -n 1`;
|
|
}
|
|
chomp($depball);
|
|
if ($depball eq ""){
|
|
send_msg(0, "[$mn->$confkeys{$mn}][copy_code] can't find dep tarball for $mn under dir $xcatdepdir");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[$mn->$confkeys{$mn}][copy_code] find dep tarball $depball for $mn");
|
|
|
|
my $res = system("scp $xcatdepdir/$depball root\@$confkeys{$mn}:/ >/dev/null");
|
|
if ($?){
|
|
send_msg(0, "[$mn->$confkeys{$mn}][copy_code] copy $depball to $mn failed");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[$mn->$confkeys{$mn}][copy_code] copy $depball to $mn successfully");
|
|
|
|
$res = system("xdsh $confkeys{$mn} 'cd / && tar xvf /$depball' >/dev/null 2>&1");
|
|
if ($?){
|
|
send_msg(0, "[$mn->$confkeys{$mn}][copy_code] untar $depball in $mn failed");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[$mn->$confkeys{$mn}][copy_code] untar $depball in $mn successfully");
|
|
|
|
#copy xcat-core source code to MN:/
|
|
if($mn =~ /ubuntux/){
|
|
$res = system("xdsh $confkeys{$mn} 'mkdir -p /xcatbuild' && scp -r $xcatcoredir root\@$confkeys{$mn}:/xcatbuild/ >/dev/null");
|
|
}else{
|
|
$res = system("scp -r $xcatcoredir root\@$confkeys{$mn}:/ >/dev/null");
|
|
}
|
|
if ($res){
|
|
send_msg(0, "[$mn->$confkeys{$mn}][copy_code] copy the latest xcat code to $mn failed");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[$mn->$confkeys{$mn}][copy_code] copy the latest xcat code to $mn successfully");
|
|
|
|
#copy /etc/hosts file to MN
|
|
$res = system("scp /etc/hosts root\@$confkeys{$mn}:/etc >/dev/null");
|
|
if ($?){
|
|
send_msg(0, "[$mn->$confkeys{$mn}][copy_code] copy /etc/hosts to $mn failed");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[$mn->$confkeys{$mn}][copy_code] copy /etc/hosts to $mn successfully");
|
|
|
|
my $os = $confkeys{$mn."os"};
|
|
my $regconf=$mn.".conf";
|
|
system("sed -i 's|os=\\S*|os=$os|g' $configfiledir/$regconf");
|
|
|
|
my $arch = "ppc64";
|
|
$arch="x86_64" if($mn !~ /aix/ && $mn =~ /x/);
|
|
my $iso;
|
|
|
|
#copy specific files to MN for each platform
|
|
if($mn =~ /rh/){
|
|
$res = system("scp /install/$os/$arch/RPM-GPG-KEY-redhat-release root\@$confkeys{$mn}:/ >/dev/null");
|
|
if ($?){
|
|
send_msg(0, "[$mn->$confkeys{$mn}][copy_code] copy RPM-GPG-KEY-redhat-release to $mn failed");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[$mn->$confkeys{$mn}][copy_code] copy RPM-GPG-KEY-redhat-release to $mn successfully");
|
|
|
|
$os =~ /(\D+)(\d+\.\d+)/;
|
|
$iso=`ls -l $osisodir|awk '{print \$9}'|grep RHEL|grep $arch|grep $2|tail -n 1`;
|
|
chomp($iso);
|
|
#print "[$mn->$confkeys{$mn}][copy_code] iso=$iso\n";
|
|
if ($iso eq ""){
|
|
send_msg(0, "[$mn->$confkeys{$mn}][copy_code] can't find iso for $os and $arch under dir $osisodir");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[$mn->$confkeys{$mn}][copy_code] find $iso for os=$os and arch=$arch");
|
|
|
|
$res = system("scp $osisodir/$iso root\@$confkeys{$mn}:/ >/dev/null");
|
|
if ($?){
|
|
send_msg(0, "[$mn->$confkeys{$mn}][copy_code] copy ISO $iso to $mn failed");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[$mn->$confkeys{$mn}][copy_code] copy ISO $iso to $mn successfully");
|
|
system ("sed -i 's|ISO=/\\S*|ISO=/$iso|g' $configfiledir/$regconf");
|
|
}
|
|
|
|
if($mn =~ /sles/){
|
|
$os =~ /(\D+)(\d+)\.?(\d?)/;
|
|
if($3){
|
|
$iso=`ls -l $osisodir| awk '{print \$9}'|grep SLE|grep $arch |grep -- "-$2-SP$3-" |tail -n 1`;
|
|
}else{
|
|
$iso=`ls -l $osisodir| awk '{print \$9}'|grep SLE|grep $arch |grep -v "SP"|grep -- "-$2-"|tail -n 1`;
|
|
}
|
|
chomp($iso);
|
|
#print "[$mn->$confkeys{$mn}][copy_code] iso=$iso\n";
|
|
if ($iso eq ""){
|
|
send_msg(0, "[$mn->$confkeys{$mn}][copy_code] can't find ISO for os=$os and arch=$arch under dir $osisodir");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[$mn->$confkeys{$mn}][copy_code] find $iso for os=$os and arch=$arch");
|
|
|
|
$res = system("scp $osisodir/$iso root\@$confkeys{$mn}:/ >/dev/null");
|
|
if ($?){
|
|
send_msg(0, "[$mn->$confkeys{$mn}][copy_code] copy ISO $iso to $mn failed");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[$mn->$confkeys{$mn}][copy_code] copy ISO $iso to $mn successfully");
|
|
system ("sed -i 's|ISO=/\\S*|ISO=/$iso|g' $configfiledir/$regconf");
|
|
|
|
if($os =~ /sles11/){
|
|
$res = system("xdsh $confkeys{$mn} \"mv /usr/lib/rpm/find-requires /usr/lib/rpm/find-requires.org\" && scp $osisodir/find-requires $confkeys{$mn}:/usr/lib/rpm/ > /dev/null");
|
|
if ($?){
|
|
send_msg(0, "[$mn->$confkeys{$mn}][copy_code] For sles11, copy find-requires to $mn failed");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[$mn->$confkeys{$mn}][copy_code] For sles11, copy find-requires to $mn successfully");
|
|
}
|
|
}
|
|
if($mn =~ /ubuntu/){
|
|
$os =~ /(\D+)(\d+\.\d+\.?\d?)/;
|
|
my $osversion = $2;
|
|
if ($mn =~ /ubuntux/){
|
|
$iso=`ls -l $osisodir| awk '{print \$9}'|grep ubuntu|grep amd64 |grep $osversion |tail -n 1`;
|
|
}elsif($mn =~ /ubuntupel/){
|
|
$iso=`ls -l $osisodir| awk '{print \$9}'|grep ubuntu|grep ppc64el |grep $osversion |tail -n 1`;
|
|
}
|
|
chomp($iso);
|
|
#print "[$mn->$confkeys{$mn}][copy_code] iso=$iso\n";
|
|
if ($iso eq ""){
|
|
send_msg(0, "[$mn->$confkeys{$mn}][copy_code] can't find ISO for os=$os and arch=$arch under dir $osisodir");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[$mn->$confkeys{$mn}][copy_code] find $iso for os=$os and arch=$arch");
|
|
|
|
$res = system("scp $osisodir/$iso root\@$confkeys{$mn}:/ >/dev/null");
|
|
if ($?){
|
|
send_msg(0, "[$mn->$confkeys{$mn}][copy_code] copy ISO $iso to $mn failed");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[$mn->$confkeys{$mn}][copy_code] copy ISO $iso to $mn successfully");
|
|
system ("sed -i 's|ISO=/\\S*|ISO=/$iso|g' $configfiledir/$regconf");
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
|
|
#######################
|
|
# build xcat
|
|
#######################
|
|
sub build_xcat {
|
|
my $mn = shift;
|
|
|
|
#print "=======[$mn->$confkeys{$mn}] build_xcat========\n";
|
|
send_msg(2, "[$mn->$confkeys{$mn}][build_xcat] starting to build xcat in $confkeys{$mn}");
|
|
|
|
#modify /etc/resolv.conf
|
|
my $dnsserver=`cat /etc/hosts |grep $confkeys{$mn}|awk '{print \$1}'|head -n 1`;
|
|
chomp($dnsserver);
|
|
my $res = system("xdsh $confkeys{$mn} \"sed -i 's:nameserver .*:nameserver $dnsserver:g' /etc/resolv.conf\" >/dev/null 2>&1");
|
|
if ($res != 0){
|
|
send_msg(0, "[$mn->$confkeys{$mn}][install_xcat] modify resolv.conf in $mn failed");
|
|
return 1;
|
|
}
|
|
|
|
if($mn =~ /rh/){
|
|
system("xdsh $confkeys{$mn} 'yum -y groupinstall \"Development tools\"' >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} 'yum -y groupinstall \"Perl Support\"' >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} 'yum -y install createrepo' >/dev/null 2>&1");
|
|
}
|
|
|
|
if($mn =~ /sles/){
|
|
system("xdsh $confkeys{$mn} 'zypper -n install perl-DBI' >/dev/null 2>&1");
|
|
}
|
|
|
|
if($mn =~ /ubuntu/){
|
|
system("xdsh $confkeys{$mn} \"echo \"nameserver 9.0.2.1\" >> /etc/resolv.conf\" >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} 'apt-get -y install software-properties-common' >/dev/null 2>&1");
|
|
if($mn =~ /ubuntux/){
|
|
system("xdsh $confkeys{$mn} 'add-apt-repository \"deb http://archive.ubuntu.com/ubuntu \$(lsb_release -sc) main\"' >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} 'add-apt-repository \"deb http://archive.ubuntu.com/ubuntu \$(lsb_release -sc)-updates main\"' >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} 'add-apt-repository \"deb http://archive.ubuntu.com/ubuntu \$(lsb_release -sc) universe\"' >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} 'add-apt-repository \"deb http://archive.ubuntu.com/ubuntu \$(lsb_release -sc)-updates universe\"' >/dev/null 2>&1");
|
|
}elsif($mn =~ /ubuntupel/){
|
|
system("xdsh $confkeys{$mn} 'add-apt-repository \"deb http://ports.ubuntu.com/ubuntu-ports \$(lsb_release -sc) main\"' >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} 'add-apt-repository \"deb http://ports.ubuntu.com/ubuntu-ports \$(lsb_release -sc)-updates main\"' >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} 'add-apt-repository \"deb http://ports.ubuntu.com/ubuntu-ports \$(lsb_release -sc) universe\"' >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} 'add-apt-repository \"deb http://ports.ubuntu.com/ubuntu-ports \$(lsb_release -sc)-updates universe\"' >/dev/null 2>&1");
|
|
}
|
|
|
|
system("xdsh $confkeys{$mn} \"wget -O - \"http://sourceforge.net/projects/xcat/files/ubuntu/apt.key/download\" | apt-key add -\" >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} 'apt-get clean all' >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} 'apt-get update' >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} 'apt-get -y install build-essential dpkg-dev dh-make debhelper fakeroot gnupg lintian pbuilder quilt reprepro libsoap-lite-perl libdbi-perl' >/dev/null 2>&1");
|
|
}
|
|
|
|
#build xcat core ball
|
|
if($mn =~ /ubuntux/){
|
|
$res = system("xdsh $confkeys{$mn} '/xcatbuild/xcat-core/build-ubunturepo -c UP=0 BUILDALL=1 > /var/log/xcat/build_xcat.log 2>&1' >/dev/null 2>&1");
|
|
}else{
|
|
$res = system("xdsh $confkeys{$mn} '/xcat-core/buildlocal.sh CURDIR=/xcat-core > /var/log/xcat/build_xcat.log 2>&1' >/dev/null 2>&1");
|
|
}
|
|
if ($res != 0){
|
|
send_msg(0, "[$mn->$confkeys{$mn}][build_xcat] build xcat in $mn failed");
|
|
return 1;
|
|
}
|
|
|
|
send_msg(2, "[$mn->$confkeys{$mn}][build_xcat] build xcat in $mn successfully");
|
|
return 0;
|
|
}
|
|
|
|
#######################################
|
|
# config mn
|
|
#######################################
|
|
sub install_xcat {
|
|
my $mn = shift;
|
|
send_msg(2, "[$mn->$confkeys{$mn}][install_xcat] starting to install xcat in $confkeys{$mn}");
|
|
|
|
my $os = $confkeys{$mn."os"};
|
|
my $arch = "ppc64";
|
|
$arch="x86_64" if($mn !~ /aix/ && $mn =~ /x/);
|
|
|
|
if($mn =~ /rh/){
|
|
$os =~ /(\D+)(\d+)\.?(\d?)/;
|
|
system("xdsh $confkeys{$mn} \"cd /xcat-dep/rh$2/$arch && ./mklocalrepo.sh\" >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} \"rpm --import /RPM-GPG-KEY-redhat-release\" >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} \"yum -y install xCAT xCAT-test > /var/log/xcat/install_xcat.log 2>&1\" >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} \"yum -y install mysql-server mysql mysql-bench mysql-devel mysql-connector-odbc\" >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} \"source /etc/profile.d/xcat.sh\" >/dev/null 2>&1");
|
|
if($mn =~ /rhx/){
|
|
system("xdsh $confkeys{$mn} \"yum install -y perl-Sys-Virt\" >/dev/null 2>&1");
|
|
}
|
|
}elsif($mn =~ /sles/){
|
|
$os =~ /(\D+)(\d+)\.?(\d?)/;
|
|
system("xdsh $confkeys{$mn} \"zypper ar file:///xcat-dep/sles$2/$arch xCAT-dep\" >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} \"zypper sl -U\" >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} \"zypper --gpg-auto-import-keys search --match-exact -s screen\" >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} \"zypper -n install xCAT xCAT-test > /var/log/xcat/install_xcat.log 2>&1\" >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} \"zypper -n install createrepo\" >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} \"zypper -n install mysql-client libmysqlclient_r15 libqt4-sql-mysql libmysqlclient15 perl-DBD-mysql mysql unixODBC\" >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} \"source /etc/profile.d/xcat.sh\" >/dev/null 2>&1");
|
|
if($mn =~ /slesx/){
|
|
system("xdsh $confkeys{$mn} \"zypper -n install perl-Sys-Virt\" >/dev/null 2>&1");
|
|
}
|
|
}elsif($mn =~ /ubuntux/){
|
|
system("xdsh $confkeys{$mn} \"/xcat-core/mklocalrepo.sh\" >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} \"/xcat-dep/mklocalrepo.sh\" >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} \"wget -O - \"http://sourceforge.net/projects/xcat/files/ubuntu/apt.key/download\" | apt-key add -\" >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} 'apt-get clean all' >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} 'apt-get update' >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} 'apt-get -y install xcat xCAT-test > /var/log/xcat/install_xcat.log 2>&1' >/dev/null 2>&1");
|
|
}
|
|
|
|
#check if MN is installed successful
|
|
my $lsdefout = `ssh -t $confkeys{$mn} 'bash -l -i -c "lsdef -v"'`;
|
|
chomp($lsdefout);
|
|
if($lsdefout !~ /^lsdef - Version/ ){
|
|
send_msg(0, "[$mn->$confkeys{$mn}][install_xcat] install xcat in $mn failed");
|
|
return 1;
|
|
}
|
|
|
|
send_msg(2, "[$mn->$confkeys{$mn}][install_xcat] install xcat in $mn successfully");
|
|
return 0;
|
|
}
|
|
|
|
#######################################
|
|
# do test
|
|
#######################################
|
|
sub do_test {
|
|
my $mn = shift;
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] starting to run regression test in $confkeys{$mn}");
|
|
|
|
my $regconf=$mn.".conf";
|
|
my $res = system("scp $configfiledir/$regconf root\@$confkeys{$mn}:/opt/xcat/share/xcat/tools/autotest/default.conf >/dev/null");
|
|
if ($res != 0){
|
|
send_msg(0, "[$mn->$confkeys{$mn}][do_test] copy $configfiledir/$regconf to $mn failed");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] copy $configfiledir/$regconf to $mn successfully");
|
|
|
|
my $diskfull_installation_flat_testcase="";
|
|
my $diskless_installation_flat_testcase="";
|
|
my $statelite_installation_flat_testcase="";
|
|
my $sn_installation_hierarchy_testcase="";
|
|
my $diskfull_installation_hierarchy_testcase="";
|
|
my $diskless_installation_hierarchy_testcase="";
|
|
my $statelite_installation_hierarchy_testcase="";
|
|
my $batbundle="";
|
|
if($mn =~ /sles/ || $mn =~ /rh/){
|
|
$diskfull_installation_flat_testcase="reg_linux_diskfull_installation_flat";
|
|
$diskless_installation_flat_testcase="reg_linux_diskless_installation_flat";
|
|
$statelite_installation_flat_testcase="reg_linux_statelite_installation_flat";
|
|
$sn_installation_hierarchy_testcase="reg_linux_SN_installation_hierarchy";
|
|
$diskfull_installation_hierarchy_testcase="reg_linux_diskfull_installation_hierarchy";
|
|
$diskless_installation_hierarchy_testcase="reg_linux_diskless_installation_hierarchy";
|
|
$statelite_installation_hierarchy_testcase="reg_linux_statelite_installation_hierarchy";
|
|
$batbundle="p_rhels_cmd.bundle" if($mn =~ /rhp/);
|
|
$batbundle="x_rhels_cmd.bundle" if($mn =~ /rhx/);
|
|
$batbundle="p_sles_cmd.bundle" if($mn =~ /slesp/);
|
|
$batbundle="x_sles_cmd.bundle" if($mn =~ /slesx/);
|
|
}elsif($mn =~ /ubuntux/){
|
|
$diskfull_installation_flat_testcase="Ubuntu_full_installation_flat_x86_vm";
|
|
$diskless_installation_flat_testcase="Ubuntu_diskless_installation_flat_x86_vm";
|
|
$batbundle="x_ubuntu_cmd.bundle";
|
|
}elsif($mn =~ /ubuntupel/){
|
|
$diskfull_installation_flat_testcase="Ubuntu_full_installation_flat_p8le_vm";
|
|
$diskless_installation_flat_testcase="Ubuntu_diskless_installation_flat_p8le_vm";
|
|
$batbundle="p_ubunut.cmd.bundle";
|
|
}
|
|
|
|
if($mn !~ /ubuntu/){
|
|
#diskfull installation in flat mode test
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] doing diskfull installation test in flat mode in $mn.....");
|
|
system("xdsh $confkeys{$mn} \"xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf -t $diskfull_installation_flat_testcase\" >/dev/null 2>&1");
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] do diskfull installation test in flat mode in $mn.....[done]");
|
|
|
|
#commands bundle in flat mode test
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] doing all commands test in flat mode in $mn.....");
|
|
system("xdsh $confkeys{$mn} \"xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf -b $batbundle\" >/dev/null 2>&1");
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] do all commands test in flat mode in $mn.....[done]");
|
|
|
|
#diskless installation in flat mode test
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] doing diskless installation test in flat mode in $mn.....");
|
|
system("xdsh $confkeys{$mn} \"xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf -t $diskless_installation_flat_testcase\" >/dev/null 2>&1");
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] do diskless installation test in flat mode in $mn.....[done]");
|
|
|
|
#statelite installation in flat mode test
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] doing statelite installation test in flat mode in $mn.....");
|
|
system("xdsh $confkeys{$mn} \"xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf -t $statelite_installation_flat_testcase\" >/dev/null 2>&1");
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] do statelite installation test in flat mode in $mn.....[done]");
|
|
|
|
#deploy SN
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] setting up mysql for installing SN in $mn.....");
|
|
$res = system("xdsh $confkeys{$mn} \"XCATMYSQLADMIN_PW=12345 XCATMYSQLROOT_PW=12345 mysqlsetup -i\" >/dev/null 2>&1");
|
|
if ($res != 0){
|
|
send_msg(0, "[$mn->$confkeys{$mn}][do_test] set up mysql for installing SN in $mn failed");
|
|
return 1;
|
|
}
|
|
$res = system ("xdsh $confkeys{$mn} \"echo \\\"GRANT ALL on xcatdb.* TO xcatadmin@\'%\' IDENTIFIED BY \'12345\'\;\\\" | mysql -u root -p12345\" >/dev/null 2>&1");
|
|
if ($res != 0){
|
|
send_msg(0, "[$mn->$confkeys{$mn}][do_test] grant right to mysql for installing SN in $mn failed");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] set up mysql for installing SN in $mn.....[done]");
|
|
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] preparing xcat ball for installing SN in $mn.....");
|
|
my $os = $confkeys{$mn."os"};
|
|
my $arch = "ppc64";
|
|
$arch="x86_64" if($mn !~ /aix/ && $mn =~ /x/);
|
|
my $xcatballpath = "/install/post/otherpkgs/$os/$arch/xcat";
|
|
$res = system("xdsh $confkeys{$mn} \"mkdir -p $xcatballpath/xcat-core && cp -r /xcat-core/build/* $xcatballpath/xcat-core && cp -r /xcat-dep $xcatballpath\"");
|
|
if ($res != 0){
|
|
send_msg(0, "[$mn->$confkeys{$mn}][do_test] prepare xcat ball for installing SN in $mn failed");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] prepare xcat ball for installing SN in $mn.....[done]");
|
|
|
|
if($mn =~ /rhp/){
|
|
my $ping_sn_on_rhp_pid = fork();
|
|
if( !defined($ping_sn_on_rhp_pid)){
|
|
send_msg(0, "fork process for trigger random number generator on redhat+ppc environment to install SN error");
|
|
exit;
|
|
}elsif($ping_sn_on_rhp_pid ==0){
|
|
send_msg(2, "....fork 1 process[pid=$$] for trigger random number generator on redhat+ppc environment to install SN..........");
|
|
my $snnodename = `cat $configfiledir/$regconf |grep SN|grep =|awk -F'=' '{print \$2}'`;
|
|
chomp($snnodename);
|
|
&runcmd("sleep 900");
|
|
my $timecnt=0;
|
|
while($timecnt < 180){
|
|
my $booted = system("xdsh $confkeys{$mn} \"lsdef -l $snnodename|grep status|grep booted\"");
|
|
last if($booted==0);
|
|
|
|
my $booting = system("xdsh $confkeys{$mn} \"lsdef -l $snnodename|grep status|grep booting\"");
|
|
system("xdsh $confkeys{$mn} \"xdsh $snnodename \"echo 1\"\"") if($booting==0);
|
|
|
|
&runcmd("sleep 10");
|
|
$timecnt++;
|
|
}
|
|
send_msg(2, "....process[pid=$$] for trigger random number generator on redhat+ppc environment to install SN....[exit]");
|
|
exit 0;
|
|
}
|
|
}
|
|
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] installing SN.....");
|
|
system("xdsh $confkeys{$mn} \"xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf -t $sn_installation_hierarchy_testcase\" >/dev/null 2>&1");
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] install SN.....[done]");
|
|
|
|
#diskfull installation in hierarchy mode test
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] doing diskfull installation test in hierarchy mode in $mn.....");
|
|
system("xdsh $confkeys{$mn} \"xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf -t $diskfull_installation_hierarchy_testcase\" >/dev/null 2>&1");
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] do diskfull installation test in hierarchy mode in $mn.....[done]");
|
|
|
|
#diskless installation in hierarchy mode test
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] doing diskless installation test in hierarchy mode in $mn.....");
|
|
system("xdsh $confkeys{$mn} \"xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf -t $diskless_installation_hierarchy_testcase\" >/dev/null 2>&1");
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] do diskless installation test in hierarchy mode in $mn.....[done]");
|
|
|
|
#statelite installation in hierarchy mode test
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] doing statelite installation test in hierarchy mode in $mn.....");
|
|
system("xdsh $confkeys{$mn} \"xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf -t $statelite_installation_hierarchy_testcase\" >/dev/null 2>&1");
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] do statelite installation test in hierarchy mode in $mn.....[done]");
|
|
|
|
}else{
|
|
#diskfull installation in flat mode test
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] doing diskfull installation test in flat mode in $mn.....");
|
|
system("ssh -t $confkeys{$mn} 'exec bash -l -i -c \"xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf -t $diskfull_installation_flat_testcase\"' >/dev/null 2>&1");
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] do diskfull installation test in flat mode in $mn.....[done]");
|
|
|
|
#commands bundle in flat mode test
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] doing all commands test in flat mode in $mn.....");
|
|
system("ssh -t $confkeys{$mn} 'exec bash -l -i -c \"xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf -b $batbundle\"' >/dev/null 2>&1");
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] do all commands test in flat mode in $mn.....[done]");
|
|
|
|
#diskless installation in flat mode test
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] doing diskless installation test in flat mode in $mn.....");
|
|
system("ssh -t $confkeys{$mn} 'exec bash -l -i -c \"xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf -t $diskless_installation_flat_testcase\"' >/dev/null 2>&1");
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] do diskless installation test in flat mode in $mn.....[done]");
|
|
}
|
|
|
|
send_msg(2, "[$mn->$confkeys{$mn}][install_xcat] run regression test in $mn finished");
|
|
|
|
$res = system("scp -r $confkeys{$mn}:/opt/xcat/share/xcat/tools/autotest/result/* $todayregresultdir/$mn/ >/dev/null 2>&1");
|
|
if ($res != 0){
|
|
send_msg(0, "[$mn->$confkeys{$mn}][do_test] copy regression result to $todayregresultdir/$mn/ failed");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] copy regression result to $todayregresultdir/$mn/ successfully");
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
#######################################
|
|
# creat report
|
|
#######################################
|
|
sub creat_report{
|
|
|
|
my $mailreport .= "======================================\n";
|
|
$mailreport .= " Regression Result Summary\n";
|
|
$mailreport .= "======================================\n\n";
|
|
|
|
# $gitupdateflag = 0 git update out of time
|
|
# $gitupdateflag = 1 there isn't new code checkin in last one day
|
|
# $gitupdateflag = 2 git update failed
|
|
if($gitupdateflag==0){
|
|
$mailreport .= "Get the latest xcat code from git out of time, stop regression test today!!!\n\n\n";
|
|
&runcmd("touch $mailfile && echo \"$mailreport\" > $mailfile");
|
|
&runcmd("mv $commitinfofile $commitinfofile.old && echo \"\" > $commitinfofile");
|
|
return 0;
|
|
}elsif($gitupdateflag==1 && ! $forceregwithoutupdate){
|
|
$mailreport .= "There isn't new code checkin last one day, stop regression test today!!!\n\n\n";
|
|
&runcmd("touch $mailfile && echo \"$mailreport\" > $mailfile");
|
|
&runcmd("mv $commitinfofile $commitinfofile.old && echo \"\" > $commitinfofile");
|
|
return 0;
|
|
}elsif($gitupdateflag==1 && $forceregwithoutupdate){
|
|
$mailreport .= "There isn't new code checkin last one day, But force to run regresson today!!!\n\n\n";
|
|
&runcmd("mv $commitinfofile $commitinfofile.old && echo \"\" > $commitinfofile");
|
|
}elsif($gitupdateflag==2){
|
|
$mailreport .= "Get the latest xcat source code from git error, stop regression test today!!!\n\n\n";
|
|
&runcmd("touch $mailfile && echo \"$mailreport\" > $mailfile");
|
|
&runcmd("mv $commitinfofile $commitinfofile.old && echo \"\" > $commitinfofile");
|
|
return 0;
|
|
}
|
|
|
|
while(my($k,$v) = each %pidrecord){
|
|
my $scenario="[".$confkeys{$k."os"};
|
|
if($k=~/p/){$scenario.="+PPC64] ";}else{$scenario.="+x86_64] ";}
|
|
$mailreport .= "$scenario\n";
|
|
|
|
if ($v == 0){ # whole regression test finished
|
|
my $h=int($consumptionrecord{$k}/3600);
|
|
my $m=int(($consumptionrecord{$k} - $h*3600)/60);
|
|
my $s=($consumptionrecord{$k} - $h*3600)%60;
|
|
my $consumption="$h hours $m minutes $s seconds";
|
|
$mailreport .= "\tTime consumption: $consumption\n";
|
|
|
|
if($piderrrecord{$k} ne ""){
|
|
$mailreport .= "\tDeploy MN result: $piderrrecord{$k}\n";
|
|
next;
|
|
}
|
|
|
|
my $totalcnt=0;
|
|
my $failcnt=0;
|
|
my $faillist="";
|
|
opendir(DIR, "$todayregresultdir/$k");
|
|
foreach my $file (readdir DIR)
|
|
{
|
|
next if($file !~ /xcattest.log/);
|
|
my $cnt = `cat $todayregresultdir/$k/$file |grep -- "------END::"|wc -l`;
|
|
$totalcnt+=int($cnt);
|
|
for(my $i=1;$i<$cnt+1;$i++){
|
|
my $line=`cat $todayregresultdir/$k/$file |grep -- "------END::"|sed -n ${i}p`;
|
|
chomp($line);
|
|
if($line =~ /------END::([a-zA-Z0-9_-]+)::([a-zA-Z0-9_-]+)::Time.+/){
|
|
my $failedcase=$1;
|
|
if($2 =~ /Failed/){
|
|
$failcnt++;
|
|
$faillist.=$failedcase.", ";
|
|
}
|
|
}
|
|
}
|
|
}
|
|
closedir(DIR);
|
|
|
|
$mailreport .= "\tTotal $totalcnt Failed $failcnt\n\tFailed cases: $faillist\n";
|
|
}else{ # regression test out of time
|
|
$mailreport .= "\tTime consumption: Out of 8 hours!!!!!!\n";
|
|
}
|
|
}
|
|
|
|
&runcmd("touch $mailfile && echo \"$mailreport\" > $mailfile");
|
|
return 0;
|
|
}
|
|
|
|
#######################################
|
|
# send email
|
|
#######################################
|
|
sub send_email{
|
|
my $checkinlist="";
|
|
my $cnt =`cat $commitinfofile |grep '20[0-9][0-9] [-+]0[0-9]00'|awk '{print \$1}' |grep -v root|sort | uniq|wc -l`;
|
|
my $out =`cat $commitinfofile |grep '20[0-9][0-9] [-+]0[0-9]00'|awk '{print \$1}' |grep -v root|sort | uniq`;
|
|
for(my $i=1;$i<$cnt+1;$i++){
|
|
my $tmpline=`echo "$out"|sed -n ${i}p`;
|
|
chomp($tmpline);
|
|
$checkinlist.= "$tmpline ";
|
|
}
|
|
|
|
my $totalcnt=0;
|
|
my $failcnt=0;
|
|
$cnt =`cat $mailfile |grep "Total"|grep "Failed"|wc -l`;
|
|
$out =`cat $mailfile |grep "Total"|grep "Failed"`;
|
|
for(my $i=1;$i<$cnt+1;$i++){
|
|
my $tmpline=`echo "$out"|sed -n ${i}p`;
|
|
chomp($tmpline);
|
|
if($tmpline =~ /Total (\d+) Failed (\d+)/){
|
|
$totalcnt+=$1;
|
|
$failcnt+=$2;
|
|
}
|
|
}
|
|
|
|
# $gitupdateflag = 0 git update out of time
|
|
# $gitupdateflag = 1 there isn't new code checkin in last one day
|
|
# $gitupdateflag = 2 git update failed
|
|
my $subject="";
|
|
if($gitupdateflag==0 || $gitupdateflag==2){
|
|
$subject = "[$mailtitle] Git update failed, stop regression test today!!!";
|
|
}elsif($gitupdateflag==1 && ! $forceregwithoutupdate){
|
|
$subject = "[$mailtitle] No code checkin, stop regression test today!!!";
|
|
}elsif($gitupdateflag==1 && $forceregwithoutupdate){
|
|
$subject = "[$mailtitle] Total:".$totalcnt.", Fail:".$failcnt.", Checkin: NO ONE!!";
|
|
}else{
|
|
$subject = "[$mailtitle] Total:".$totalcnt.", Fail:".$failcnt.", Checkin: ".$checkinlist;
|
|
}
|
|
|
|
my $mailreport .= "\n\n======================================\n";
|
|
$mailreport .= " Commit Codes Last Day\n";
|
|
$mailreport .= "======================================\n\n";
|
|
|
|
&runcmd("echo \"$mailreport\">> $mailfile");
|
|
&runcmd("cat $commitinfofile >> $mailfile");
|
|
|
|
my $attachfile="/tmp/failed_case_detail.txt";
|
|
&runcmd("echo \"\" > $attachfile");
|
|
if($failcnt > 0){
|
|
opendir(DIR, "$todayregresultdir");
|
|
foreach my $subdir (readdir DIR){
|
|
next if($subdir =~ /^\./);
|
|
#print "subdir-> $subdir\n";
|
|
|
|
my $scenario="[".$confkeys{$subdir."os"};
|
|
if($subdir=~/p/){$scenario.="+PPC64] ";}else{$scenario.="+x86_64] ";}
|
|
my $separator .= "\n###############################################################\n";
|
|
$separator .= " $scenario\n";
|
|
$separator .= "###############################################################\n\n";
|
|
&runcmd("echo \"$separator\" >> $attachfile");
|
|
opendir(SUBDIR, "$todayregresultdir/$subdir");
|
|
foreach my $file (readdir SUBDIR){
|
|
next if($file !~ /^failedcases/);
|
|
next if(-z "$todayregresultdir/$subdir/$file");
|
|
#print "\t$file\n";
|
|
system("cat $todayregresultdir/$subdir/$file >> $attachfile");
|
|
}
|
|
}
|
|
}
|
|
|
|
if($mail_list){
|
|
if($failcnt > 0){
|
|
system("cat $mailfile"." | /bin/mail -s \"$subject\" -a $attachfile \"$mail_list\"");
|
|
}else{
|
|
system("cat $mailfile"." | /bin/mail -s \"$subject\" \"$mail_list\"");
|
|
}
|
|
}else{
|
|
if($failcnt > 0){
|
|
system("cat $mailfile"." | /bin/mail -s \"$subject\" -a $attachfile \"$confkeys{mailgroup}\"");
|
|
}else{
|
|
system("cat $mailfile"." | /bin/mail -s \"$subject\" \"$confkeys{mailgroup}\"");
|
|
}
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
|
|
#######################################
|
|
#git update
|
|
#return value:
|
|
#0 successful
|
|
#1 code is already at latest version
|
|
#2 someting error
|
|
#######################################
|
|
sub git_update{
|
|
send_msg(2, "[git update] starting to update xcat source code from git");
|
|
|
|
my $res = system("cd $xcatcoredir && git checkout $branch > /dev/nul 2>&1");
|
|
if ($res != 0){
|
|
send_msg(0, "[git update] change to branch $branch....[failed]");
|
|
return 2;
|
|
}
|
|
send_msg(2, "[git update] change to branch $branch....[ok]");
|
|
|
|
my $gitpulloutput = "/tmp/gitpulloutput";
|
|
my $trytime=1;
|
|
while($trytime < 4){
|
|
$res = system("cd $xcatcoredir && git pull > $gitpulloutput 2>&1");
|
|
if ($res != 0){
|
|
send_msg(0, "[git update] pull the latest code to control node....[$trytime failed]");
|
|
$trytime++;
|
|
&runcmd("sleep 120");
|
|
}else{
|
|
last;
|
|
}
|
|
}
|
|
|
|
if($trytime == 4){
|
|
send_msg(0, "[git update] pull the latest code to control node....[finally failed]");
|
|
return 2;
|
|
}
|
|
|
|
$res = system("grep 'Already up-to-date' $gitpulloutput > /dev/nul 2>&1");
|
|
if ($res == 0 && ! $forceregwithoutupdate) {
|
|
send_msg(2, "[git update] code is already at latest version. exit regresson");
|
|
return 1;
|
|
}elsif($res == 0 && $forceregwithoutupdate){
|
|
send_msg(2, "[git update] code is already at latest version. But force to run regresson");
|
|
return 0;
|
|
}
|
|
send_msg(2, "[git update] pull the latest code to control node....[ok]");
|
|
|
|
my $lastcommitfile="/tmp/lastcommitfile";
|
|
my $lastcommitinfo;
|
|
if (-f $lastcommitfile){
|
|
$lastcommitinfo = `cat $lastcommitfile`;
|
|
chomp($lastcommitinfo);
|
|
}else{
|
|
$lastcommitinfo = "";
|
|
}
|
|
#print "lastcommitinfo=$lastcommitinfo\n";
|
|
|
|
if($lastcommitinfo eq ""){
|
|
$res = system("cd $xcatcoredir && git log --since=\"1 days ago\" --pretty=format:\"%cn %ad %s\" --name-status > $commitinfofile");
|
|
if ($res != 0 || (-f $commitinfofile and -z _)){
|
|
send_msg(0, "[git update] get commit information....[failed]");
|
|
return 2;
|
|
}
|
|
send_msg(0, "[git update] get commit information....[ok]");
|
|
}else{
|
|
$res = system("cd $xcatcoredir && git log $lastcommitinfo..HEAD --pretty=format:\"%cn %ad %s\" --name-status > $commitinfofile");
|
|
if ($res != 0 || (-f $commitinfofile and -z _)){
|
|
send_msg(0, "[git update] get commit information....[failed]");
|
|
return 2;
|
|
}
|
|
send_msg(2, "[git update] get commit information....[ok]");
|
|
}
|
|
|
|
$res = system("cd $xcatcoredir && git log -n 1|awk 'NR==1'|awk '{print \$2}' > $lastcommitfile");
|
|
if ($res != 0){
|
|
send_msg(0, "[git update] get latest commit number failed");
|
|
exit;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
xCATreg_init();
|
|
|
|
###############################################################
|
|
# Mainfunction
|
|
###############################################################
|
|
send_msg(2,"........................");
|
|
send_msg(2,"........................");
|
|
send_msg(2,".....ooooO..............");
|
|
send_msg(2,"....(....)....Ooooo.....");
|
|
send_msg(2,".....\\..(.....(....)....");
|
|
send_msg(2,"......\\__).....)../.....");
|
|
send_msg(2,"..............(_ /......");
|
|
send_msg(2,"........................");
|
|
send_msg(2,".........START .........");
|
|
send_msg(2,"........................");
|
|
|
|
#######################################
|
|
# step 0. Parse input arguments
|
|
#######################################
|
|
send_msg(2, "step 1, Parse input arguments............");
|
|
=pod
|
|
if (
|
|
!GetOptions("h|?" => \$needhelp,
|
|
"f=s" => \$configfile,
|
|
"b=s" => \$branch,
|
|
"m=s" => \$management_node,
|
|
"V" => \$verbose_mode,
|
|
"U" => \$updates_regression,
|
|
"e=s" => \$send_msg)
|
|
)
|
|
=cut
|
|
if (
|
|
!GetOptions("h|?" => \$needhelp,
|
|
"f=s" => \$configfile,
|
|
"b=s" => \$branch,
|
|
"m=s" => \$management_node,
|
|
"a" => \$forceregwithoutupdate,
|
|
"e=s" => \$mail_list)
|
|
)
|
|
{
|
|
&usage;
|
|
send_msg(0, "step 0, PARSE ARGUMENTS returns error, exit");
|
|
exit;
|
|
}
|
|
if ($needhelp)
|
|
{
|
|
&usage;
|
|
exit 0;
|
|
}
|
|
|
|
#######################################
|
|
# step 1. Read configuration files
|
|
#######################################
|
|
send_msg(2, "step 2, Reading xCATreg conf............");
|
|
my $rst = read_conf();
|
|
unless (! $rst) {
|
|
send_msg(0, "read xCATreg configuration failed...........exit");
|
|
exit;
|
|
}
|
|
|
|
foreach my $k (keys %confkeys) {
|
|
print "$k = $confkeys{$k}\n";
|
|
}
|
|
|
|
# begin to find which mn should be used
|
|
if ($management_node) {
|
|
my @all;
|
|
if ($management_node =~ /,/) {
|
|
@all = split /,/, $management_node;
|
|
} else {
|
|
push @all, $management_node;
|
|
}
|
|
foreach my $mn (@all) {
|
|
foreach my $k (keys %confkeys) {
|
|
if ($confkeys{$k} eq $mn) {
|
|
$mns{$k} = $mn;
|
|
send_msg(2, "specify mn :$mn");
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
pipe PREAD,CWRITE;
|
|
my $gitpid = fork();
|
|
if( !defined($gitpid)){
|
|
send_msg(0, "fork process for git update error");
|
|
exit;
|
|
}elsif($gitpid ==0){
|
|
close PREAD;
|
|
send_msg(2, "..........fork 1 process[pid=$$] for git update..........");
|
|
my $res = git_update();
|
|
if($res > 1){
|
|
send_msg(0, "git_update......[failed]");
|
|
syswrite CWRITE,"git update failed\n";
|
|
close CWRITE;
|
|
exit 1;
|
|
}
|
|
if($res == 1){
|
|
send_msg(2, "git_update......[not find new code]");
|
|
syswrite CWRITE,"git update got nothing\n";
|
|
close CWRITE;
|
|
exit 1;
|
|
}
|
|
send_msg(2, "git_update......[ok]");
|
|
syswrite CWRITE,"git update successful\n";
|
|
close CWRITE;
|
|
exit 0;
|
|
}
|
|
|
|
{#main process
|
|
close CWRITE;
|
|
my $gitstarttime = time();
|
|
my $gitprocess=1;
|
|
my $gitselect = new IO::Select;
|
|
$gitselect->add(\*PREAD);
|
|
while($gitprocess){
|
|
my @hdls;
|
|
if (@hdls = $gitselect->can_read(0)) {
|
|
my $hdl;
|
|
foreach $hdl (@hdls) {
|
|
if ($hdl == \*PREAD) {
|
|
my $line;
|
|
chomp($line=<PREAD>);
|
|
if ($line){
|
|
if($line =~ /successful/){
|
|
send_msg(2, "[[main]]: $line");
|
|
$gitupdateflag=3;
|
|
}
|
|
if($line =~ /failed/){
|
|
send_msg(0, "[[main]]: $line");
|
|
$gitupdateflag=2;
|
|
}
|
|
if($line =~ /nothing/){
|
|
send_msg(2, "[[main]]: there isn't new code checkin during last one day.");
|
|
$gitupdateflag=1;
|
|
}
|
|
$gitprocess = 0;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if(time() - $gitstarttime > 7200) {
|
|
#if(time() - $gitstarttime > 2) {
|
|
send_msg(0, "[[main]] 2 hours is expired. git update out of time");
|
|
last;
|
|
}
|
|
&runcmd("sleep 2");
|
|
}
|
|
close PREAD;
|
|
|
|
&runcmd("kill $gitpid") if($gitprocess==1);
|
|
send_msg(2, "[[main]]........git process[$gitpid] exit successful..............");
|
|
|
|
# $gitupdateflag = 0 git update out of time
|
|
# $gitupdateflag = 1 there isn't new code checkin in last one day
|
|
# $gitupdateflag = 2 there is something wrong during git update
|
|
if($gitupdateflag<3){
|
|
send_msg(2, "[[main]]: creat regression result report...........");
|
|
$rst = creat_report();
|
|
if ($rst) {
|
|
send_msg(0, "[[main]]: creat regression result report...........[FAILED]");
|
|
}else{
|
|
send_msg(2, "[[main]]: creat regression result report...........[OK]");
|
|
|
|
send_msg(2, "[[main]]: send report email...........");
|
|
$rst = send_email();
|
|
if ($rst) {
|
|
send_msg(0, "[[main]]: send report email...........[FAILED]");
|
|
}else{
|
|
send_msg(2, "[[main]]: send report email...........[OK]");
|
|
}
|
|
}
|
|
exit 1;
|
|
}
|
|
send_msg(2, "[[main]]: git update successful, start regression today........");
|
|
|
|
#exit 0
|
|
}
|
|
|
|
|
|
my $platformfork = 0;
|
|
my $timingork = 0;
|
|
# begin child process
|
|
pipe CONTROLREAD,MNWRITE;
|
|
foreach my $m (keys %mns) {
|
|
$platformfork += 1;
|
|
my $mn = $mns{$m};
|
|
my $pid = fork();
|
|
if ( !defined($pid) ) {
|
|
send_msg(0, "fork process for $m error");
|
|
exit;
|
|
} elsif ( $pid == 0 ) { # child process
|
|
send_msg(2, "..........fork process[pid=$$] for $m [$confkeys{$m}]..........");
|
|
close CONTROLREAD;
|
|
my $res;
|
|
|
|
#######################################
|
|
# step 3. Install MNs,
|
|
#######################################
|
|
send_msg(2, "[$m]step 3, Running mn_install...............");
|
|
$res = mn_install($m);
|
|
if ($res) {
|
|
syswrite MNWRITE,"REPORTFROM:$m->$mn: install $mn failed\n";
|
|
exit 1;
|
|
}
|
|
send_msg(2, "[$m]step 3, Run mn_install...............[OK]");
|
|
|
|
#######################################
|
|
# step 4. Copy code to MNs
|
|
#######################################
|
|
send_msg(2, "[$m]step 4, Running copy_code...............");
|
|
$res = copy_code($m);
|
|
if ($res) {
|
|
syswrite MNWRITE,"REPORTFROM:$m->$mn: copy xcat source code to $mn failed\n";
|
|
exit 1;
|
|
}
|
|
send_msg(2, "[$m]step 4, Run copy_code...............[OK]");
|
|
|
|
#######################################
|
|
# step 5. Build xcat code in MNs
|
|
#######################################
|
|
send_msg(2, "[$m]step 5, Running build_xcat...............");
|
|
$res = build_xcat($m);
|
|
if ($res) {
|
|
syswrite MNWRITE,"REPORTFROM:$m->$mn: build new xcat in $mn failed\n";
|
|
exit 1;
|
|
}
|
|
send_msg(2, "[$m]step 5, Run build_xcat...............[OK]");
|
|
|
|
#######################################
|
|
# step 6. Read xCAT MN's configuration
|
|
#######################################
|
|
send_msg(2, "[$m]step 6, Running install_xcat...............");
|
|
$res = install_xcat($m);
|
|
if ($res) {
|
|
syswrite MNWRITE,"REPORTFROM:$m->$mn: install new xcat in $mn failed\n";
|
|
exit 1;
|
|
}
|
|
send_msg(2, "[$m]step 6, Run install_xcat...............[OK]");
|
|
|
|
#######################################
|
|
# step 7. Genrate local configuration file for xcattest
|
|
# Do test
|
|
# Write log
|
|
#######################################
|
|
send_msg(2, "[$m]step 7, Running do_test...............");
|
|
$res = do_test($m);
|
|
if ($res) {
|
|
syswrite MNWRITE,"REPORTFROM:$m->$mn: run regression test cases in $mn failed\n";
|
|
exit 1;
|
|
}
|
|
send_msg(2, "[$m]step 7, Running do_test...............[OK]");
|
|
|
|
send_msg(2, "whole regression test for $mn finished");
|
|
syswrite MNWRITE,"REPORTFROM:$m->$mn: whole regression test are successful\n";
|
|
exit 0;
|
|
} # end of child process
|
|
&runcmd("mkdir -p $todayregresultdir/$m");
|
|
$pidrecord{$m} = $pid;
|
|
$piderrrecord{$m} = "";
|
|
} # end of foreach
|
|
|
|
|
|
{ # parent process
|
|
close MNWRITE;
|
|
my $rst;
|
|
my $regstarttime = time();
|
|
|
|
my $select = new IO::Select;
|
|
$select->add(\*CONTROLREAD);
|
|
while ($platformfork) {
|
|
my @hdls;
|
|
if (@hdls = $select->can_read(0)) {
|
|
my $hdl;
|
|
foreach $hdl (@hdls) {
|
|
if ($hdl == \*CONTROLREAD) {
|
|
my $line;
|
|
chomp($line=<CONTROLREAD>);
|
|
if ($line){
|
|
if($line =~ /successful/){
|
|
send_msg(2, "[[main]]: $line");
|
|
$line =~ s/^REPORTFROM:(.+)->.+/$1/g;
|
|
$pidrecord{$line} = 0;
|
|
$consumptionrecord{$line} = time() - $regstarttime;
|
|
}
|
|
if($line =~ /failed/){
|
|
send_msg(0, "[[main]]: $line");
|
|
my $tmp = $line;
|
|
$line =~ s/^REPORTFROM:(.+)->.+/$1/g;
|
|
$pidrecord{$line} = 0;
|
|
$consumptionrecord{$line} = time() - $regstarttime;
|
|
$tmp =~ s/REPORTFROM:.+->.+:(.+)/$1/g;
|
|
$piderrrecord{$line} = $tmp;
|
|
}
|
|
$platformfork -= 1;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if(time() - $regstarttime > 28800) {
|
|
#if($curtime - $regstarttime > 10) {
|
|
send_msg(1, "[timing] 8 hours is expired");
|
|
last;
|
|
}
|
|
&runcmd("sleep 1");
|
|
}
|
|
|
|
close CONTROLREAD;
|
|
if($platformfork){
|
|
send_msg(0, "[[main]]: there are $platformfork platform's regression test return out of time");
|
|
while(my($k,$v) = each %pidrecord){
|
|
&runcmd("kill $v") if ($v != 0);
|
|
}
|
|
}else{
|
|
send_msg(2, "[[main]]: all platform's regression test return on time");
|
|
}
|
|
|
|
send_msg(2, "[[main]]: creat regression result report...........");
|
|
$rst = creat_report();
|
|
if ($rst) {
|
|
send_msg(0, "[[main]]: creat regression result report...........[FAILED]");
|
|
}else{
|
|
send_msg(2, "[[main]]: creat regression result report...........[OK]");
|
|
|
|
send_msg(2, "[[main]]: send report email...........");
|
|
$rst = send_email();
|
|
if ($rst) {
|
|
send_msg(0, "[[main]]: send report email...........[FAILED]");
|
|
}else{
|
|
send_msg(2, "[[main]]: send report email...........[OK]");
|
|
}
|
|
}
|
|
send_msg(2, "[[main]]: main process exit");
|
|
exit 0;
|
|
}
|
|
|
|
|