mirror of
https://github.com/xcat2/xcat-core.git
synced 2025-05-30 09:36:41 +00:00
1600 lines
66 KiB
Perl
Executable File
1600 lines
66 KiB
Perl
Executable File
#!/usr/bin/env perl
|
|
# IBM(c) 2010 EPL license http://www.eclipse.org/legal/epl-v10.html
|
|
#
|
|
#############################################################################################################################################
|
|
#
|
|
# This script will be used to run xCAT daily regression
|
|
#
|
|
# Usage:xCATreg - Run xCAT daily regression
|
|
# xCATreg [-?|-h]
|
|
# xCATreg [-b branch][-m MN][-f configure file][-a][-V]
|
|
#
|
|
# -?/-h: help information for the xCATreg usage.
|
|
# -b: when -b is specified, the regression will be done against the specified branch. You can use 2.7, 2.8,master.The default value is master.
|
|
# -m: when -m is specified, the regression will be done the specified mn. The default value is all the mn in configuration file.
|
|
# -f :specify the configuration file,default value is /xCATreg/regresson.conf
|
|
# -a: when -a is specified, run the regression even there is no code updates
|
|
# -V: log and message in verbose mode
|
|
#
|
|
# Example usage:
|
|
# ./xCATreg Run regressoins on master branch and on all platforms in configuration file
|
|
# ./xCATreg -b 2.8 -m slesmn -f /root/regressoin.conf -a Run regression on 2.8 branch on slesmn even when there is no code updates.
|
|
#
|
|
#############################################################################################################################################
|
|
|
|
$ENV{PATH} = "/opt/xcat/bin:/opt/xcat/sbin:/opt/xcat/share/xcat/tools:/usr/sbin:/usr/bin:/sbin:/bin";
|
|
|
|
use strict;
|
|
use warnings;
|
|
use Getopt::Long;
|
|
use Data::Dumper;
|
|
use Term::ANSIColor;
|
|
use Time::Local;
|
|
use IO::Select;
|
|
|
|
BEGIN
|
|
{
|
|
$::XCATROOT = $ENV{'XCATROOT'} ? $ENV{'XCATROOT'} : -d '/opt/xcat' ? '/opt/xcat' : '/usr';
|
|
}
|
|
use lib "$::XCATROOT/lib/perl";
|
|
|
|
my $date;
|
|
my $regrootdir = "/home/autotest_code";
|
|
my $mailtitle = "PPC64LE Cluster";
|
|
|
|
#my $regrootdir="/xCATreg";
|
|
#my $mailtitle="X Cluster";
|
|
my $logfiledir = "$regrootdir/log";
|
|
my $logfile = "";
|
|
my $configfiledir = "$regrootdir/conf";
|
|
my $configfile = "$regrootdir/conf/xCATreg.conf";
|
|
my $xcatcoredowndir = "$regrootdir/xcatsource/xcat-core-build-down";
|
|
my $xcatcoredir = "$regrootdir/xcatsource/xcat-core";
|
|
my $xcatdepdir = "$regrootdir/xcatsource/xcat-dep";
|
|
my $osisodir = "$regrootdir/iso";
|
|
my $regresultdir = "$regrootdir/regresult";
|
|
my $todayregresultdir = "";
|
|
my $maildir = "$regrootdir/mail";
|
|
my $mailfile = "";
|
|
my $rootdir = "$::XCATROOT/share/xcat/tools/autotest";
|
|
my %confkeys;
|
|
my $mail_list = 0;
|
|
my $needhelp = 0;
|
|
my $branch = "master";
|
|
my $management_node = undef;
|
|
|
|
#my $verbose_mode = 0;
|
|
my $updates_regression = 0;
|
|
my %mns;
|
|
my %pidrecord;
|
|
my %piderrrecord;
|
|
my %consumptionrecord;
|
|
my $commitinfofile = "/tmp/commitinfo";
|
|
my $gitupdateflag = 0;
|
|
my $forceregwithoutupdate = 0;
|
|
my $depupdateflag = 0;
|
|
|
|
#######################################
|
|
# runcmd
|
|
#######################################
|
|
sub runcmd {
|
|
my ($cmd) = @_;
|
|
my $rc = 0;
|
|
$::RUNCMD_RC = 0;
|
|
my $outref = [];
|
|
@$outref = `$cmd 2>&1`;
|
|
if ($?)
|
|
{
|
|
$rc = $?;
|
|
$::RUNCMD_RC = $rc;
|
|
}
|
|
chomp(@$outref);
|
|
return @$outref;
|
|
}
|
|
|
|
#######################################
|
|
# clean_outdated_file
|
|
#######################################
|
|
sub is5daysago {
|
|
my $targetdate = shift;
|
|
my $targetdatey = "";
|
|
my $targetdatem = "";
|
|
my $targetdated = "";
|
|
my $datey = "";
|
|
my $datem = "";
|
|
my $dated = "";
|
|
if ($targetdate =~ /(\d{4})(\d{2})(\d{2})/) {
|
|
$targetdatey = $1;
|
|
$targetdatem = $2 - 1;
|
|
$targetdated = $3;
|
|
}
|
|
if ($date =~ /(\d{4})(\d{2})(\d{2})/) {
|
|
$datey = $1;
|
|
$datem = $2 - 1;
|
|
$dated = $3;
|
|
}
|
|
my $todays = timelocal(0, 0, 0, $dated, $datem, $datey);
|
|
my $targets = timelocal(0, 0, 0, $targetdated, $targetdatem, $targetdatey);
|
|
if ($todays - $targets > 432000) {
|
|
return 1;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
#######################################
|
|
# clean_outdated_file
|
|
#######################################
|
|
sub clean_outdated_file {
|
|
opendir(DIR, "$logfiledir");
|
|
foreach my $file (readdir DIR) {
|
|
if ($file =~ /xCATreg(\d+).log/) {
|
|
my $res = is5daysago($1);
|
|
if ($res) {
|
|
&runcmd("rm -f $logfiledir/$file");
|
|
}
|
|
}
|
|
}
|
|
closedir(DIR);
|
|
|
|
opendir(DIR, "$maildir");
|
|
foreach my $file (readdir DIR) {
|
|
if ($file =~ /mailreport.(\d+)/) {
|
|
my $res = is5daysago($1);
|
|
if ($res) {
|
|
&runcmd("rm -f $maildir/$file");
|
|
}
|
|
}
|
|
}
|
|
closedir(DIR);
|
|
|
|
opendir(DIR, "$regresultdir");
|
|
foreach my $file (readdir DIR) {
|
|
if ($file =~ /(\d+)/) {
|
|
my $res = is5daysago($1);
|
|
if ($res) {
|
|
&runcmd("rm -rf $regresultdir/$file");
|
|
}
|
|
}
|
|
}
|
|
closedir(DIR);
|
|
|
|
return 0;
|
|
}
|
|
|
|
#######################################
|
|
# xCATreg_init
|
|
#######################################
|
|
sub xCATreg_init {
|
|
|
|
#init important attributes
|
|
|
|
$date = `date +"%Y%m%d%H%M%S"`;
|
|
chomp($date);
|
|
|
|
&runcmd("mkdir -p $logfiledir") if (!-e $logfiledir);
|
|
&runcmd("mkdir -p $maildir") if (!-e $maildir);
|
|
&runcmd("mkdir -p $regresultdir") if (!-e $regresultdir);
|
|
&runcmd("mkdir -p $xcatcoredir") if (!-e $xcatcoredir);
|
|
|
|
$logfile = "xCATreg" . $date . ".log";
|
|
$todayregresultdir = $regresultdir . "/" . $date;
|
|
&runcmd("mkdir -p $todayregresultdir");
|
|
$mailfile = $maildir . "/mailreport." . $date;
|
|
|
|
my $cnt = `ps aux|grep xCATreg |grep -v grep|awk '{print \$2}'|wc -l`;
|
|
my $out = `ps aux|grep xCATreg |grep -v grep|awk '{print \$2}'`;
|
|
for (my $i = 1 ; $i < $cnt + 1 ; $i++) {
|
|
my $pid = `echo "$out"|sed -n ${i}p`;
|
|
chomp($pid);
|
|
next if ($pid == $$);
|
|
&runcmd("kill $pid");
|
|
}
|
|
|
|
clean_outdated_file();
|
|
return 0;
|
|
}
|
|
|
|
#######################################
|
|
# usage for arguments
|
|
#######################################
|
|
sub usage {
|
|
print "Usage:xCATreg - Run xcat autotest test cases.\n
|
|
Explanation for the options:
|
|
-b: when -b is specified, the regression will be done against the specified branch. The default value is master.\n
|
|
-m: when -m is specified, run regression on the specified mn. The default value is all the mn in configuration file.\n
|
|
-f: specify the configuration file location.\n
|
|
-a: when -a is specified, run the regression even there is no code updates.\n
|
|
-e: <email_addr> send the test result to email_addr\n\n";
|
|
print " xCATreg [-?|-h]\n";
|
|
print " xCATreg [-f configure file] [-b branch] [-m mangement node] [-e mail list] [-a] Even code not updates, there will be regression.\n";
|
|
print " xCATreg [-f configure file] [-b branch] [-m mangement node] [-e mail list] if code updates there will be regression.\n";
|
|
print "\n";
|
|
return;
|
|
}
|
|
|
|
#######################################
|
|
# send messages
|
|
#######################################
|
|
sub send_msg {
|
|
my $num = shift;
|
|
my $msg = shift;
|
|
my $content;
|
|
if ($num == 0) {
|
|
$content = "Fatal error:";
|
|
} elsif ($num == 1) {
|
|
$content = "Warning:";
|
|
} elsif ($num == 2) {
|
|
$content = "Notice:";
|
|
}
|
|
my $timestamp = `date +"%Y%m%d%H%M%S"`;
|
|
chomp($timestamp);
|
|
if (!open(LOGFILE, ">> $logfiledir/$logfile")) {
|
|
return 1;
|
|
}
|
|
print LOGFILE "$content $timestamp $$: $msg.\n";
|
|
close LOGFILE;
|
|
}
|
|
|
|
|
|
#######################################
|
|
# read_conf
|
|
#######################################
|
|
sub read_conf {
|
|
my $myfile = undef;
|
|
my $line = undef;
|
|
|
|
if (!open($myfile, "$configfile")) {
|
|
send_msg(0, "Open $configfile failed");
|
|
return 1;
|
|
}
|
|
while ($line = <$myfile>) {
|
|
$line =~ s/\s//g;
|
|
next if ($line =~ /^#/);
|
|
next if ($line eq "");
|
|
my @attr = split(/=/, $line);
|
|
$confkeys{ $attr[0] } = $attr[1];
|
|
}
|
|
close($myfile);
|
|
return 0;
|
|
}
|
|
|
|
#######################################
|
|
# mn_install to install all mn
|
|
#######################################
|
|
sub mn_install {
|
|
my $mn = shift;
|
|
|
|
&runcmd("lsdef|grep $confkeys{$mn}");
|
|
if ($?) {
|
|
send_msg(0, "[$mn->$confkeys{$mn}][mn_install] Didn't defined $confkeys{$mn} in current control node");
|
|
return 1;
|
|
}
|
|
|
|
my $os = $confkeys{ $mn . "os" };
|
|
my $arch = "ppc64";
|
|
if ($mn !~ /aix/ && $mn =~ /x/) {
|
|
$arch = "x86_64";
|
|
} elsif ($mn !~ /aix/ && $mn =~ /LE/) {
|
|
$arch = "ppc64le";
|
|
} elsif ($mn !~ /aix/ && $mn =~ /EL/) {
|
|
$arch = "ppc64el";
|
|
}
|
|
my $osimage = "";
|
|
|
|
if ($mn !~ /aix/) {
|
|
$osimage = $os . "-" . $arch . "-install-compute";
|
|
} else {
|
|
$osimage = "aix"; #????????need to test for aix
|
|
}
|
|
|
|
#print "[$mn] : os= $os arch=$arch osimage=$osimage\n";
|
|
|
|
&runcmd("lsdef -t osimage|grep $osimage");
|
|
if ($?) {
|
|
send_msg(0, "[$mn->$confkeys{$mn}][mn_install] Didn't defined $osimage in current control node");
|
|
return 1;
|
|
}
|
|
|
|
send_msg(2, "[$mn->$confkeys{$mn}][mn_install] installing $osimage for node $confkeys{$mn}................");
|
|
|
|
if ($arch =~ /x86_64/ || $arch =~ /ppc64le/ || $arch =~ /ppc64el/) {
|
|
&runcmd("rmvm $confkeys{$mn} -f -p ");
|
|
&runcmd("chdef $confkeys{$mn} mac= ");
|
|
&runcmd("mkvm $confkeys{$mn} -s 40G");
|
|
}
|
|
|
|
&runcmd("makedhcp -n");
|
|
&runcmd("makedns -n");
|
|
&runcmd("chdef $confkeys{$mn} status=");
|
|
if ($?) {
|
|
send_msg(0, "[$mn->$confkeys{$mn}][mn_install] Failed to change MN status in current control node");
|
|
return 1;
|
|
}
|
|
|
|
|
|
&runcmd("nodeset $confkeys{$mn} osimage=$osimage");
|
|
if ($arch =~ /x86_64/ || $arch =~ /ppc64le/ || $arch =~ /ppc64el/) {
|
|
&runcmd("rpower $confkeys{$mn} boot");
|
|
} else {
|
|
&runcmd("rnetboot $confkeys{$mn}");
|
|
}
|
|
|
|
&runcmd("sleep 600");
|
|
&runcmd("a=0;while ! `lsdef -l $confkeys{$mn}|grep status|grep booted >/dev/null`; do sleep 30;((a++));if [ \$a -gt 100 ];then break;fi done");
|
|
|
|
# if($mn !~ /ubuntu/){
|
|
# &runcmd("sleep 300");
|
|
# &runcmd("a=0;while ! `lsdef -l $confkeys{$mn}|grep status|grep booted >/dev/null`; do sleep 10;((a++));if [ \$a -gt 200 ];then break;fi done");}
|
|
# else{
|
|
# &runcmd("sleep 1800");
|
|
# }
|
|
|
|
&runcmd("lsdef -l $confkeys{$mn}|grep status|grep booted >/dev/null");
|
|
if ($?) {
|
|
send_msg(0, "[$mn->$confkeys{$mn}][mn_install] install $osimage for node $confkeys{$mn} failed");
|
|
return 1;
|
|
}
|
|
|
|
if ($mn =~ /ubuntu/) {
|
|
system("xdsh $confkeys{$mn} \"rm -rf /bin/sh\" >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} \"ln -s /bin/bash /bin/sh\"");
|
|
}
|
|
|
|
# if($mn =~ /ubuntu/){
|
|
# system("updatenode $confkeys{$mn} -P \"confignics -s\" >/dev/null 2>&1");
|
|
# system("xdsh $confkeys{$mn} \"iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE\" >/dev/null 2>&1");
|
|
# system("xdsh $confkeys{$mn} \"iptables -A FORWARD -i eth1 -j ACCEPT\" >/dev/null 2>&1");
|
|
# system("xdsh $confkeys{$mn} \"iptables -A FORWARD -i eth1 -o eth0 -m state --state RELATED,ESTABLISHED -j ACCEPT\" >/dev/null 2>&1");
|
|
# }
|
|
if ($os =~ /rhels7/) {
|
|
system("xdsh $confkeys{$mn} \"yum -y install bzip2 > /var/log/xcat/install_xcat.log 2>&1\" >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} \"yum -y install createrepo > /var/log/xcat/install_xcat.log 2>&1\" >/dev/null 2>&1");
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
#######################################
|
|
# down dep
|
|
#######################################
|
|
sub down_dep {
|
|
my $mn = shift;
|
|
send_msg(2, "[$mn->$confkeys{$mn}][down xcatdep] starting to down the latest xcat dep");
|
|
my $downxcatdepdir = "$regrootdir/curl/$mn";
|
|
`mkdir -p $downxcatdepdir`;
|
|
my $curllink = $confkeys{ $mn . "curllink" };
|
|
my $deplink;
|
|
if (!-f "$downxcatdepdir/curlhtml1") {
|
|
send_msg(2, "it is first time to down xcatdep");
|
|
` curl "$curllink/" |grep "xcat-dep-" > "$downxcatdepdir/curlhtml1" `;
|
|
` tail -1 "$downxcatdepdir/curlhtml1" > "$downxcatdepdir/curlfile"`;
|
|
$deplink = `cat "$downxcatdepdir"/"curlfile" |awk '{print \$6}' |sed 's/"/ /g' |awk '{print \$2}' `;
|
|
`mkdir -p $xcatdepdir/$mn`;
|
|
` wget -P "$xcatdepdir/$mn" -c $curllink/$deplink `;
|
|
if ($?) {
|
|
send_msg(0, "download xcatdep failed");
|
|
return 1;
|
|
}
|
|
$depupdateflag = 1;
|
|
} else {
|
|
`curl "$curllink/" |grep "xcat-dep-" > "$downxcatdepdir/curlhtml2" `;
|
|
` diff "$downxcatdepdir"/"curlhtml1" "$downxcatdepdir"/"curlhtml2"`;
|
|
if ($? == 0) {
|
|
send_msg(2, "no update for xcatdep ");
|
|
$depupdateflag = 0;
|
|
} else {
|
|
` tail -1 "$downxcatdepdir"/"curlhtml2"> "$downxcatdepdir"/"curlfile"`;
|
|
$deplink = `cat "$downxcatdepdir"/"curlfile" |awk '{print \$6}' |sed 's/"/ /g' |awk '{print \$2}' `;
|
|
` rm -rf $xcatdepdir/$mn`;
|
|
`mkdir -p $xcatdepdir/$mn`;
|
|
` wget -P "$xcatdepdir/$mn" -c $curllink/$deplink `;
|
|
if ($?) {
|
|
send_msg(2, "download xcatdep failed");
|
|
return 1;
|
|
}
|
|
`mv "$downxcatdepdir"/"curlhtml2" "$downxcatdepdir"/"curlhtml1" -f `;
|
|
$depupdateflag = 2;
|
|
}
|
|
}
|
|
return ($depupdateflag);
|
|
}
|
|
|
|
#########################################
|
|
# down xcat-core build
|
|
###########################################
|
|
sub down_xcat_core_build {
|
|
my $mn = shift;
|
|
send_msg(2, "[$mn->$confkeys{$mn}][down xcatcore] starting to down the latest xcat core");
|
|
my $downxcatcoredowndir = "$xcatcoredowndir/$mn";
|
|
if (!-f "$xcatcoredowndir/$mn") {
|
|
`mkdir -p $xcatcoredowndir/$mn`;
|
|
}
|
|
my $downxcatcoreip = $confkeys{ $mn . "downxcatcoreip" };
|
|
my $downxcatcoredir = $confkeys{ $mn . "downxcatcoredir" };
|
|
my $latestxcatcoredir = `ssh $downxcatcoreip ls -rt $downxcatcoredir / |tail -1 |awk '{printf \$0}'`;
|
|
my $xcatcorename = $confkeys{ $mn . "xcatcorename" };
|
|
`scp -r $downxcatcoreip\:$downxcatcoredir/$latestxcatcoredir/$xcatcorename $downxcatcoredowndir `;
|
|
if ($? != 0) {
|
|
send_msg(0, "[$mn->$confkeys{$mn}][down_xcat_core_build] failed");
|
|
return 1;
|
|
}
|
|
}
|
|
|
|
#######################################
|
|
# copy code
|
|
#######################################
|
|
sub copy_code {
|
|
my $mn = shift;
|
|
|
|
send_msg(2, "[$mn->$confkeys{$mn}][copy_code] starting to copy the latest xcat code and dep");
|
|
my $dnsserver = `cat /etc/hosts |grep $confkeys{$mn}|awk '{print \$1}'|head -n 1`;
|
|
chomp($dnsserver);
|
|
my $res = system("xdsh $confkeys{$mn} \"sed -i 's:nameserver .*:nameserver $dnsserver:g' /etc/resolv.conf\" >/dev/null 2>&1");
|
|
if ($res != 0) {
|
|
send_msg(0, "[$mn->$confkeys{$mn}][install_xcat] modify resolv.conf in $mn failed");
|
|
return 1;
|
|
}
|
|
|
|
#copy xcat-dep tarball to MN:/
|
|
my $depball = "";
|
|
if ($mn =~ /aix/) {
|
|
$depball = `ls -l $xcatdepdir/$mn|awk '{print \$9}'|grep aix|tail -n 1`;
|
|
} elsif ($mn =~ /ubuntu/) {
|
|
$depball = `ls -l $xcatdepdir/$mn|awk '{print \$9}'|grep ubuntu|tail -n 1`;
|
|
} else {
|
|
$depball = `ls -l $xcatdepdir/$mn|awk '{print \$9}'|grep 'xcat-dep'|tail -n 1`;
|
|
}
|
|
chomp($depball);
|
|
print "depball is $depball\n";
|
|
my $coreball = `ls -l $xcatcoredowndir/$mn|awk '{print \$9}'|tail -n 1`;
|
|
chomp($coreball);
|
|
if ($depball eq "") {
|
|
send_msg(0, "[$mn->$confkeys{$mn}][copy_code] can't find dep tarball for $mn under dir $xcatdepdir");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[$mn->$confkeys{$mn}][copy_code] find dep tarball $depball for $mn");
|
|
if ($coreball eq "") {
|
|
send_msg(0, "[$mn->$confkeys{$mn}][copy_code] can't find core tarball for $mn under dir $coreball");
|
|
return 1;
|
|
}
|
|
|
|
send_msg(2, "[$mn->$confkeys{$mn}][copy_code] find core tarball $coreball for $mn");
|
|
|
|
$res = system("scp $xcatdepdir/$mn/$depball root\@$confkeys{$mn}:/ >/dev/null");
|
|
if ($res) {
|
|
send_msg(0, "[$mn->$confkeys{$mn}][copy_code] copy $depball to $mn failed");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[$mn->$confkeys{$mn}][copy_code] copy $depball to $mn successfully");
|
|
|
|
$res = system("scp $xcatcoredowndir/$mn/$coreball root\@$confkeys{$mn}:/ >/dev/null");
|
|
if ($res) {
|
|
send_msg(0, "[$mn->$confkeys{$mn}][copy_code] copy $coreball to $mn failed");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[$mn->$confkeys{$mn}][copy_code] copy $coreball to $mn successfully");
|
|
|
|
$res = system("xdsh $confkeys{$mn} 'cd / && tar xvf /$depball' >/dev/null 2>&1");
|
|
if ($?) {
|
|
send_msg(0, "[$mn->$confkeys{$mn}][copy_code] untar $depball in $mn failed");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[$mn->$confkeys{$mn}][copy_code] untar $depball in $mn successfully");
|
|
$res = system("xdsh $confkeys{$mn} 'cd / && tar xvf /$coreball' >/dev/null 2>&1");
|
|
if ($?) {
|
|
send_msg(0, "[$mn->$confkeys{$mn}][copy_code] untar $coreball in $mn failed");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[$mn->$confkeys{$mn}][copy_code] untar $coreball in $mn successfully");
|
|
|
|
#copy xcat-core source code to MN:/
|
|
if ($mn =~ /ubuntux/) {
|
|
|
|
#defect 107 Automation ENV replaces the link file to real file before running xcat build
|
|
$res = system("xdsh $confkeys{$mn} 'mkdir -p /xcatbuild' && rsync -avz $xcatcoredir root\@$confkeys{$mn}:/xcatbuild/ >/dev/null");
|
|
} else {
|
|
|
|
#defect 107 Automation ENV replaces the link file to real file before running xcat build
|
|
$res = system("rsync -avz $xcatcoredir root\@$confkeys{$mn}:/ >/dev/null");
|
|
}
|
|
if ($res) {
|
|
send_msg(0, "[$mn->$confkeys{$mn}][copy_code] copy the latest xcat code to $mn failed");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[$mn->$confkeys{$mn}][copy_code] copy the latest xcat code to $mn successfully");
|
|
|
|
#copy /etc/hosts file to MN
|
|
$res = system("scp /etc/hosts root\@$confkeys{$mn}:/etc >/dev/null");
|
|
if ($?) {
|
|
send_msg(0, "[$mn->$confkeys{$mn}][copy_code] copy /etc/hosts to $mn failed");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[$mn->$confkeys{$mn}][copy_code] copy /etc/hosts to $mn successfully");
|
|
|
|
my $os = $confkeys{ $mn . "os" };
|
|
my $regconf = $mn . ".conf";
|
|
system("sed -i 's|os=\\S*|os=$os|g' $configfiledir/$regconf");
|
|
|
|
my $arch = "ppc64";
|
|
if ($mn !~ /aix/ && $mn =~ /x/) {
|
|
$arch = "x86_64";
|
|
} elsif ($mn !~ /aix/ && $mn =~ /LE/) {
|
|
$arch = "ppc64le";
|
|
} elsif ($mn !~ /aix/ && $mn =~ /EL/) {
|
|
$arch = "ppc64el";
|
|
}
|
|
my $iso;
|
|
|
|
#copy specific files to MN for each platform
|
|
if ($mn =~ /rh/) {
|
|
$res = system("scp /install/$os/$arch/RPM-GPG-KEY-redhat-release root\@$confkeys{$mn}:/ >/dev/null");
|
|
if ($?) {
|
|
send_msg(0, "[$mn->$confkeys{$mn}][copy_code] copy RPM-GPG-KEY-redhat-release to $mn failed");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[$mn->$confkeys{$mn}][copy_code] copy RPM-GPG-KEY-redhat-release to $mn successfully");
|
|
|
|
$os =~ /(\D+)(\d+\.\d+)/;
|
|
$iso = `ls -l $osisodir|awk '{print \$9}'|grep RHEL`;
|
|
chomp($iso);
|
|
|
|
#print "[$mn->$confkeys{$mn}][copy_code] iso=$iso\n";
|
|
if ($iso eq "") {
|
|
send_msg(0, "[$mn->$confkeys{$mn}][copy_code] can't find iso for $os and $arch under dir $osisodir");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[$mn->$confkeys{$mn}][copy_code] find $iso for os=$os and arch=$arch");
|
|
|
|
$res = system("scp $osisodir/$iso root\@$confkeys{$mn}:/ >/dev/null");
|
|
if ($?) {
|
|
send_msg(0, "[$mn->$confkeys{$mn}][copy_code] copy ISO $iso to $mn failed");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[$mn->$confkeys{$mn}][copy_code] copy ISO $iso to $mn successfully");
|
|
system("sed -i 's|^ISO=/\\S*|ISO=/$iso|g' $configfiledir/$regconf");
|
|
}
|
|
|
|
if ($mn =~ /sles/) {
|
|
$os =~ /(\D+)(\d+)\.?(\d?)/;
|
|
if ($3) {
|
|
$iso = `ls -l $osisodir| awk '{print \$9}'|grep SLE|grep $arch |grep -- "-$2-SP$3-" |tail -n 1`;
|
|
} else {
|
|
$iso = `ls -l $osisodir| awk '{print \$9}'|grep SLE|grep $arch |grep -v "SP"|grep -- "-$2-"|tail -n 1`;
|
|
}
|
|
chomp($iso);
|
|
|
|
#print "[$mn->$confkeys{$mn}][copy_code] iso=$iso\n";
|
|
if ($iso eq "") {
|
|
send_msg(0, "[$mn->$confkeys{$mn}][copy_code] can't find ISO for os=$os and arch=$arch under dir $osisodir");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[$mn->$confkeys{$mn}][copy_code] find $iso for os=$os and arch=$arch");
|
|
|
|
$res = system("scp $osisodir/$iso root\@$confkeys{$mn}:/ >/dev/null");
|
|
if ($?) {
|
|
send_msg(0, "[$mn->$confkeys{$mn}][copy_code] copy ISO $iso to $mn failed");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[$mn->$confkeys{$mn}][copy_code] copy ISO $iso to $mn successfully");
|
|
system("sed -i 's|^ISO=/\\S*|ISO=/$iso|g' $configfiledir/$regconf");
|
|
|
|
if ($os =~ /sles11/) {
|
|
$res = system("xdsh $confkeys{$mn} \"mv /usr/lib/rpm/find-requires /usr/lib/rpm/find-requires.org\" && scp $osisodir/find-requires $confkeys{$mn}:/usr/lib/rpm/ > /dev/null");
|
|
if ($?) {
|
|
send_msg(0, "[$mn->$confkeys{$mn}][copy_code] For sles11, copy find-requires to $mn failed");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[$mn->$confkeys{$mn}][copy_code] For sles11, copy find-requires to $mn successfully");
|
|
}
|
|
}
|
|
if ($mn =~ /ubuntu/) {
|
|
$os =~ /(\D+)(\d+\.\d+\.?\d?)/;
|
|
my $osversion = $2;
|
|
if ($mn =~ /ubuntux/) {
|
|
$iso = `ls -l $osisodir| awk '{print \$9}'|grep ubuntu|grep amd64 |grep $osversion |tail -n 1`;
|
|
} elsif ($mn =~ /ubuntuEL/) {
|
|
$iso = `ls -l $osisodir| awk '{print \$9}'|grep ubuntu|grep ppc64el |grep $osversion |tail -n 1`;
|
|
}
|
|
chomp($iso);
|
|
|
|
#print "[$mn->$confkeys{$mn}][copy_code] iso=$iso\n";
|
|
if ($iso eq "") {
|
|
send_msg(0, "[$mn->$confkeys{$mn}][copy_code] can't find ISO for os=$os and arch=$arch under dir $osisodir");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[$mn->$confkeys{$mn}][copy_code] find $iso for os=$os and arch=$arch");
|
|
|
|
$res = system("scp $osisodir/$iso root\@$confkeys{$mn}:/ >/dev/null");
|
|
if ($?) {
|
|
send_msg(0, "[$mn->$confkeys{$mn}][copy_code] copy ISO $iso to $mn failed");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[$mn->$confkeys{$mn}][copy_code] copy ISO $iso to $mn successfully");
|
|
system("sed -i 's|^ISO=/\\S*|ISO=/$iso|g' $configfiledir/$regconf");
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
#######################################
|
|
# config mn
|
|
#######################################
|
|
sub install_xcat {
|
|
my $mn = shift;
|
|
send_msg(2, "[$mn->$confkeys{$mn}][install_xcat] starting to install xcat in $confkeys{$mn}");
|
|
|
|
my $os = $confkeys{ $mn . "os" };
|
|
my $arch = "ppc64";
|
|
if ($mn !~ /aix/ && $mn =~ /x/) {
|
|
$arch = "x86_64";
|
|
} elsif ($mn !~ /aix/ && $mn =~ /LE/) {
|
|
$arch = "ppc64le";
|
|
} elsif ($mn !~ /aix/ && $mn =~ /EL/) {
|
|
$arch = "ppc64el";
|
|
}
|
|
if ($mn =~ /ubuntu/) {
|
|
system("xdsh $confkeys{$mn} \"echo \"nameserver 9.0.2.1\" >> /etc/resolv.conf\" >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} 'apt-get -y install software-properties-common' >/dev/null 2>&1");
|
|
if ($mn =~ /ubuntux/) {
|
|
system("xdsh $confkeys{$mn} 'add-apt-repository \"deb http://archive.ubuntu.com/ubuntu \$(lsb_release -sc) main\"' >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} 'add-apt-repository \"deb http://archive.ubuntu.com/ubuntu \$(lsb_release -sc)-updates main\"' >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} 'add-apt-repository \"deb http://archive.ubuntu.com/ubuntu \$(lsb_release -sc) universe\"' >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} 'add-apt-repository \"deb http://archive.ubuntu.com/ubuntu \$(lsb_release -sc)-updates universe\"' >/dev/null 2>&1");
|
|
} elsif ($mn =~ /ubuntuEL/) {
|
|
system("xdsh $confkeys{$mn} 'add-apt-repository \"deb http://ports.ubuntu.com/ubuntu-ports \$(lsb_release -sc) main\"' >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} 'add-apt-repository \"deb http://ports.ubuntu.com/ubuntu-ports \$(lsb_release -sc)-updates main\"' >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} 'add-apt-repository \"deb http://ports.ubuntu.com/ubuntu-ports \$(lsb_release -sc) universe\"' >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} 'add-apt-repository \"deb http://ports.ubuntu.com/ubuntu-ports \$(lsb_release -sc)-updates universe\"' >/dev/null 2>&1");
|
|
}
|
|
|
|
system("xdsh $confkeys{$mn} \"wget -O - \"http://sourceforge.net/projects/xcat/files/ubuntu/apt.key/download\" | apt-key add -\" >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} 'apt-get clean all' >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} 'apt-get update' >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} 'apt-get -y install build-essential dpkg-dev dh-make debhelper fakeroot gnupg lintian pbuilder quilt reprepro libsoap-lite-perl libdbi-perl' >/dev/null 2>&1");
|
|
}
|
|
|
|
if ($mn =~ /rh/) {
|
|
$os =~ /(\D+)(\d+)\.?(\d?)/;
|
|
system("xdsh $confkeys{$mn} \"cd /xcat-core && ./mklocalrepo.sh\" >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} \"cd /xcat-dep/rh$2/$arch && ./mklocalrepo.sh\" >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} \"rpm --import /RPM-GPG-KEY-redhat-release\" >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} \"yum -y install xCAT xCAT-test > /var/log/xcat/install_xcat.log 2>&1\" >/dev/null 2>&1");
|
|
if ($os =~ /rhels7/) {
|
|
system("xdsh $confkeys{$mn} \"yum -y install mariadb-devel mariadb-libs mariadb-server mariadb-bench mariadb perl-DBD-MySQL mysql-connector-odbc unixODBC\" >/dev/null 2>&1");
|
|
} elsif ($os =~ /rhels6/) {
|
|
system("xdsh $confkeys{$mn} \"yum -y install mysql-server mysql mysql-bench mysql-devel mysql-connector-odbc\" >/dev/null 2>&1");
|
|
}
|
|
system("xdsh $confkeys{$mn} \"source /etc/profile.d/xcat.sh\" >/dev/null 2>&1");
|
|
if ($mn =~ /rhx/) {
|
|
system("xdsh $confkeys{$mn} \"yum install -y perl-Sys-Virt\" >/dev/null 2>&1");
|
|
}
|
|
} elsif ($mn =~ /sles/) {
|
|
$os =~ /(\D+)(\d+)\.?(\d?)/;
|
|
system("xdsh $confkeys{$mn} \"zypper ar file:///xcat-dep/sles$2/$arch xCAT-dep\" >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} \"zypper ar file:///xcat-core xCAT-core\" >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} \"zypper sl -U\" >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} \"zypper --gpg-auto-import-keys search --match-exact -s screen\" >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} \"zypper -n install xCAT xCAT-test > /var/log/xcat/install_xcat.log 2>&1\" >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} \"zypper -n install createrepo\" >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} \"zypper -n install mysql-client libmysqlclient_r15 libqt4-sql-mysql libmysqlclient15 perl-DBD-mysql mysql unixODBC\" >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} \"source /etc/profile.d/xcat.sh\" >/dev/null 2>&1");
|
|
|
|
if ($mn =~ /slesLE/) {
|
|
system("xdsh $confkeys{$mn} \"zypper -n install perl-Net-DNS-0.80-1.ppc64le\">/dev/null 2>&1");
|
|
}
|
|
if ($mn =~ /slesx/) {
|
|
system("xdsh $confkeys{$mn} \"zypper -n install perl-Sys-Virt\" >/dev/null 2>&1");
|
|
}
|
|
} elsif ($mn =~ /ubuntux/) {
|
|
system("xdsh $confkeys{$mn} \"/xcat-core/mklocalrepo.sh\" >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} \"/xcat-dep/mklocalrepo.sh\" >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} 'apt-get clean all' >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} 'apt-get update' >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} 'apt-get -y install xcat xCAT-test > /var/log/xcat/install_xcat.log 2>&1' >/dev/null 2>&1");
|
|
} elsif ($mn =~ /ubuntuEL/) {
|
|
system("xdsh $confkeys{$mn} \"/xcat-core/mklocalrepo.sh\" >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} \"/xcat-dep/mklocalrepo.sh\" >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} 'apt-get clean all' >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} 'apt-get update' >/dev/null 2>&1");
|
|
system("xdsh $confkeys{$mn} 'apt-get -y install xcat xCAT-test > /var/log/xcat/install_xcat.log 2>&1' >/dev/null 2>&1");
|
|
system("scp $osisodir/mini.iso $confkeys{$mn}:/ ");
|
|
}
|
|
|
|
#check if MN is installed successful
|
|
my $lsdefout = `xdsh $confkeys{$mn} '/opt/xcat/bin/lsdef -v'`;
|
|
chomp($lsdefout);
|
|
if ($lsdefout !~ /lsdef - Version/) {
|
|
send_msg(0, "[$mn->$confkeys{$mn}][install_xcat] install xcat in $mn failed");
|
|
return 1;
|
|
}
|
|
|
|
send_msg(2, "[$mn->$confkeys{$mn}][install_xcat] install xcat in $mn successfully");
|
|
return 0;
|
|
}
|
|
|
|
#######################################
|
|
# do test
|
|
#######################################
|
|
sub do_test {
|
|
my $mn = shift;
|
|
my $os = $confkeys{ $mn . "os" };
|
|
my $casestop = 1;
|
|
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] starting to run regression test in $confkeys{$mn}");
|
|
|
|
my $regconf = $mn . ".conf";
|
|
my $res = system("scp $configfiledir/$regconf root\@$confkeys{$mn}:/opt/xcat/share/xcat/tools/autotest/default.conf >/dev/null");
|
|
if ($res != 0) {
|
|
send_msg(0, "[$mn->$confkeys{$mn}][do_test] copy $configfiledir/$regconf to $mn failed");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] copy $configfiledir/$regconf to $mn successfully");
|
|
if ($os =~ /rhels7/) {
|
|
system("scp /opt/xcat/share/xcat/install/rh/service.rhels7.ppc64.otherpkgs.pkglist root\@$confkeys{$mn}:/opt/xcat/share/xcat/install/rh >/dev/null");
|
|
}
|
|
my $diskfull_installation_flat_testcase = "";
|
|
my $diskless_installation_flat_testcase = "";
|
|
my $statelite_installation_flat_testcase = "";
|
|
my $sn_installation_hierarchy_testcase = "";
|
|
my $diskfull_installation_hierarchy_testcase = "";
|
|
my $diskless_installation_hierarchy_testcase = "";
|
|
my $statelite_installation_hierarchy_by_ramdisk_testcase = "";
|
|
my $statelite_installation_hierarchy_by_nfs_testcase = "";
|
|
my $setup_dockerhost_testcase = "";
|
|
my $batbundle = "";
|
|
|
|
if ((($mn =~ /sles/) || ($mn =~ /rh/)) && ($mn !~ /E/)) {
|
|
$diskfull_installation_flat_testcase = "reg_linux_diskfull_installation_flat";
|
|
$diskless_installation_flat_testcase = "reg_linux_diskless_installation_flat";
|
|
$statelite_installation_flat_testcase = "reg_linux_statelite_installation_flat";
|
|
$sn_installation_hierarchy_testcase = "reg_linux_SN_installation_hierarchy";
|
|
$diskfull_installation_hierarchy_testcase = "reg_linux_diskfull_installation_hierarchy";
|
|
$diskless_installation_hierarchy_testcase = "reg_linux_diskless_installation_hierarchy";
|
|
$statelite_installation_hierarchy_by_ramdisk_testcase = "reg_linux_statelite_installation_hierarchy_by_ramdisk";
|
|
$statelite_installation_hierarchy_by_nfs_testcase = "reg_linux_statelite_installation_hierarchy_by_nfs";
|
|
$batbundle = "p_rhels_cmd.bundle" if ($mn =~ /rhp/);
|
|
$batbundle = "x_rhels_cmd.bundle" if ($mn =~ /rhx/);
|
|
$batbundle = "p_sles_cmd.bundle" if ($mn =~ /slesp/);
|
|
$batbundle = "x_sles_cmd.bundle" if ($mn =~ /slesx/);
|
|
} elsif ($mn =~ /ubuntux/) {
|
|
$diskfull_installation_flat_testcase = "Ubuntu_full_installation_flat_x86_vm";
|
|
$diskless_installation_flat_testcase = "Ubuntu_diskless_installation_flat_x86_vm";
|
|
$setup_dockerhost_testcase = "Full_installation_flat_docker";
|
|
$batbundle = "x_ubuntu_cmd.bundle";
|
|
} elsif ($mn =~ /E/) {
|
|
$diskfull_installation_flat_testcase = "Full_installation_flat_p8_le";
|
|
$diskless_installation_flat_testcase = "Diskless_installation_flat_p8_le";
|
|
$batbundle = "le_rhels_cmd.bundle" if ($mn =~ /rhLE/);
|
|
$batbundle = "le_sles_cmd.bundle" if ($mn =~ /slesLE/);
|
|
$batbundle = "le_ubuntu_cmd.bundle" if ($mn =~ /ubuntuEL/);
|
|
}
|
|
if (($mn !~ /ubuntu/) && ($mn !~ /E/)) {
|
|
{
|
|
my $rc;
|
|
|
|
#diskfull installation in flat mode test
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] doing diskfull installation test in flat mode in $mn.....");
|
|
$rc = system("xdsh $confkeys{$mn} \"xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf -t $diskfull_installation_flat_testcase\" >/dev/null 2>&1");
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] do diskfull installation test in flat mode in $mn.....[done]");
|
|
last if ($rc != 0);
|
|
|
|
#commands bundle in flat mode test
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] doing all commands test in flat mode in $mn.....");
|
|
$rc = system("xdsh $confkeys{$mn} \"xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf -b $batbundle\" >/dev/null 2>&1");
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] do all commands test in flat mode in $mn.....[done]");
|
|
last if ($rc != 0);
|
|
|
|
#diskless installation in flat mode test
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] doing diskless installation test in flat mode in $mn.....");
|
|
$rc = system("xdsh $confkeys{$mn} \"xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf -t $diskless_installation_flat_testcase\" >/dev/null 2>&1");
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] do diskless installation test in flat mode in $mn.....[done]");
|
|
last if ($rc != 0);
|
|
|
|
#statelite installation in flat mode test
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] doing statelite installation test in flat mode in $mn.....");
|
|
$rc = system("xdsh $confkeys{$mn} \"xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf -t $statelite_installation_flat_testcase\" >/dev/null 2>&1");
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] do statelite installation test in flat mode in $mn.....[done]");
|
|
last if ($rc != 0);
|
|
|
|
#deploy SN
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] setting up mysql for installing SN in $mn.....");
|
|
$res = system("ssh -t $confkeys{$mn} 'exec bash -l -i -c \"XCATMYSQLADMIN_PW=12345 XCATMYSQLROOT_PW=12345 /opt/xcat/bin/mysqlsetup -i\"' >/dev/null 2>&1");
|
|
if ($res != 0) {
|
|
send_msg(0, "[$mn->$confkeys{$mn}][do_test] set up mysql for installing SN in $mn failed");
|
|
return 1;
|
|
}
|
|
$res = system("xdsh $confkeys{$mn} \"echo \\\"GRANT ALL on xcatdb.* TO xcatadmin@\'%\' IDENTIFIED BY \'12345\'\;\\\" | mysql -u root -p12345\" >/dev/null 2>&1");
|
|
if ($res != 0) {
|
|
send_msg(0, "[$mn->$confkeys{$mn}][do_test] grant right to mysql for installing SN in $mn failed");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] set up mysql for installing SN in $mn.....[done]");
|
|
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] preparing xcat ball for installing SN in $mn.....");
|
|
my $os = $confkeys{ $mn . "os" };
|
|
my $arch = "ppc64";
|
|
if (($mn !~ /aix/) && ($mn =~ /x/)) {
|
|
$arch = "x86_64";
|
|
} elsif ($mn !~ /aix/ && $mn =~ /LE/) {
|
|
$arch = "ppc64le";
|
|
} elsif ($mn !~ /aix/ && $mn =~ /EL/) {
|
|
$arch = "ppc64el";
|
|
}
|
|
my $xcatballpath = "/install/post/otherpkgs/$os/$arch/xcat";
|
|
$res = system("xdsh $confkeys{$mn} \"mkdir -p $xcatballpath/xcat-core && cp -r /xcat-core/* $xcatballpath/xcat-core && cp -r /xcat-dep $xcatballpath\"");
|
|
if ($res != 0) {
|
|
send_msg(0, "[$mn->$confkeys{$mn}][do_test] prepare xcat ball for installing SN in $mn failed");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] prepare xcat ball for installing SN in $mn.....[done]");
|
|
|
|
if ($mn =~ /rhp/) {
|
|
my $ping_sn_on_rhp_pid = fork();
|
|
if (!defined($ping_sn_on_rhp_pid)) {
|
|
send_msg(0, "fork process for trigger random number generator on redhat+ppc environment to install SN error");
|
|
exit;
|
|
} elsif ($ping_sn_on_rhp_pid == 0) {
|
|
send_msg(2, "....fork 1 process[pid=$$] for trigger random number generator on redhat+ppc environment to install SN..........");
|
|
my $snnodename = `cat $configfiledir/$regconf |grep SN|grep =|awk -F'=' '{print \$2}'`;
|
|
chomp($snnodename);
|
|
&runcmd("sleep 900");
|
|
my $timecnt = 0;
|
|
while ($timecnt < 180) {
|
|
my $booted = system("xdsh $confkeys{$mn} \"lsdef -l $snnodename|grep status|grep booted\"");
|
|
last if ($booted == 0);
|
|
|
|
my $booting = system("xdsh $confkeys{$mn} \"lsdef -l $snnodename|grep status|grep booting\"");
|
|
system("xdsh $confkeys{$mn} \"xdsh $snnodename \"echo 1\"\"") if ($booting == 0);
|
|
|
|
&runcmd("sleep 10");
|
|
$timecnt++;
|
|
}
|
|
send_msg(2, "....process[pid=$$] for trigger random number generator on redhat+ppc environment to install SN....[exit]");
|
|
exit 0;
|
|
}
|
|
}
|
|
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] installing SN.....");
|
|
$rc = system("xdsh $confkeys{$mn} \"xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf -t $sn_installation_hierarchy_testcase\" >/dev/null 2>&1");
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] install SN.....[done]");
|
|
last if ($rc != 0);
|
|
|
|
#diskfull installation in hierarchy mode test
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] doing diskfull installation test in hierarchy mode in $mn.....");
|
|
$rc = system("xdsh $confkeys{$mn} \"xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf -t $diskfull_installation_hierarchy_testcase\" >/dev/null 2>&1");
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] do diskfull installation test in hierarchy mode in $mn.....[done]");
|
|
last if ($rc != 0);
|
|
|
|
#diskless installation in hierarchy mode test
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] doing diskless installation test in hierarchy mode in $mn.....");
|
|
$rc = system("xdsh $confkeys{$mn} \"xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf -t $diskless_installation_hierarchy_testcase\" >/dev/null 2>&1");
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] do diskless installation test in hierarchy mode in $mn.....[done]");
|
|
last if ($rc != 0);
|
|
|
|
#statelite installation by ramdisk in hierarchy mode test
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] doing statelite installation by ramdisk test in hierarchy mode in $mn.....");
|
|
$rc = system("xdsh $confkeys{$mn} \"xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf -t $statelite_installation_hierarchy_by_ramdisk_testcase\" >/dev/null 2>&1");
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] do statelite installation by ramdisk test in hierarchy mode in $mn.....[done]");
|
|
last if ($rc != 0);
|
|
|
|
#statelite installation by nfs in hierarchy mode test
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] doing statelite installation by nfs test in hierarchy mode in $mn.....");
|
|
$rc = system("xdsh $confkeys{$mn} \"xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf -t $statelite_installation_hierarchy_by_nfs_testcase\" >/dev/null 2>&1");
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] do statelite installation by nfs test in hierarchy mode in $mn.....[done]");
|
|
last if ($rc != 0);
|
|
|
|
$casestop = 0;
|
|
}
|
|
} else {
|
|
{
|
|
my $rc = 0;
|
|
|
|
#diskless installation in flat mode test
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] doing diskless installation test in flat mode in $mn.....");
|
|
$rc = system("ssh -t $confkeys{$mn} 'exec bash -l -i -c \"xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf -t $diskless_installation_flat_testcase\"' >/dev/null 2>&1");
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] do diskless installation test in flat mode in $mn.....[done]");
|
|
last if ($rc != 0);
|
|
|
|
#diskfull installation in flat mode test
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] doing diskfull installation test in flat mode in $mn.....");
|
|
$rc = system("ssh -t $confkeys{$mn} 'exec bash -l -i -c \"xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf -t $diskfull_installation_flat_testcase\"' >/dev/null 2>&1");
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] do diskfull installation test in flat mode in $mn.....[done]");
|
|
last if ($rc != 0);
|
|
|
|
if ($mn =~ /ubuntux/) {
|
|
|
|
#set up dockerhosts
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] setupdockerhost $mn.....");
|
|
$rc = system("ssh -t $confkeys{$mn} 'exec bash -l -i -c \"xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf -t $setup_dockerhost_testcase\"' >/dev/null 2>&1");
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] setup dockerhosts $mn.....[done]");
|
|
last if ($rc != 0);
|
|
}
|
|
|
|
#commands bundle in flat mode test
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] doing all commands test in flat mode in $mn.....");
|
|
$rc = system("ssh -t $confkeys{$mn} 'exec bash -l -i -c \"xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf -b $batbundle\"' >/dev/null 2>&1");
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] do all commands test in flat mode in $mn.....[done]");
|
|
$casestop = 0;
|
|
}
|
|
}
|
|
|
|
if ($casestop) {
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] the regression job in $mn was broken by one case.");
|
|
} else {
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] run the whole regression test in $mn finished");
|
|
}
|
|
|
|
send_msg(2, "[$mn->$confkeys{$mn}][install_xcat] run regression test in $mn finished");
|
|
|
|
$res = system("scp -r $confkeys{$mn}:/opt/xcat/share/xcat/tools/autotest/result/* $todayregresultdir/$mn/ >/dev/null 2>&1");
|
|
if ($res != 0) {
|
|
send_msg(0, "[$mn->$confkeys{$mn}][do_test] copy regression result to $todayregresultdir/$mn/ failed");
|
|
return 1;
|
|
}
|
|
send_msg(2, "[$mn->$confkeys{$mn}][do_test] copy regression result to $todayregresultdir/$mn/ successfully");
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
#######################################
|
|
# creat report
|
|
#######################################
|
|
sub creat_report {
|
|
|
|
my $mailreport .= "======================================\n";
|
|
$mailreport .= " Regression Result Summary\n";
|
|
$mailreport .= "======================================\n\n";
|
|
|
|
# $gitupdateflag = 0 git update out of time
|
|
# $gitupdateflag = 1 there isn't new code checkin in last one day
|
|
# $gitupdateflag = 2 git update failed
|
|
if ($gitupdateflag == 0) {
|
|
$mailreport .= "Get the latest xcat code from git out of time, stop regression test today!!!\n\n\n";
|
|
&runcmd("touch $mailfile && echo \"$mailreport\" > $mailfile");
|
|
&runcmd("mv $commitinfofile $commitinfofile.old && echo \"\" > $commitinfofile");
|
|
return 0;
|
|
} elsif ($gitupdateflag == 1 && !$forceregwithoutupdate) {
|
|
$mailreport .= "There isn't new code checkin last one day, stop regression test today!!!\n\n\n";
|
|
&runcmd("touch $mailfile && echo \"$mailreport\" > $mailfile");
|
|
&runcmd("mv $commitinfofile $commitinfofile.old && echo \"\" > $commitinfofile");
|
|
return 0;
|
|
} elsif ($gitupdateflag == 1 && $forceregwithoutupdate) {
|
|
$mailreport .= "There isn't new code checkin last one day, But force to run regresson today!!!\n\n\n";
|
|
&runcmd("mv $commitinfofile $commitinfofile.old && echo \"\" > $commitinfofile");
|
|
} elsif ($gitupdateflag == 2) {
|
|
$mailreport .= "Get the latest xcat source code from git error, stop regression test today!!!\n\n\n";
|
|
&runcmd("touch $mailfile && echo \"$mailreport\" > $mailfile");
|
|
&runcmd("mv $commitinfofile $commitinfofile.old && echo \"\" > $commitinfofile");
|
|
return 0;
|
|
}
|
|
|
|
while (my ($k, $v) = each %pidrecord) {
|
|
my $scenario = "[" . $confkeys{ $k . "os" };
|
|
if ($k =~ /p/) {
|
|
$scenario .= "+PPC64] ";
|
|
} elsif ($k =~ /E/) {
|
|
$scenario .= "+PPC64LE] ";
|
|
} elsif ($k =~ /x/) {
|
|
$scenario .= "+x86_64] ";
|
|
}
|
|
$mailreport .= "$scenario\n";
|
|
my $batbundle = "";
|
|
my $totalcase = 0;
|
|
|
|
if ($k =~ /slesp/ || $k =~ /rhx/ || $k =~ /slesx/ || $k =~ /rhp/) {
|
|
$batbundle = "$xcatcoredir/xCAT-test/autotest/bundle/p_rhels_cmd.bundle" if ($k =~ /rhp/);
|
|
$batbundle = "$xcatcoredir/xCAT-test/autotest/bundle/x_rhels_cmd.bundle" if ($k =~ /rhx/);
|
|
$batbundle = "$xcatcoredir/xCAT-test/autotest/bundle/p_sles_cmd.bundle" if ($k =~ /slesp/);
|
|
$batbundle = "$xcatcoredir/xCAT-test/autotest/bundle/x_sles_cmd.bundle" if ($k =~ /slesx/);
|
|
$totalcase = `cat $batbundle|wc -l`;
|
|
$totalcase += 8;
|
|
} elsif ($k =~ /ubuntux/) {
|
|
$batbundle = "/$xcatcoredir/xCAT-test/autotest/bundle/x_ubuntu_cmd.bundle";
|
|
$totalcase = `cat $batbundle|wc -l`;
|
|
$totalcase += 3;
|
|
} elsif ($k =~ /E/) {
|
|
$batbundle = "$xcatcoredir/xCAT-test/autotest/bundle/le_rhels_cmd.bundle" if ($k =~ /rhLE/);
|
|
$batbundle = "$xcatcoredir/xCAT-test/autotest/bundle/le_sles_cmd.bundle" if ($k =~ /slesLE/);
|
|
$batbundle = "$xcatcoredir/xCAT-test/autotest/bundle/le_ubuntu_cmd.bundle" if ($k =~ /ubuntuEL/);
|
|
$totalcase = `cat $batbundle|wc -l`;
|
|
$totalcase += 2;
|
|
}
|
|
|
|
|
|
if ($v == 0) { # whole regression test finished
|
|
my $h = int($consumptionrecord{$k} / 3600);
|
|
my $m = int(($consumptionrecord{$k} - $h * 3600) / 60);
|
|
my $s = ($consumptionrecord{$k} - $h * 3600) % 60;
|
|
my $consumption = "$h hours $m minutes $s seconds";
|
|
$mailreport .= "\tTime consumption: $consumption\n";
|
|
|
|
if ($piderrrecord{$k} ne "") {
|
|
$mailreport .= "\tDeploy MN result: $piderrrecord{$k}\n";
|
|
$mailreport .= "\tTotalCase $totalcase TotalRun 0 Failed 0\n";
|
|
next;
|
|
}
|
|
|
|
my $totalcnt = 0;
|
|
my $failcnt = 0;
|
|
my $faillist = "";
|
|
opendir(DIR, "$todayregresultdir/$k");
|
|
foreach my $file (readdir DIR)
|
|
{
|
|
next if ($file !~ /xcattest.log/);
|
|
my $cnt = `cat $todayregresultdir/$k/$file |grep -- "------END::"|wc -l`;
|
|
$totalcnt += int($cnt);
|
|
for (my $i = 1 ; $i < $cnt + 1 ; $i++) {
|
|
my $line = `cat $todayregresultdir/$k/$file |grep -- "------END::"|sed -n ${i}p`;
|
|
chomp($line);
|
|
if ($line =~ /------END::([a-zA-Z0-9_-]+)::([a-zA-Z0-9_-]+)::Time.+/) {
|
|
my $failedcase = $1;
|
|
if ($2 =~ /Failed/) {
|
|
$failcnt++;
|
|
$faillist .= $failedcase . ", ";
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
closedir(DIR);
|
|
|
|
my $lastfile = `ls -lrt $todayregresultdir/$k|grep "xcattest.log"|tail -1|awk '{print \$9}'`;
|
|
chomp($lastfile);
|
|
my $lastline = `cat $todayregresultdir/$k/$lastfile |grep -- "------END::"|tail -1`;
|
|
chomp($lastline);
|
|
my $lastcase;
|
|
$lastcase = $1 if ($lastline =~ /------END::([a-zA-Z0-9_-]+)::([a-zA-Z0-9_-]+)::Time.+/);
|
|
|
|
$mailreport .= "\tTotalCase $totalcase TotalRun $totalcnt Failed $failcnt\n\tFailed cases: $faillist\n\tFinal environment stop at $lastcase\n";
|
|
} else { # regression test out of time
|
|
$mailreport .= "\tTime consumption: Out of 9 hours!!!!!!\n";
|
|
}
|
|
}
|
|
|
|
&runcmd("touch $mailfile && echo \"$mailreport\" > $mailfile");
|
|
return 0;
|
|
}
|
|
|
|
#######################################
|
|
# send email
|
|
#######################################
|
|
sub send_email {
|
|
my $checkinlist = "";
|
|
my $cnt = `cat $commitinfofile |grep '20[0-9][0-9] [-+]0[0-9]00'|awk '{print \$1}' |grep -v root|sort | uniq|wc -l`;
|
|
my $out = `cat $commitinfofile |grep '20[0-9][0-9] [-+]0[0-9]00'|awk '{print \$1}' |grep -v root|sort | uniq`;
|
|
for (my $i = 1 ; $i < $cnt + 1 ; $i++) {
|
|
my $tmpline = `echo "$out"|sed -n ${i}p`;
|
|
chomp($tmpline);
|
|
$checkinlist .= "$tmpline ";
|
|
}
|
|
|
|
my $totalcnt = 0;
|
|
my $failcnt = 0;
|
|
my $totalcase = 0;
|
|
$cnt = `cat $mailfile |grep "Total"|grep "Failed"|wc -l`;
|
|
$out = `cat $mailfile |grep "Total"|grep "Failed"`;
|
|
for (my $i = 1 ; $i < $cnt + 1 ; $i++) {
|
|
my $tmpline = `echo "$out"|sed -n ${i}p`;
|
|
chomp($tmpline);
|
|
if ($tmpline =~ /TotalCase (\d+) TotalRun (\d+) Failed (\d+)/) {
|
|
$totalcase += $1;
|
|
$totalcnt += $2;
|
|
$failcnt += $3;
|
|
}
|
|
}
|
|
|
|
# $gitupdateflag = 0 git update out of time
|
|
# $gitupdateflag = 1 there isn't new code checkin in last one day
|
|
# $gitupdateflag = 2 git update failed
|
|
$depupdateflag = `cat $logfiledir/$logfile |grep "updated for xcat dep" `;
|
|
my $subject = "";
|
|
if ($gitupdateflag == 0 || $gitupdateflag == 2) {
|
|
$subject = "[$mailtitle] Git update failed, stop regression test today!!!";
|
|
} elsif ($gitupdateflag == 1 && !$forceregwithoutupdate) {
|
|
$subject = "[$mailtitle] No code checkin, stop regression test today!!!";
|
|
} elsif ($gitupdateflag == 1 && $forceregwithoutupdate) {
|
|
`cat $logfiledir/$logfile |grep "updated for xcat dep" `;
|
|
if ($? == 0) {
|
|
$subject = "[$mailtitle] TotalCase:" . $totalcase . ", TotalRun:" . $totalcnt . ", Fail:" . $failcnt . ", Branch:" . $branch . ", xcatdep updated, Checkin: NO ONE !!";
|
|
} else {
|
|
$subject = "[$mailtitle] TotalCase:" . $totalcase . ", TotalRun:" . $totalcnt . ", Fail:" . $failcnt . ", Branch:" . $branch . ", xcatdep no updated, Checkin: NO ONE !!";
|
|
}
|
|
} else {
|
|
`cat $logfiledir/$logfile |grep "updated for xcat dep" `;
|
|
if ($? == 0) {
|
|
$subject = "[$mailtitle] TotalCase:" . $totalcase . ", TotalRun:" . $totalcnt . ", Fail:" . $failcnt . ", Branch:" . $branch . ", xcatdep updated, Checkin: " . $checkinlist;
|
|
} else {
|
|
$subject = "[$mailtitle] TotalCase:" . $totalcase . ", TotalRun:" . $totalcnt . ", Fail:" . $failcnt . ", Branch:" . $branch . ", xcatdep no updated, Checkin: " . $checkinlist;
|
|
}
|
|
}
|
|
my $mailreport .= "\n\n======================================\n";
|
|
$mailreport .= " Commit Codes Last Day\n";
|
|
$mailreport .= "======================================\n\n";
|
|
|
|
&runcmd("echo \"$mailreport\">> $mailfile");
|
|
&runcmd("cat $commitinfofile >> $mailfile");
|
|
|
|
my $attachfile = "/tmp/failed_case_detail.txt";
|
|
&runcmd("echo \"\" > $attachfile");
|
|
if ($failcnt > 0) {
|
|
opendir(DIR, "$todayregresultdir");
|
|
foreach my $subdir (readdir DIR) {
|
|
next if ($subdir =~ /^\./);
|
|
|
|
#print "subdir-> $subdir\n";
|
|
|
|
my $scenario = "[" . $confkeys{ $subdir . "os" };
|
|
if ($subdir =~ /p/) {
|
|
$scenario .= "+PPC64] ";
|
|
} elsif ($subdir =~ /x/) {
|
|
$scenario .= "+x86_64] ";
|
|
} elsif ($subdir =~ /E/) {
|
|
$scenario .= "+ppc64le] ";
|
|
}
|
|
|
|
my $separator .= "\n###############################################################\n";
|
|
$separator .= " $scenario\n";
|
|
$separator .= "###############################################################\n\n";
|
|
&runcmd("echo \"$separator\" >> $attachfile");
|
|
opendir(SUBDIR, "$todayregresultdir/$subdir");
|
|
foreach my $file (readdir SUBDIR) {
|
|
next if ($file !~ /^failedcases/);
|
|
next if (-z "$todayregresultdir/$subdir/$file");
|
|
|
|
#print "\t$file\n";
|
|
system("cat $todayregresultdir/$subdir/$file >> $attachfile");
|
|
}
|
|
}
|
|
}
|
|
|
|
if ($mail_list) {
|
|
if ($failcnt > 0) {
|
|
system("cat $mailfile" . " | /bin/mail -s \"$subject\" -a $attachfile \"$mail_list\"");
|
|
} else {
|
|
system("cat $mailfile" . " | /bin/mail -s \"$subject\" \"$mail_list\"");
|
|
}
|
|
} else {
|
|
if ($failcnt > 0) {
|
|
system("cat $mailfile" . " | /bin/mail -s \"$subject\" -a $attachfile \"$confkeys{mailgroup}\"");
|
|
} else {
|
|
system("cat $mailfile" . " | /bin/mail -s \"$subject\" \"$confkeys{mailgroup}\"");
|
|
}
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
|
|
#######################################
|
|
#git update
|
|
#return value:
|
|
#0 successful
|
|
#1 code is already at latest version
|
|
#2 someting error
|
|
#######################################
|
|
sub git_update {
|
|
send_msg(2, "[git update] starting to update xcat source code from git");
|
|
|
|
my $res = system("cd $xcatcoredir && git checkout $branch > /dev/nul 2>&1");
|
|
if ($res != 0) {
|
|
send_msg(0, "[git update] change to branch $branch....[failed]");
|
|
return 2;
|
|
}
|
|
send_msg(2, "[git update] change to branch $branch....[ok]");
|
|
|
|
my $gitpulloutput = "/tmp/gitpulloutput";
|
|
my $trytime = 1;
|
|
while ($trytime < 4) {
|
|
$res = system("cd $xcatcoredir && git pull > $gitpulloutput 2>&1");
|
|
if ($res != 0) {
|
|
send_msg(0, "[git update] pull the latest code to control node....[$trytime failed]");
|
|
$trytime++;
|
|
&runcmd("sleep 120");
|
|
} else {
|
|
last;
|
|
}
|
|
}
|
|
|
|
if ($trytime == 4) {
|
|
send_msg(0, "[git update] pull the latest code to control node....[finally failed]");
|
|
return 2;
|
|
}
|
|
|
|
$res = system("grep 'Already up-to-date' $gitpulloutput > /dev/nul 2>&1");
|
|
if ($res == 0 && !$forceregwithoutupdate) {
|
|
send_msg(2, "[git update] code is already at latest version. exit regresson");
|
|
return 1;
|
|
} elsif ($res == 0 && $forceregwithoutupdate) {
|
|
send_msg(2, "[git update] code is already at latest version. But force to run regresson");
|
|
return 0;
|
|
}
|
|
send_msg(2, "[git update] pull the latest code to control node....[ok]");
|
|
|
|
my $lastcommitfile = "/tmp/lastcommitfile";
|
|
my $lastcommitinfo;
|
|
if (-f $lastcommitfile) {
|
|
$lastcommitinfo = `cat $lastcommitfile`;
|
|
chomp($lastcommitinfo);
|
|
} else {
|
|
$lastcommitinfo = "";
|
|
}
|
|
|
|
#print "lastcommitinfo=$lastcommitinfo\n";
|
|
|
|
if ($lastcommitinfo eq "") {
|
|
$res = system("cd $xcatcoredir && git log --since=\"1 days ago\" --pretty=format:\"%cn %ad %s\" --name-status > $commitinfofile");
|
|
if ($res != 0 || (-f $commitinfofile and -z _)) {
|
|
send_msg(0, "[git update] get commit information....[failed]");
|
|
return 2;
|
|
}
|
|
send_msg(0, "[git update] get commit information....[ok]");
|
|
} else {
|
|
$res = system("cd $xcatcoredir && git log $lastcommitinfo..HEAD --pretty=format:\"%cn %ad %s\" --name-status > $commitinfofile");
|
|
if ($res != 0 || (-f $commitinfofile and -z _)) {
|
|
send_msg(0, "[git update] get commit information....[failed]");
|
|
return 2;
|
|
}
|
|
send_msg(2, "[git update] get commit information....[ok]");
|
|
}
|
|
|
|
$res = system("cd $xcatcoredir && git log -n 1|awk 'NR==1'|awk '{print \$2}' > $lastcommitfile");
|
|
if ($res != 0) {
|
|
send_msg(0, "[git update] get latest commit number failed");
|
|
exit;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
#######################################
|
|
# step 0. Parse input arguments
|
|
#######################################
|
|
send_msg(2, "step 1, Parse input arguments............");
|
|
|
|
=pod
|
|
if (
|
|
!GetOptions("--help|h|?" => \$needhelp,
|
|
"f=s" => \$configfile,
|
|
"b=s" => \$branch,
|
|
"m=s" => \$management_node,
|
|
"V" => \$verbose_mode,
|
|
"U" => \$updates_regression,
|
|
"e=s" => \$send_msg)
|
|
)
|
|
=cut
|
|
|
|
if (
|
|
!GetOptions("--help|h|?" => \$needhelp,
|
|
"f=s" => \$configfile,
|
|
"b=s" => \$branch,
|
|
"m=s" => \$management_node,
|
|
"a" => \$forceregwithoutupdate,
|
|
"e=s" => \$mail_list)
|
|
)
|
|
{
|
|
&usage;
|
|
send_msg(0, "step 0, PARSE ARGUMENTS returns error, exit");
|
|
exit;
|
|
}
|
|
if ($needhelp)
|
|
{
|
|
&usage;
|
|
exit 0;
|
|
}
|
|
|
|
|
|
xCATreg_init();
|
|
|
|
###############################################################
|
|
# Mainfunction
|
|
###############################################################
|
|
send_msg(2, "........................");
|
|
send_msg(2, "........................");
|
|
send_msg(2, ".....ooooO..............");
|
|
send_msg(2, "....(....)....Ooooo.....");
|
|
send_msg(2, ".....\\..(.....(....)....");
|
|
send_msg(2, "......\\__).....)../.....");
|
|
send_msg(2, "..............(_ /......");
|
|
send_msg(2, "........................");
|
|
send_msg(2, ".........START .........");
|
|
send_msg(2, "........................");
|
|
#######################################
|
|
# step 1. Read configuration files
|
|
#######################################
|
|
send_msg(2, "step 2, Reading xCATreg conf............");
|
|
my $rst = read_conf();
|
|
unless (!$rst) {
|
|
send_msg(0, "read xCATreg configuration failed...........exit");
|
|
exit;
|
|
}
|
|
|
|
foreach my $k (keys %confkeys) {
|
|
print "$k = $confkeys{$k}\n";
|
|
}
|
|
|
|
# begin to find which mn should be used
|
|
if ($management_node) {
|
|
my @all;
|
|
if ($management_node =~ /,/) {
|
|
@all = split /,/, $management_node;
|
|
} else {
|
|
push @all, $management_node;
|
|
}
|
|
foreach my $mn (@all) {
|
|
foreach my $k (keys %confkeys) {
|
|
if ($confkeys{$k} eq $mn) {
|
|
$mns{$k} = $mn;
|
|
send_msg(2, "specify mn :$mn");
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
pipe PREAD, CWRITE;
|
|
my $gitpid = fork();
|
|
if (!defined($gitpid)) {
|
|
send_msg(0, "fork process for git update error");
|
|
exit;
|
|
} elsif ($gitpid == 0) {
|
|
close PREAD;
|
|
send_msg(2, "..........fork 1 process[pid=$$] for git update..........");
|
|
my $res = git_update();
|
|
if ($res > 1) {
|
|
send_msg(0, "git_update......[failed]");
|
|
syswrite CWRITE, "git update failed\n";
|
|
close CWRITE;
|
|
exit 1;
|
|
}
|
|
if ($res == 1) {
|
|
send_msg(2, "git_update......[not find new code]");
|
|
syswrite CWRITE, "git update got nothing\n";
|
|
close CWRITE;
|
|
exit 1;
|
|
}
|
|
send_msg(2, "git_update......[ok]");
|
|
syswrite CWRITE, "git update successful\n";
|
|
close CWRITE;
|
|
exit 0;
|
|
}
|
|
|
|
{ #main process
|
|
close CWRITE;
|
|
my $gitstarttime = time();
|
|
my $gitprocess = 1;
|
|
my $gitselect = new IO::Select;
|
|
$gitselect->add(\*PREAD);
|
|
while ($gitprocess) {
|
|
my @hdls;
|
|
if (@hdls = $gitselect->can_read(0)) {
|
|
my $hdl;
|
|
foreach $hdl (@hdls) {
|
|
if ($hdl == \*PREAD) {
|
|
my $line;
|
|
chomp($line = <PREAD>);
|
|
if ($line) {
|
|
if ($line =~ /successful/) {
|
|
send_msg(2, "[[main]]: $line");
|
|
$gitupdateflag = 3;
|
|
}
|
|
if ($line =~ /failed/) {
|
|
send_msg(0, "[[main]]: $line");
|
|
$gitupdateflag = 2;
|
|
}
|
|
if ($line =~ /nothing/) {
|
|
send_msg(2, "[[main]]: there isn't new code checkin during last one day.");
|
|
$gitupdateflag = 1;
|
|
}
|
|
$gitprocess = 0;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if (time() - $gitstarttime > 7200) {
|
|
|
|
#if(time() - $gitstarttime > 2) {
|
|
send_msg(0, "[[main]] 2 hours is expired. git update out of time");
|
|
last;
|
|
}
|
|
&runcmd("sleep 2");
|
|
}
|
|
close PREAD;
|
|
|
|
&runcmd("kill $gitpid") if ($gitprocess == 1);
|
|
send_msg(2, "[[main]]........git process[$gitpid] exit successful..............");
|
|
|
|
# $gitupdateflag = 0 git update out of time
|
|
# $gitupdateflag = 1 there isn't new code checkin in last one day
|
|
# $gitupdateflag = 2 there is something wrong during git update
|
|
if ($gitupdateflag < 3) {
|
|
send_msg(2, "[[main]]: creat regression result report...........");
|
|
$rst = creat_report();
|
|
if ($rst) {
|
|
send_msg(0, "[[main]]: creat regression result report...........[FAILED]");
|
|
} else {
|
|
send_msg(2, "[[main]]: creat regression result report...........[OK]");
|
|
|
|
send_msg(2, "[[main]]: send report email...........");
|
|
$rst = send_email();
|
|
if ($rst) {
|
|
send_msg(0, "[[main]]: send report email...........[FAILED]");
|
|
} else {
|
|
send_msg(2, "[[main]]: send report email...........[OK]");
|
|
}
|
|
}
|
|
exit 1;
|
|
}
|
|
send_msg(2, "[[main]]: git update successful, start regression today........");
|
|
|
|
#exit 0
|
|
}
|
|
|
|
|
|
my $platformfork = 0;
|
|
my $timingork = 0;
|
|
|
|
# begin child process
|
|
pipe CONTROLREAD, MNWRITE;
|
|
foreach my $m (keys %mns) {
|
|
$platformfork += 1;
|
|
my $mn = $mns{$m};
|
|
my $pid = fork();
|
|
if (!defined($pid)) {
|
|
send_msg(0, "fork process for $m error");
|
|
exit;
|
|
} elsif ($pid == 0) { # child process
|
|
send_msg(2, "..........fork process[pid=$$] for $m [$confkeys{$m}]..........");
|
|
close CONTROLREAD;
|
|
my $res;
|
|
|
|
#######################################
|
|
# step 3. download xcat core
|
|
########################################
|
|
send_msg(2, "[$m]step 3, Running down_xcat_core...............");
|
|
$res = down_xcat_core_build($m);
|
|
if ($res) {
|
|
syswrite MNWRITE, "REPORTFROM:$m->$mn: download xcat core for $mn failed\n";
|
|
exit 1;
|
|
}
|
|
send_msg(2, "[$m]step 3, Run down_xcat_core...............[OK]");
|
|
|
|
#######################################
|
|
# step 4. Install MNs,
|
|
#######################################
|
|
send_msg(2, "[$m]step 4, Running mn_install...............");
|
|
$res = mn_install($m);
|
|
if ($res) {
|
|
syswrite MNWRITE, "REPORTFROM:$m->$mn: install $mn failed\n";
|
|
exit 1;
|
|
}
|
|
send_msg(2, "[$m]step 4, Run mn_install...............[OK]");
|
|
#######################################
|
|
# step 5. Down xcatdep to MNs
|
|
#######################################
|
|
send_msg(2, "[$m]step 5, Down xcatdep...............");
|
|
$depupdateflag = down_dep($m);
|
|
if ($depupdateflag == 2) {
|
|
send_msg(2, "[$m]step 5, updated for xcat dep");
|
|
} else {
|
|
send_msg(2, "[$m]step 5, no update for xcat dep");
|
|
}
|
|
#######################################
|
|
# step 6. Copy code to MNs
|
|
#######################################
|
|
send_msg(2, "[$m]step 6, Running copy_code...............");
|
|
$res = copy_code($m);
|
|
if ($res) {
|
|
syswrite MNWRITE, "REPORTFROM:$m->$mn: copy xcat source code to $mn failed\n";
|
|
exit 1;
|
|
}
|
|
send_msg(2, "[$m]step 6, Run copy_code...............[OK]");
|
|
|
|
########################################
|
|
# step 7. Read xCAT MN's configuration
|
|
#######################################
|
|
send_msg(2, "[$m]step 7, Running install_xcat...............");
|
|
$res = install_xcat($m);
|
|
if ($res) {
|
|
syswrite MNWRITE, "REPORTFROM:$m->$mn: install new xcat in $mn failed\n";
|
|
exit 1;
|
|
}
|
|
send_msg(2, "[$m]step 7, Run install_xcat...............[OK]");
|
|
|
|
#######################################
|
|
# step 8. Genrate local configuration file for xcattest
|
|
# Do test
|
|
# Write log
|
|
#######################################
|
|
send_msg(2, "[$m]step 8, Running do_test...............");
|
|
$res = do_test($m);
|
|
if ($res) {
|
|
syswrite MNWRITE, "REPORTFROM:$m->$mn: run regression test cases in $mn failed\n";
|
|
exit 1;
|
|
}
|
|
send_msg(2, "[$m]step 8, Running do_test...............[OK]");
|
|
|
|
send_msg(2, "whole regression test for $mn finished");
|
|
syswrite MNWRITE, "REPORTFROM:$m->$mn: whole regression test are successful\n";
|
|
exit 0;
|
|
} # end of child process
|
|
&runcmd("mkdir -p $todayregresultdir/$m");
|
|
$pidrecord{$m} = $pid;
|
|
$piderrrecord{$m} = "";
|
|
} # end of foreach
|
|
|
|
|
|
{ # parent process
|
|
close MNWRITE;
|
|
my $rst;
|
|
my $regstarttime = time();
|
|
|
|
my $select = new IO::Select;
|
|
$select->add(\*CONTROLREAD);
|
|
while ($platformfork) {
|
|
my @hdls;
|
|
if (@hdls = $select->can_read(0)) {
|
|
my $hdl;
|
|
foreach $hdl (@hdls) {
|
|
if ($hdl == \*CONTROLREAD) {
|
|
my $line;
|
|
chomp($line = <CONTROLREAD>);
|
|
if ($line) {
|
|
if ($line =~ /successful/) {
|
|
send_msg(2, "[[main]]: $line");
|
|
$line =~ s/^REPORTFROM:(.+)->.+/$1/g;
|
|
$pidrecord{$line} = 0;
|
|
$consumptionrecord{$line} = time() - $regstarttime;
|
|
}
|
|
if ($line =~ /failed/) {
|
|
send_msg(0, "[[main]]: $line");
|
|
my $tmp = $line;
|
|
$line =~ s/^REPORTFROM:(.+)->.+/$1/g;
|
|
$pidrecord{$line} = 0;
|
|
$consumptionrecord{$line} = time() - $regstarttime;
|
|
$tmp =~ s/REPORTFROM:.+->.+:(.+)/$1/g;
|
|
$piderrrecord{$line} = $tmp;
|
|
}
|
|
$platformfork -= 1;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if (time() - $regstarttime > 36000) {
|
|
|
|
#if($curtime - $regstarttime > 10) {
|
|
send_msg(1, "[timing] 8 hours is expired");
|
|
last;
|
|
}
|
|
&runcmd("sleep 1");
|
|
}
|
|
|
|
close CONTROLREAD;
|
|
if ($platformfork) {
|
|
send_msg(0, "[[main]]: there are $platformfork platform's regression test return out of time");
|
|
while (my ($k, $v) = each %pidrecord) {
|
|
&runcmd("kill $v") if ($v != 0);
|
|
}
|
|
} else {
|
|
send_msg(2, "[[main]]: all platform's regression test return on time");
|
|
}
|
|
|
|
send_msg(2, "[[main]]: creat regression result report...........");
|
|
$rst = creat_report();
|
|
if ($rst) {
|
|
send_msg(0, "[[main]]: creat regression result report...........[FAILED]");
|
|
} else {
|
|
send_msg(2, "[[main]]: creat regression result report...........[OK]");
|
|
|
|
send_msg(2, "[[main]]: send report email...........");
|
|
$rst = send_email();
|
|
if ($rst) {
|
|
send_msg(0, "[[main]]: send report email...........[FAILED]");
|
|
} else {
|
|
send_msg(2, "[[main]]: send report email...........[OK]");
|
|
}
|
|
}
|
|
send_msg(2, "[[main]]: main process exit");
|
|
exit 0;
|
|
}
|
|
|
|
|