#!/usr/bin/env perl # IBM(c) 2010 EPL license http://www.eclipse.org/legal/epl-v10.html # ############################################################################################################################################# # # This script will be used to run xCAT daily regression # # Usage:xCATreg - Run xCAT daily regression # xCATreg [-?|-h] # xCATreg [-b branch][-m MN][-f configure file][-a][-V] # # -?/-h: help information for the xCATreg usage. # -b: when -b is specified, the regression will be done against the specified branch. You can use 2.7, 2.8,master.The default value is master. # -m: when -m is specified, the regression will be done the specified mn. The default value is all the mn in configuration file. # -f :specify the configuration file,default value is /xCATreg/regresson.conf # -a: when -a is specified, run the regression even there is no code updates # -V: log and message in verbose mode # # Example usage: # ./xCATreg Run regressoins on master branch and on all platforms in configuration file # ./xCATreg -b 2.8 -m slesmn -f /root/regressoin.conf -a Run regression on 2.8 branch on slesmn even when there is no code updates. # ############################################################################################################################################# use strict; use warnings; use Getopt::Long; use Data::Dumper; use Term::ANSIColor; use Time::Local; BEGIN { $::XCATROOT = $ENV{'XCATROOT'} ? $ENV{'XCATROOT'} : -d '/opt/xcat' ? '/opt/xcat' : '/usr'; } use lib "$::XCATROOT/lib/perl"; ####################################### # usage for arguments ####################################### sub usage { print "Usage:xCATreg - Run xcat autotest test cases.\n Explanation for the options: -b: when -b is specified, the regression will be done against the specified branch. The default value is master.\n -m: when -m is specified, run regression on the specified mn. The default value is all the mn in configuration file.\n -f: specify the configuration file location.\n -U: when -U is specified, only code updates can trigger the regression.\n -V: log and message in verbose mode.\n -e: send the test result to email_addr\n"; print " xCATreg [-?|-h]\n"; print " xCATreg [-f configure file] [-b branch] [-m mangement node][-V][-e] install mn \n"; print " xCATreg [-f configure file] [-b branch] [-m mangement node ] [-U][-V][-e] if code updates there will be regression.\n"; print "\n"; return; } ####################################### # config for rhppc64env ####################################### sub config_rhppc64 { send_msg("******************************"); send_msg("Reading Configure"); send_msg("******************************"); if(!(-e $configfile)){ send_msg("Warning: The xCAT test Configure file doesn't exist!"); send_msg("Warning: The xCAT test Configure file doesn't exist!"); return 0; } my $sub_type = undef; # The string after $type_ # Script--> # Script_Prev # Script_Post # Table---> # Table_xxxxx # Object--> # Object_xxxx # System----> # Custom----> my $name = undef; my $attr = undef; my $value = undef; my $c = 0; my $cmd = undef; my $mgt_name = undef; open(FILE, "$configfile") or die "can't to open $configfile"; while(my $line = ) { $line = &trim($line); next if(length($line) == 0); #Table name can not contain "_" if($line =~ /\[\s*(\w+)\_(\w+)\s*\]/) { $type = $1; $sub_type = $2; $name = undef; $c = 0; }elsif($line =~ /\[\s*rhppc64System|Custom\s*\]/){ $type = "rhppc64Varible"; }elsif ($type eq "rhppc64Table") { ##TABLE BLOCK## if($line =~ /(\w+)\s*=\s*([\w\.\-]+)/) { $attr = $1; $value = $2; if($name&&($rhppc64config{table}{$sub_type}{$name}{__KEY__} ne $attr)){ $rhppc64config{table}{$sub_type}{$name}{$attr}=$value; } else { $name = $value; $rhppc64config{table}{$sub_type}{$name}{__KEY__}=$attr; } } }elsif ($type eq "rhppc64Object") { ##OBJECT BLOCK## if($line =~ /(\w+)\s*=\s*([\w\.\-]+)/) { $attr = $1; $value = $2; #print "rhppc64node attr is $attr\n"; #print "rhppc64node value is $value\n"; if($attr eq "Name"){ $name = $value; } elsif(!defined($name)){ print "Please give name for Object\n"; close FILE; return 1; } else { $rhppc64config{object}{$sub_type}{$name}{$attr}=$value; } } }elsif ($type eq "rhppc64Script") { ##SCRIPT_BLOCK## if($sub_type eq "Prev") { $rhppc64config{script_prev}->[$c] = $line; $c = $c + 1; } elsif ($sub_type eq "rhppc64Post") { $rhppc64config{script_post}->[$c] = $line; $c = $c + 1; } } elsif ($type eq "rhppc64Varible") { ##NODE_BLOCK## if($line =~ /(\w+)\s*=\s*([\w\.\-\/]+)/) { $rhppc64config{var}{$1} = $2; print "var $1,$2\n"; } } } if(exists $rhppc64config{object}){ foreach my $type (keys %{$rhppc64config{object}}){ foreach my $name (keys %{$rhppc64config{object}{$type}}){ send_msg("OBJECT:$name,TYPE:$type"); #print "$name,TYPE:$type \n"; foreach my $attr (keys %{$rhppc64config{object}{$type}{$name}}){ send_msg(" $attr = $config{object}{$type}{$name}{$attr};"); } } } } if(exists $rhppc64config{table}){ foreach my $type (keys %{$rhppc64config{table}}){ send_msg("TABLE:$type"); foreach my $name (keys %{$rhppc64config{table}{$type}}){ send_msg(" $rhppc64config{table}{$type}{$name}{__KEY__} = $name"); foreach my $attr (keys %{$rhppc64config{table}{$type}{$name}}){ if($attr ne '__KEY__'){ send_msg(" $attr = $rhppc64config{table}{$type}{$name}{$attr}"); } } rhppc64 send_msg("\n"); } } } if(exists $rhppc64config{script_prev}){ send_msg("Script_Prev:"); foreach $cmd (@{$config{script_prev}}){ send_msg(" $cmd"); } } if(exists $rhppc64config{script_post}){ send_msg("Script_Post:"); foreach $cmd (@{$config{script_post}}){ send_msg(" $cmd"); } } if (exists $rhppc64config{var}){ #my $MN=$rhppc64config{var}{MN}; #my $MNIP=$rhppc64config{var}{MNIP}; #&runcmd( "echo $MN $MN.$DOMAIN $MNIP>>/etc/hosts"); #print "MN is $MN\n";} send_msg("Varible:"); foreach my $varname (keys %{$rhppc64config{var}}){ send_msg(" $varname = $config{var}{$varname}"); #print "var is $rhppc64config{var}\n"; } } close FILE; return 0; } ####################################### # install xcat and init rhppc64 env ####################################### sub init { if (exists $rhppc64config{var}){ my $MN=$rhppc64config{var}{MN}; my $MNIP=$rhppc64config{var}{MNIP}; my $CN=$rhppc64config{var}{CN}; my $CNIP=$rhppc64config{var}{CNIP}; my $SN=$rhppc64config{var}{SN}; my $SNIP=$rhppc64config{var}{SNIP}; my $SNCN=$rhppc64config{var}{SNCN}; my $SNCNIP=$rhppc64config{var}{SNCNIP}; my $HMC=$rhppc64config{var}{HMC}; my $HMCIP=$rhppc64config{var}{HMCIP}; my $MOUNTIP=$rhppc64config{var}{MOUNTIP}; my $DOMAIN=$rhppc64config{var}{DOMAIN}; my $nodedir=$rhppc64config{var}{nodedir}; system("xdsh $MN mkdir -p /iso/mountpoint"); print "--prepareing redhat iso file.......\n"; print "--mount NF ......."; system ("scp -r /iso/RHEL6.4-20130130.0-Server-ppc64-DVD1.iso $MNIP:/iso/mountpoint"); system("xdsh $MN mount -o loop /iso/mountpoint/RHEL6.4-20130130.0-Server-ppc64-DVD1.iso /iso"); &runcmd( " echo $HMCIP $HMC.$DOMAIN $HMC>>/etc/hosts.rhppc64"); &runcmd( " echo $MNIP $MN.$DOMAIN $MN>>/etc/hosts.rhppc64"); &runcmd( " echo $CNIP $CN.$DOMAIN $CN>>/etc/hosts.rhppc64"); &runcmd( " echo $SNCNIP $SNCN.$DOMAIN $SNCN>>/etc/hosts.rhppc64"); system ("scp -r /etc/hosts.rhppc64 $MN:/etc/hosts"); &runcmd(" cp /etc/resolve.conf /etc/resolve.config.rhppc64"); &runcmd(" echo nameserver $MNIP >>/etc/resolve.config.rhppc64"); system ("scp -r /etc/resolve.conf.rhppc64 $MOUNTIP:/etc/resolve.conf"); print "[OK]\n"; print "--get the latest XCAT tarball.......\n"; system("xdsh $MN rm -rf /etc/yum.repos.d/*"); system("scp -r rhel6.4.repo $MN:/etc/yum.repos.d/rhel6.4.repo"); ######## redhat 6.4 system("xdsh $MN perl $nodedir/xcatbuild/xcat-core/mklocalrepo.sh"); system("xdsh $MN perl $nodedir/xcatbuild/xcat-dep/rh6/ppc64/mklocalrepo.sh"); print "--install XCAT .......\n"; system("xdsh $MN yum clean metadata"); system("xdsh $MN rpm --import /iso/mountpoint/RPM-GPG-KEY-redhat-release"); system("xdsh $MN yum -y install xCAT"); print "--install XCATTEST .......\n"; system("xdsh $MN yum -y install xCAT-test"); print "--install createrepo .......\n"; system("xdsh $MN yum -y install createrepo"); system("xdsh $MN yum -y install screen"); system("xdsh $MN yum -y install mysql-server mysql mysql-bench mysql-devel mysql-connector-odbc"); system("xdsh $MN rm -rf ./result"); system("xdsh $MN mkdir result"); system("xdsh $MN sysctl -n net.ipv4.ip_forward=1"); send_msg( " rhppc64 env is ready\n"); } # if($restore){ # send_msg("******************************"); # send_msg("Backup current xCAT database"); # sned_msg("******************************"); # &runcmd("mkdir -p /tmp/xCATdbbackup"); # &runcmd("dumpxCATdb -p /tmp/xCATdbbackup"); # if($::RUNCMD_RC != 0){ # &send_msg("Fail to backup xCAT database"); # &runcmd("rm -rf /tmp/xCATdbbackup"); # $restore = 0; # return 1; #} #} # send_msg("******************************"); # send_msg("Initialize xCAT test evironment"); # send_msg("******************************"); my $cmd = undef; foreach $cmd (@{$rhppc64config{script_prev}}){ send_msg("$cmd"); # &runcmd($cmd); system("xdsh $MN $cmd"); if($::RUNCMD_RC != 0){ &send_msg("Fail to run $cmd"); return 1; } } if(exists $rhppc64config{object}){ foreach my $type (keys %{$rhppc64config{object}}){ foreach my $name (keys %{$rhppc64config{object}{$type}}){ $cmd = "chdef -t $type -o $name"; foreach my $attr (keys %{$rhppc64config{object}{$type}{$name}}){ $cmd = $cmd." $attr=$rhppc64config{object}{$type}{$name}{$attr}"; } send_msg($cmd); my $MN=$rhppc64config{var}{MN}; # xdsh $MN &runcmd($cmd); system("xdsh $MN $cmd"); if($::RUNCMD_RC != 0){ send_msg("Fail to run var $cmd"); return 1; } } } } if(exists $rhppc64config{table}){ foreach my $type (keys %{$rhppc64config{table}}){ foreach my $name (keys %{$rhppc64config{table}{$type}}){ $cmd = "chtab $rhppc64config{table}{$type}{$name}{__KEY__}=$name"; foreach my $attr (keys %{$rhppc64config{table}{$type}{$name}}){ if($attr ne '__KEY__'){ $cmd = $cmd." $type.$attr=$rhppc64config{table}{$type}{$name}{$attr}"; } } send_msg($cmd); # &runcmd($cmd); system("xdsh $MN $cmd"); if($::RUNCMD_RC != 0){ &send_msg("Fail to run table $cmd"); return 1; } } } } return 0; } ####################################### # run all test ####################################### sub do_test { my $MN=$rhppc64config{var}{MN}; $fullinst=0; $dsklsinst=0; $bundlerun=1; $stateliteinst=0; $snfullinst=0; $dsklscnsninst=0; $fullcnsninst=0; $statelitecnsninst=0; print "Start to run diskless installation ...\n"; send_msg("******************************"); send_msg("start diskless test"); send_msg("******************************"); system("xdsh $MN mkdir -p /autotest/result"); if($dsklsinst){ system("xdsh $MN rm -f /opt/xcat/share/xcat/tools/autotest/testcase/installation/linux_diskless_installation_vm"); system("xdsh $MN /opt/xcat/bin/xcattest -t Linux_diskless_installation_flat_ppc64"); $output = system("xdsh $MN ls /opt/xcat/share/xcat/tools/autotest/result/failedcases.*"); if($output =~ /failedcases.(\d+)/){ $timestamp=$1; } system("xdsh $MN mv /opt/xcat/share/xcat/tools/autotest/result/* /autotest/result/"); $output = ("xdsh $MN tail /autotest/result/xcattest.log.$timestamp"); if($output =~ /Failed: (\d+)/){ if($1 != 0){ $send_msg{linux_diskless_installation_flat}{pass} = 0; $send_msg{linux_diskless_installation_flat}{timestamp} = $timestamp; } else { $send_msg{linux_diskless_installation_flat}{pass} = 1; } } } if($bundlerun){ print "Start to run the automation test bucket ....\n"; system("xdsh $MN mkdir -p /autotest/result"); system("xdsh $MN /opt/xcat/bin/xcattest -b /opt/xcat/share/xcat/tools/autotest/bundle/bat.bundle"); $output = system("xdsh $MN ls /opt/xcat/share/xcat/tools/autotest/result/xcattest.log.*"); if($output =~ /xcattest\.log\.(\d+)/){ $timestamp = $1; } system("xdsh $MN mv /opt/xcat/share/xcat/tools/autotest/result/* /autotest/result/"); system("xdsh $MN cp /autotest/result/xcattest.log.$timestamp /autotest/result/log/xcattest.log.$timestamp.current"); $output = ("xdsh $MN tail /autotest/result/xcattest.log.$timestamp"); if($output =~ /Total: (\d+) , Failed: (\d+)/){ $send_msg{command}{total} = $1; $send_msg{command}{fail} = $2; $send_msg{command}{timestamp} = $timestamp; if($send_msg{command}{fail} != 0){ $send_msg{command}{failcase} = "| | | Failed cases:"."\n"; $output = (xdsh $MN cat /autotest/result/failedcases.$timestamp | grep END); while($output =~ /END::(\w+)/g){ $send_msg{command}{failcase} = $send_msg{command}{failcase}."| | | ".$1."\n"; # print "$msg{command}{failcase}"; } print "$send_msg{command}{failcase}"; } } } if($stateliteinst){ system("xdsh $MN /opt/xcat/bin/xcattest -t Linux_statelite_installation_flat_ppc64"); $output = (xdsh $MN ls /opt/xcat/share/xcat/tools/autotest/result/failedcases.*"); if($output =~ /failedcases\.(\d+)/){ $timestamp = $1; } system(xdsh $MN mv /opt/xcat/share/xcat/tools/autotest/result/* /autotest/result/"); system(xdsh $MN cp /autotest/result/xcattest.log.$timestamp /autotest/result/log/xcattest.log.$timestamp.current"); $output = xdsh $MN tail /autotest/result/xcattest.log.$timestamp; if($output =~ /Failed: (\d+)/){ if($1 != 0){ $send_msg{linux_statelite_installation_flat}{pass} = 0; $send_msg{linux_statelite_installation_flat}{timestamp} = $timestamp; } else { $send_msg{linux_statelite_installation_flat}{pass} = 1; } } } if($fullinst){ system("xdsh $MN /opt/xcat/bin/xcattest -t Linux_full_installation_flat_ppc64"); $output = xdsh $MN ls /opt/xcat/share/xcat/tools/autotest/result/failedcases.*; if($output =~ /failedcases\.(\d+)/){ $timestamp = $1; } system(xdsh $MN mv /opt/xcat/share/xcat/tools/autotest/result/* /autotest/result/"); system(xdsh $MN cp /autotest/result/xcattest.log.$timestamp /autotest/result/log/xcattest.log.$timestamp.current"); $output = xdsh $MN tail /autotest/result/xcattest.log.$timestamp; if($output =~ /Failed: (\d+)/){ if($1 != 0){ $send_msg{linux_full_installation_flat}{pass} = 0; $send_msg{linux_full_installation_flat}{timestamp} = $timestamp; } else { $send_msg{linux_full_installation_flat}{pass} = 1; } } } if($snfullinst){ system("xdsh $MN /opt/xcat/bin/xcattest -t Linux_sn_installation_flat_x86_vm"); $output = xdsh $MN ls /opt/xcat/share/xcat/tools/autotest/result/failedcases.*; if($output =~ /failedcases\.(\d+)/){ $timestamp = $1; } xdsh $MN mv /opt/xcat/share/xcat/tools/autotest/result/* /autotest/result/; xdsh $MN cp /autotest/result/xcattest.log.$timestamp /autotest/result/log/xcattest.log.$timestamp.current`; $output = xdsh $MN tail /autotest/result/xcattest.log.$timestamp; if($output =~ /Failed: (\d+)/){ if($1 != 0){ $send_msg{linux_sn_installation_flat}{pass} = 0; $send_msg{linux_sn_installation_flat}{timestamp} = $timestamp; } else { $send_msg{linux_sn_installation_flat}{pass} = 1; } } } if($dsklscnsninst){ system("xdsh $MN /opt/xcat/bin/xcattest -t Linux_cn_with_sn_diskless_installation_flat_x86_vm"); $output = xdsh $MN ls /opt/xcat/share/xcat/tools/autotest/result/failedcases.*; if($output =~ /failedcases\.(\d+)/){ $timestamp = $1; } xdsh $MN mv /opt/xcat/share/xcat/tools/autotest/result/* /autotest/result/; xdsh $MN cp /autotest/result/xcattest.log.$timestamp /autotest/result/log/xcattest.log.$timestamp.current; $output = xdsh $MN tail /autotest/result/xcattest.log.$timestamp; if($output =~ /Failed: (\d+)/){ if($1 != 0){ $send_msg{linux_cn_with_sn_diskless_installation_flat}{pass} = 0; $send_msg{linux_cn_with_sn_diskless_installation_flat}{timestamp} = $timestamp; } else { $send_msg{linux_cn_with_sn_diskless_installation_flat}{pass} = 1; } } } if($statelitecnsninst){ system("xdsh $MN /opt/xcat/bin/xcattest -t Linux_cn_with_sn_statelite_installation_flat_x86_vm"); $output = xdsh $MN ls /opt/xcat/share/xcat/tools/autotest/result/failedcases.*; if($output =~ /failedcases\.(\d+)/){ $timestamp = $1; } xdsh $MN mv /opt/xcat/share/xcat/tools/autotest/result/* /autotest/result/; xdsh $MN cp /autotest/result/xcattest.log.$timestamp /autotest/result/log/xcattest.log.$timestamp.current; $output = xdsh $MN tail /autotest/result/xcattest.log.$timestamp; if($output =~ /Failed: (\d+)/){ if($1 != 0){ $send_msg{linux_cn_with_sn_statelite_installation_flat}{pass} = 0; $send_msg{linux_cn_with_sn_statelite_installation_flat}{timestamp} = $timestamp; } else { $send_msg{linux_cn_with_sn_statelite_installation_flat}{pass} = 1; } } } if($fullcnsninst){ system("xdsh $MN /opt/xcat/bin/xcattest -t Linux_cn_with_sn_full_installation_flat_x86_vm"); $output = xdsh $MN ls /opt/xcat/share/xcat/tools/autotest/result/failedcases.*; if($output =~ /failedcases\.(\d+)/){ $timestamp = $1; } xdsh $MN mv /opt/xcat/share/xcat/tools/autotest/result/* /autotest/result/; xdsh $MN cp /autotest/result/xcattest.log.$timestamp /autotest/result/log/xcattest.log.$timestamp.current; $output = xdsh $MN tail /autotest/result/xcattest.log.$timestamp; if($output =~ /Failed: (\d+)/){ if($1 != 0){ $send_msg{linux_cn_with_sn_full_installation_flat}{pass} = 0; $send_msg{linux_cn_with_sn_full_installation_flat}{timestamp} = $timestamp; } else { $send_msg{linux_cn_with_sn_full_installation_flat}{pass} = 1; } } } system("mkdir -p $nodedir/result"); system("scp -r $MN:/autotest/result $nodedir/result"); } ####################################### # runcmd ####################################### sub runcmd { my ($cmd) = @_; my $rc = 0; $::RUNCMD_RC = 0; my $outref = []; @$outref = `$cmd 2>&1`; if ($?) { $rc = $? ; $::RUNCMD_RC = $rc; } chomp(@$outref); return @$outref; } sub trim { my $str = shift @_; if($str){ # $str =~ s/\#/__wellnumber__/g; $str =~ s/^\s+|#.+|\s+$//g; # $str =~ s/__wellnumber__/#/g; } return $str; } ######## ####################################### # step 0. Parse input arguments ####################################### my $rootdir = "$::XCATROOT/share/xcat/tools/autotest"; my $needhelp = 0; my $configfile = "$::XCATROOT/share/xcat/tools/autotest"; my $branch = 0; my $configfile = "$rootdir/default.conf"; my $management_node = undef; my $verbose_mode = 0; my $updates_regression = 0; my $send_msg = 0; my $ret = 0; my $string1 = undef; if ( !GetOptions("h|?" => \$needhelp, "f=s" => \$configfile, "b=s" => \$branch, "m=s" => \$management_node, "V" => \$verbose_mode, "U" => \$updates_regression, "e=s" => \$send_msg) ) { &usage; send_msg("PARSE ARGUMENTS returns error, exit"); exit 1; } if ($needhelp) { &usage; exit 0; } send_msg("PARSE ARGUMENTS returns error, exit"); my $res; my $confile = "regconf"; ####################################### # send messages ####################################### sub send_msg { my $msg = shift; print "xcatreg message: $msg.\n"; } ####################################### # read_conf ####################################### sub read_conf{ my %confkeys; my $keys; if (!open($keys, "<$confile")) { send_msg("Open configuration file error"); } my $line; while ($line = <$keys>) { if ($line =~ /end/) { last; } if ($line =~ /^\s*log\s*=\s*(\S*)\s*/) { $confkeys{log} = $1; } if ($line =~ /^\s*mailgroup\s*=\s*(\S*)\s*/) { $confkeys{mailgroup} = $1; } } return %confkeys; ####################################### # step 1. Parse arguments ####################################### my %reghash = arg_phase(); unless (%reghash) { send_msg("PARSE ARGUMENTS returns error, exit"); exit; } ####################################### # step 2. Read configuration files ####################################### my %confhash = read_conf(); unless (%confhash) { send_msg("READ CONF returns error, exit"); exit; } ####################################### # step 3. Install MNs ####################################### $res = mn_install(); if ($res) { send_msg("INSTALL MNS returns error, exit"); exit; } ####################################### # step 4. Copy code to MNs ####################################### $res = copy_code(); if ($res) { send_msg("COPY CODE returns error, exit"); exit; } ####################################### # step 5. Build xcat code in MNs ####################################### $res = build_xcat(); if ($res) { send_msg("BUILD XCAT returns error, exit"); exit; } ####################################### # step 6. Configure xCAT MN ####################################### mkdir $resultdir unless -d $resultdir; my $timestamp = `date +"%Y%m%d%H%M%S"`; my %rhppc64config=(); my %slesppc64config=(); my %rhx8664config=(); my %slesx8664config=(); my @osname = &runcmd("uname -a"); if ( $osname [0] =~ /^Linux\s*/ && -f "/etc/redhat-release" && $osname [0] =~ /ppc64/){ print "ppc64 redhat env\n"; $os="rhels6.4"; $arch="ppc64"; print "os is $os,arch is $arch\n"; $res = &config_rhppc64(); if ($res) { send_msg("CONFIGURE MN returns error, exit"); exit; } } ####################################### # step 7. Genrate local configuration file # Do test # Write log ####################################### ####################################### # step 7.1 Install xcat and init mn # init ###################################### $res = &init; if ($res != 0){ goto EXIT; } $res = &do_test(); if ($res) { send_msg("DO TEST returns error, exit"); exit; } ####################################### # step 8. Copy code and configuration ####################################### $res = pro_result(); if ($res) { send_msg("PROCESS RESULT returns error, exit"); exit; }