#!/usr/bin/env perl
# IBM(c) 2010 EPL license http://www.eclipse.org/legal/epl-v10.html
#
#############################################################################################################################################
#
#   This script will be used to run xCAT daily regression
#
#   Usage:xCATreg - Run xCAT daily regression
#   xCATreg [-?|-h]
#   xCATreg [-b branch][-m MN][-f configure file][-a][-V]
#
#   -?/-h:  help information for the xCATreg usage.
#   -b: when -b is specified, the regression will be done against the specified branch. You can use 2.7, 2.8,master.The default value is master.
#   -m: when -m is specified, the regression will be done the specified mn. The default value is all the mn in configuration file.
#   -f :specify the configuration file,default value is /xCATreg/regresson.conf
#   -a: when -a is specified, run the regression even there is no code updates
#   -V: log and message in verbose mode
#
#   Example usage:
#   ./xCATreg   Run regressoins on master branch and on all platforms in configuration file
#   ./xCATreg -b 2.8 -m slesmn -f /root/regressoin.conf -a    Run regression on 2.8 branch on slesmn even when there is no code updates.
#
#############################################################################################################################################

$ENV{PATH} = "/opt/xcat/bin:/opt/xcat/sbin:/opt/xcat/share/xcat/tools:/usr/sbin:/usr/bin:/sbin:/bin"; 

use strict;
use warnings;
use Getopt::Long;
use Data::Dumper;
use Term::ANSIColor;
use Time::Local;
use IO::Select;

BEGIN
{
    $::XCATROOT = $ENV{'XCATROOT'} ? $ENV{'XCATROOT'} : -d '/opt/xcat' ? '/opt/xcat' : '/usr';
}
use lib "$::XCATROOT/lib/perl";

my $date;
#my $regrootdir="/home/autotest_code";
#my $mailtitle="P Cluster";
my $regrootdir="/xCATreg";
my $mailtitle="X Cluster";
my $logfiledir = "$regrootdir/log";
my $logfile="";
my $configfiledir = "$regrootdir/conf";
my $configfile = "$regrootdir/conf/xCATreg.conf";
my $xcatcoredir = "$regrootdir/xcatsource/xcat-core";
my $xcatdepdir = "$regrootdir/xcatsource/xcat-dep";
my $osisodir = "$regrootdir/iso";
my $regresultdir = "$regrootdir/regresult";
my $todayregresultdir="";
my $maildir="$regrootdir/mail";
my $mailfile="";
my $rootdir = "$::XCATROOT/share/xcat/tools/autotest";
my %confkeys;
my $mail_list = 0;
my $needhelp  = 0;
my $branch = "master";
my $management_node = undef;
#my $verbose_mode = 0;
my $updates_regression = 0;
my %mns;
my %pidrecord;
my %piderrrecord;
my %consumptionrecord;
my $commitinfofile="/tmp/commitinfo";
my $gitupdateflag = 0;
my $forceregwithoutupdate=0;


#######################################
# runcmd
#######################################
sub runcmd {
    my ($cmd) = @_;
    my $rc = 0;
    $::RUNCMD_RC = 0;
    my $outref = [];
    @$outref = `$cmd 2>&1`;
    if ($?)
    {
        $rc = $? ;
    $::RUNCMD_RC = $rc;
    }
    chomp(@$outref);
    return @$outref;
}

#######################################
# clean_outdated_file
#######################################
sub is5daysago{
    my $targetdate= shift;
    my $targetdatey="";
    my $targetdatem="";
    my $targetdated="";
    my $datey="";
    my $datem="";
    my $dated="";
    if($targetdate =~ /(\d{4})(\d{2})(\d{2})/){
        $targetdatey=$1;
        $targetdatem=$2-1;
        $targetdated=$3;
    }
    if($date =~ /(\d{4})(\d{2})(\d{2})/){
        $datey=$1;
        $datem=$2-1;
        $dated=$3;
    }
    my $todays=timelocal(0,0,0,$dated,$datem,$datey);
    my $targets=timelocal(0,0,0,$targetdated,$targetdatem,$targetdatey);
    if($todays - $targets > 432000){
        return 1;
    }
    return 0;
}

#######################################
# clean_outdated_file
#######################################
sub clean_outdated_file{
    opendir(DIR, "$logfiledir");
    foreach my $file (readdir DIR){
        if($file =~ /xCATreg(\d+).log/){
            my $res = is5daysago($1);
            if($res){
                &runcmd("rm -f $logfiledir/$file");
            }
        }
    }
    closedir(DIR);
	
    opendir(DIR, "$maildir");
    foreach my $file (readdir DIR){
        if($file =~ /mailreport.(\d+)/){
            my $res = is5daysago($1);
            if($res){
                &runcmd("rm -f $maildir/$file");
            }
        }
    }
    closedir(DIR);

    opendir(DIR, "$regresultdir");
    foreach my $file (readdir DIR){
        if($file =~ /(\d+)/){
            my $res = is5daysago($1);
            if($res){
                &runcmd("rm -rf $regresultdir/$file");
            }
        }
    }
    closedir(DIR);

    return 0;
}

#######################################
# xCATreg_init
#######################################
sub xCATreg_init{

    #init important attributes
    
    $date = `date  +"%Y%m%d"`; 
    chomp($date);

    &runcmd("mkdir -p $logfiledir") if(! -e $logfiledir);
    &runcmd("mkdir -p $maildir") if(! -e $maildir);
    &runcmd("mkdir -p $regresultdir") if(! -e $regresultdir);
    &runcmd("mkdir -p $xcatcoredir") if(! -e $xcatcoredir);
	
    $logfile="xCATreg".$date.".log";
    $todayregresultdir=$regresultdir."/".$date;
    &runcmd("mkdir -p $todayregresultdir");
    $mailfile=$maildir."/mailreport.".$date;
	
	my $cnt=`ps aux|grep xCATreg |grep -v grep|awk '{print \$2}'|wc -l`;
	my $out=`ps aux|grep xCATreg |grep -v grep|awk '{print \$2}'`;
	for(my $i=1;$i<$cnt+1;$i++){
        my $pid=`echo "$out"|sed -n ${i}p`;
		chomp($pid);
		next if($pid == $$);
		&runcmd("kill $pid");
	}
	
	clean_outdated_file();
	return 0;
}

#######################################
# usage for arguments
#######################################
sub usage{
    print "Usage:xCATreg - Run xcat autotest  test cases.\n
  Explanation for the options:
    -b: when -b is specified, the regression will be done against the specified branch. The default value is master.\n
    -m: when -m is specified, run regression on the specified mn. The default value is all the mn in configuration file.\n
    -f: specify the configuration file location.\n
    -a: when -a is specified, run the regression even there is no code updates.\n
    -e: <email_addr> send the test result to email_addr\n\n";
    print "  xCATreg [-?|-h]\n";
    print "  xCATreg [-f configure file] [-b branch] [-m mangement node] [-e mail list] [-a] Even code not updates, there will be regression.\n";
    print "  xCATreg [-f configure file] [-b branch] [-m mangement node] [-e mail list] if code updates there will be regression.\n";
    print "\n";
    return;
}

#######################################
# send messages
#######################################
sub send_msg {
    my $num = shift;
    my $msg = shift;
    my $content;
    if ($num == 0) {
        $content = "Fatal error:";
    } elsif($num == 1) {
        $content = "Warning:";
    } elsif($num == 2) {
        $content = "Notice:";
    }
    my $timestamp = `date +"%Y%m%d%H%M%S"`;
    chomp($timestamp);
    if ( !open (LOGFILE, ">> $logfiledir/$logfile") ) {
       return 1;
    }
    print LOGFILE "$content $timestamp $$: $msg.\n";
    close LOGFILE;
}


#######################################
# read_conf
#######################################
sub read_conf{
    my $myfile=undef;
	my $line=undef;
   
	if (!open($myfile, "$configfile")) {
        send_msg(0, "Open $configfile failed");
        return 1;
    }
    while ($line = <$myfile>) {
	    $line =~ s/\s//g;
		next if($line =~ /^#/);
		next if($line eq "");
		my @attr=split(/=/,$line);  
        $confkeys{$attr[0]} = $attr[1];
	}
	close($myfile); 
    return 0;	
}

#######################################
# mn_install to install all mn
#######################################
sub mn_install {
    my $mn=shift;
	
	&runcmd("lsdef|grep $confkeys{$mn}");
	if($?){
	    send_msg(0, "[$mn->$confkeys{$mn}][mn_install] Didn't defined $confkeys{$mn} in current control node");
	    return 1;
	}

	my $os = $confkeys{$mn."os"};
	my $arch = "ppc64";
	$arch="x86_64" if($mn !~ /aix/ && $mn =~ /x/); 
    my $osimage="";
	
	if($mn !~ /aix/){
	    $osimage=$os."-".$arch."-install-compute";
	}else{
	    $osimage="aix";     #????????need to test for aix
	}

	#print "[$mn] : os= $os arch=$arch osimage=$osimage\n";
	
	&runcmd("lsdef -t osimage|grep $osimage");
	if($?){
	    send_msg(0, "[$mn->$confkeys{$mn}][mn_install] Didn't defined $osimage in current control node");
	    return 1;
	}
	
	send_msg(2, "[$mn->$confkeys{$mn}][mn_install] installing $osimage for node $confkeys{$mn}................");

	&runcmd("nodeset $confkeys{$mn} osimage=$osimage");
	if( $arch =~ /x86_64/){
		&runcmd("rpower $confkeys{$mn} boot");
	}else{
	    &runcmd("rnetboot $confkeys{$mn}");
	}
	
	if($mn !~ /ubuntu/){
	    &runcmd("sleep 300");
	    &runcmd("a=0;while ! `lsdef -l $confkeys{$mn}|grep status|grep booted >/dev/null`; do sleep 10;((a++));if [ \$a -gt 200 ];then break;fi done");}
	else{
	    &runcmd("sleep 1800");	
	}
	
	&runcmd("lsdef -l $confkeys{$mn}|grep status|grep booted >/dev/null");
	if($?){
        send_msg(0, "[$mn->$confkeys{$mn}][mn_install] install $osimage for node $confkeys{$mn} failed");
        return 1;
	}
	
	if($mn =~ /ubuntu/){
	    system("updatenode $confkeys{$mn} -P \"confignics -s\" >/dev/null 2>&1");  
	    system("xdsh $confkeys{$mn} \"iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE\" >/dev/null 2>&1");   
		system("xdsh $confkeys{$mn} \"iptables -A FORWARD -i eth1 -j ACCEPT\" >/dev/null 2>&1");     
        system("xdsh $confkeys{$mn} \"iptables -A FORWARD -i eth1 -o eth0 -m state --state RELATED,ESTABLISHED -j ACCEPT\" >/dev/null 2>&1");
	}
	
    return 0;
}


#######################################
# copy code
#######################################
sub copy_code {
    my $mn=shift;
	
    send_msg(2, "[$mn->$confkeys{$mn}][copy_code] starting to copy the latest xcat code and dep");

	#copy xcat-dep tarball to MN:/
	my $depball="";
	if($mn =~ /aix/){
	    $depball = `ls -l $xcatdepdir|awk '{print \$9}'|grep aix|tail -n 1`;
	}elsif($mn =~ /ubuntu/){
	    $depball = `ls -l $xcatdepdir|awk '{print \$9}'|grep ubuntu|tail -n 1`;
	}else{
	    $depball = `ls -l $xcatdepdir|awk '{print \$9}'|grep 'xcat-dep-[0-9]\\{12\\}.tar.bz'|tail -n 1`;
	}
	chomp($depball);
	if ($depball eq ""){
	    send_msg(0, "[$mn->$confkeys{$mn}][copy_code] can't find dep tarball for $mn under dir $xcatdepdir");
	    return 1;
	}
	send_msg(2, "[$mn->$confkeys{$mn}][copy_code] find dep tarball $depball for $mn");

    my $res = system("scp $xcatdepdir/$depball root\@$confkeys{$mn}:/  >/dev/null");
    if ($?){
        send_msg(0, "[$mn->$confkeys{$mn}][copy_code] copy $depball to $mn failed");
        return 1;
    }
	send_msg(2, "[$mn->$confkeys{$mn}][copy_code] copy $depball to $mn successfully");
	
	$res = system("xdsh $confkeys{$mn} 'cd / && tar xvf /$depball' >/dev/null 2>&1");
    if ($?){
        send_msg(0, "[$mn->$confkeys{$mn}][copy_code] untar $depball in $mn failed");
        return 1;
    }
	send_msg(2, "[$mn->$confkeys{$mn}][copy_code] untar $depball in $mn successfully");
	
    #copy xcat-core source code to MN:/
	if($mn =~ /ubuntux/){
        $res = system("xdsh $confkeys{$mn} 'mkdir -p /xcatbuild' && scp -r $xcatcoredir root\@$confkeys{$mn}:/xcatbuild/  >/dev/null");
	}else{
        $res = system("scp -r $xcatcoredir root\@$confkeys{$mn}:/  >/dev/null");
	}
    if ($res){
        send_msg(0, "[$mn->$confkeys{$mn}][copy_code] copy the latest xcat code to $mn failed");
        return 1;
    }
	send_msg(2, "[$mn->$confkeys{$mn}][copy_code] copy the latest xcat code to $mn successfully");

    #copy /etc/hosts file to MN
	$res = system("scp /etc/hosts root\@$confkeys{$mn}:/etc  >/dev/null");
	if ($?){
        send_msg(0, "[$mn->$confkeys{$mn}][copy_code] copy /etc/hosts to $mn failed");
        return 1;
    }
	send_msg(2, "[$mn->$confkeys{$mn}][copy_code] copy /etc/hosts  to $mn successfully");

	my $os = $confkeys{$mn."os"};
	my $regconf=$mn.".conf";
	system("sed -i 's|os=\\S*|os=$os|g' $configfiledir/$regconf");
	
	my $arch = "ppc64";
	$arch="x86_64" if($mn !~ /aix/ && $mn =~ /x/);
	my $iso;
	
	#copy specific files to MN for each platform
	if($mn =~ /rh/){
        $res = system("scp /install/$os/$arch/RPM-GPG-KEY-redhat-release root\@$confkeys{$mn}:/  >/dev/null");
        if ($?){
            send_msg(0, "[$mn->$confkeys{$mn}][copy_code] copy RPM-GPG-KEY-redhat-release to $mn failed");
            return 1;
        }
		send_msg(2, "[$mn->$confkeys{$mn}][copy_code] copy RPM-GPG-KEY-redhat-release to $mn successfully");
		
		$os =~ /(\D+)(\d+\.\d+)/;
		$iso=`ls -l $osisodir|awk '{print \$9}'|grep RHEL|grep $arch|grep $2|tail -n 1`;
		chomp($iso);
		#print "[$mn->$confkeys{$mn}][copy_code] iso=$iso\n";
		if ($iso eq ""){
            send_msg(0, "[$mn->$confkeys{$mn}][copy_code] can't find iso for $os and $arch under dir $osisodir");
            return 1;
		}
		send_msg(2, "[$mn->$confkeys{$mn}][copy_code] find $iso for os=$os and arch=$arch");

        $res = system("scp $osisodir/$iso root\@$confkeys{$mn}:/  >/dev/null");
        if ($?){
            send_msg(0, "[$mn->$confkeys{$mn}][copy_code] copy ISO $iso to $mn failed");
            return 1;
        }
		send_msg(2, "[$mn->$confkeys{$mn}][copy_code] copy ISO $iso to $mn successfully");
		system ("sed -i 's|ISO=/\\S*|ISO=/$iso|g' $configfiledir/$regconf");
	}
	
	if($mn =~ /sles/){
        $os =~ /(\D+)(\d+)\.?(\d?)/;
        if($3){
            $iso=`ls -l $osisodir| awk '{print \$9}'|grep SLE|grep $arch |grep -- "-$2-SP$3-" |tail -n 1`;
        }else{
            $iso=`ls -l $osisodir| awk '{print \$9}'|grep SLE|grep $arch |grep -v "SP"|grep -- "-$2-"|tail -n 1`;
        }
        chomp($iso);
		#print "[$mn->$confkeys{$mn}][copy_code] iso=$iso\n";
        if ($iso eq ""){
            send_msg(0, "[$mn->$confkeys{$mn}][copy_code] can't find ISO for os=$os and arch=$arch under dir $osisodir");
            return 1;
		}
		send_msg(2, "[$mn->$confkeys{$mn}][copy_code] find $iso for os=$os and arch=$arch");
		
        $res = system("scp $osisodir/$iso root\@$confkeys{$mn}:/  >/dev/null");
        if ($?){
            send_msg(0, "[$mn->$confkeys{$mn}][copy_code] copy ISO $iso to $mn failed");
            return 1;
        }
		send_msg(2, "[$mn->$confkeys{$mn}][copy_code] copy ISO $iso to $mn successfully");	
        system ("sed -i 's|ISO=/\\S*|ISO=/$iso|g' $configfiledir/$regconf");	

        if($os =~ /sles11/){
            $res = system("xdsh $confkeys{$mn}  \"mv /usr/lib/rpm/find-requires /usr/lib/rpm/find-requires.org\" && scp $osisodir/find-requires $confkeys{$mn}:/usr/lib/rpm/ > /dev/null");
            if ($?){
                send_msg(0, "[$mn->$confkeys{$mn}][copy_code] For sles11, copy find-requires to $mn failed");
                return 1;
            }
            send_msg(2, "[$mn->$confkeys{$mn}][copy_code] For sles11, copy find-requires to $mn successfully");	
        }		
	}
	if($mn =~ /ubuntu/){
        $os =~ /(\D+)(\d+\.\d+\.?\d?)/;
		my $osversion = $2;
        if ($mn =~ /ubuntux/){
            $iso=`ls -l $osisodir| awk '{print \$9}'|grep ubuntu|grep amd64 |grep $osversion |tail -n 1`;
        }elsif($mn =~ /ubuntupel/){
            $iso=`ls -l $osisodir| awk '{print \$9}'|grep ubuntu|grep ppc64el |grep $osversion |tail -n 1`;
		}
		chomp($iso);
		#print "[$mn->$confkeys{$mn}][copy_code] iso=$iso\n";
        if ($iso eq ""){
            send_msg(0, "[$mn->$confkeys{$mn}][copy_code] can't find ISO for os=$os and arch=$arch under dir $osisodir");
            return 1;
		}
		send_msg(2, "[$mn->$confkeys{$mn}][copy_code] find $iso for os=$os and arch=$arch");
		
		$res = system("scp $osisodir/$iso root\@$confkeys{$mn}:/  >/dev/null");
        if ($?){
            send_msg(0, "[$mn->$confkeys{$mn}][copy_code] copy ISO $iso to $mn failed");
            return 1;
        }
		send_msg(2, "[$mn->$confkeys{$mn}][copy_code] copy ISO $iso to $mn successfully");	
        system ("sed -i 's|ISO=/\\S*|ISO=/$iso|g' $configfiledir/$regconf");	
	}
    return 0;
}


#######################
# build xcat
#######################
sub build_xcat {
    my $mn = shift;
	
    #print "=======[$mn->$confkeys{$mn}] build_xcat========\n";
	send_msg(2, "[$mn->$confkeys{$mn}][build_xcat] starting to build xcat in $confkeys{$mn}");
            
    #modify /etc/resolv.conf	
	my $dnsserver=`cat /etc/hosts |grep $confkeys{$mn}|awk '{print \$1}'|head -n 1`;
	chomp($dnsserver);
    my $res = system("xdsh $confkeys{$mn} \"sed -i 's:nameserver .*:nameserver $dnsserver:g' /etc/resolv.conf\" >/dev/null 2>&1");
    if ($res != 0){
        send_msg(0, "[$mn->$confkeys{$mn}][install_xcat] modify resolv.conf in $mn failed");
        return 1;
    }

	if($mn =~ /rh/){
	    system("xdsh $confkeys{$mn} 'yum  -y groupinstall \"Development tools\"' >/dev/null 2>&1");	
	    system("xdsh $confkeys{$mn} 'yum -y groupinstall \"Perl Support\"' >/dev/null 2>&1");
	    system("xdsh $confkeys{$mn} 'yum  -y install createrepo' >/dev/null 2>&1");		
    }
	
    if($mn =~ /sles/){
	    system("xdsh $confkeys{$mn} 'zypper -n install perl-DBI' >/dev/null 2>&1");
    }
	
    if($mn =~ /ubuntu/){
	    system("xdsh $confkeys{$mn} \"echo \"nameserver 9.0.2.1\" >> /etc/resolv.conf\" >/dev/null 2>&1");
	    system("xdsh $confkeys{$mn} 'apt-get -y install software-properties-common' >/dev/null 2>&1");
		if($mn =~ /ubuntux/){
	        system("xdsh $confkeys{$mn} 'add-apt-repository \"deb http://archive.ubuntu.com/ubuntu \$(lsb_release -sc) main\"' >/dev/null 2>&1");
	        system("xdsh $confkeys{$mn} 'add-apt-repository \"deb http://archive.ubuntu.com/ubuntu \$(lsb_release -sc)-updates main\"' >/dev/null 2>&1");
	        system("xdsh $confkeys{$mn} 'add-apt-repository \"deb http://archive.ubuntu.com/ubuntu \$(lsb_release -sc) universe\"' >/dev/null 2>&1");
	        system("xdsh $confkeys{$mn} 'add-apt-repository \"deb http://archive.ubuntu.com/ubuntu \$(lsb_release -sc)-updates universe\"' >/dev/null 2>&1");
		}elsif($mn =~ /ubuntupel/){
	        system("xdsh $confkeys{$mn} 'add-apt-repository \"deb http://ports.ubuntu.com/ubuntu-ports \$(lsb_release -sc) main\"' >/dev/null 2>&1");
	        system("xdsh $confkeys{$mn} 'add-apt-repository \"deb http://ports.ubuntu.com/ubuntu-ports \$(lsb_release -sc)-updates main\"' >/dev/null 2>&1");
	        system("xdsh $confkeys{$mn} 'add-apt-repository \"deb http://ports.ubuntu.com/ubuntu-ports \$(lsb_release -sc) universe\"' >/dev/null 2>&1");
	        system("xdsh $confkeys{$mn} 'add-apt-repository \"deb http://ports.ubuntu.com/ubuntu-ports \$(lsb_release -sc)-updates universe\"' >/dev/null 2>&1");
		}
		
        system("xdsh $confkeys{$mn} \"wget -O - \"http://sourceforge.net/projects/xcat/files/ubuntu/apt.key/download\" | apt-key add -\" >/dev/null 2>&1");
		system("xdsh $confkeys{$mn} 'apt-get clean all' >/dev/null 2>&1");
		system("xdsh $confkeys{$mn} 'apt-get update' >/dev/null 2>&1");
		system("xdsh $confkeys{$mn} 'apt-get -y install build-essential dpkg-dev dh-make debhelper fakeroot gnupg lintian pbuilder quilt reprepro libsoap-lite-perl libdbi-perl' >/dev/null 2>&1");
    }
	
    #build xcat core ball 
	if($mn =~ /ubuntux/){
        $res = system("xdsh $confkeys{$mn} '/xcatbuild/xcat-core/build-ubunturepo -c UP=0 BUILDALL=1 > /var/log/xcat/build_xcat.log 2>&1' >/dev/null  2>&1");
	}else{
        $res = system("xdsh $confkeys{$mn} '/xcat-core/buildlocal.sh CURDIR=/xcat-core > /var/log/xcat/build_xcat.log 2>&1' >/dev/null 2>&1");
	}
    if ($res != 0){
        send_msg(0, "[$mn->$confkeys{$mn}][build_xcat] build xcat in $mn failed");
        return 1;
    }

    send_msg(2, "[$mn->$confkeys{$mn}][build_xcat] build xcat in $mn successfully");
    return 0;
}

#######################################
# config mn
#######################################
sub install_xcat {
    my $mn = shift;
	send_msg(2, "[$mn->$confkeys{$mn}][install_xcat] starting to install xcat in $confkeys{$mn}");
	
	my $os = $confkeys{$mn."os"};
	my $arch = "ppc64";
	$arch="x86_64" if($mn !~ /aix/ && $mn =~ /x/);
	
	if($mn =~ /rh/){
	    $os =~ /(\D+)(\d+)\.?(\d?)/;
		system("xdsh $confkeys{$mn} \"cd /xcat-dep/rh$2/$arch && ./mklocalrepo.sh\" >/dev/null 2>&1");
        system("xdsh $confkeys{$mn} \"rpm --import /RPM-GPG-KEY-redhat-release\" >/dev/null 2>&1");
		system("xdsh $confkeys{$mn} \"yum -y install xCAT xCAT-test > /var/log/xcat/install_xcat.log 2>&1\" >/dev/null 2>&1");
		system("xdsh $confkeys{$mn} \"yum -y install mysql-server mysql mysql-bench mysql-devel mysql-connector-odbc\" >/dev/null 2>&1");
		system("xdsh $confkeys{$mn} \"source /etc/profile.d/xcat.sh\" >/dev/null 2>&1");
		if($mn =~ /rhx/){
            system("xdsh $confkeys{$mn} \"yum install -y perl-Sys-Virt\" >/dev/null 2>&1");
		}
	}elsif($mn =~ /sles/){
	    $os =~ /(\D+)(\d+)\.?(\d?)/;
		system("xdsh $confkeys{$mn} \"zypper ar file:///xcat-dep/sles$2/$arch xCAT-dep\" >/dev/null 2>&1");   
		system("xdsh $confkeys{$mn} \"zypper sl -U\" >/dev/null 2>&1"); 
		system("xdsh $confkeys{$mn} \"zypper --gpg-auto-import-keys search --match-exact -s screen\" >/dev/null 2>&1");   
		system("xdsh $confkeys{$mn} \"zypper -n install xCAT xCAT-test > /var/log/xcat/install_xcat.log 2>&1\" >/dev/null 2>&1"); 
		system("xdsh $confkeys{$mn} \"zypper -n install createrepo\" >/dev/null 2>&1"); 
		system("xdsh $confkeys{$mn} \"zypper -n install mysql-client libmysqlclient_r15  libqt4-sql-mysql libmysqlclient15 perl-DBD-mysql mysql unixODBC\" >/dev/null 2>&1"); 
		system("xdsh $confkeys{$mn} \"source /etc/profile.d/xcat.sh\" >/dev/null 2>&1");
		if($mn =~ /slesx/){
		    system("xdsh $confkeys{$mn} \"zypper -n install perl-Sys-Virt\" >/dev/null 2>&1");
		}
	}elsif($mn =~ /ubuntux/){
        system("xdsh $confkeys{$mn} \"/xcat-core/mklocalrepo.sh\" >/dev/null 2>&1");
        system("xdsh $confkeys{$mn} \"/xcat-dep/mklocalrepo.sh\" >/dev/null 2>&1");
        system("xdsh $confkeys{$mn} \"wget -O - \"http://sourceforge.net/projects/xcat/files/ubuntu/apt.key/download\" | apt-key add -\" >/dev/null 2>&1");
        system("xdsh $confkeys{$mn} 'apt-get clean all' >/dev/null 2>&1");
        system("xdsh $confkeys{$mn} 'apt-get update' >/dev/null 2>&1");
        system("xdsh $confkeys{$mn} 'apt-get -y install xcat xCAT-test  > /var/log/xcat/install_xcat.log 2>&1' >/dev/null 2>&1");
	}

	#check if MN is installed successful 
	my $lsdefout = `ssh -t $confkeys{$mn} 'bash -l -i -c "lsdef -v"'`;
    chomp($lsdefout);
	if($lsdefout !~ /^lsdef - Version/ ){
	    send_msg(0, "[$mn->$confkeys{$mn}][install_xcat] install xcat in $mn failed");
	    return 1;
    }
	
    send_msg(2, "[$mn->$confkeys{$mn}][install_xcat] install xcat in $mn successfully");
    return 0;
}

#######################################
# do test
#######################################
sub do_test {
    my $mn = shift;
    send_msg(2, "[$mn->$confkeys{$mn}][do_test] starting to run regression test in $confkeys{$mn}");  
	
	my $regconf=$mn.".conf";
	my $res = system("scp $configfiledir/$regconf root\@$confkeys{$mn}:/opt/xcat/share/xcat/tools/autotest/default.conf  >/dev/null");
	if ($res != 0){
	    send_msg(0, "[$mn->$confkeys{$mn}][do_test] copy $configfiledir/$regconf to $mn failed");
	    return 1;
	}
	send_msg(2, "[$mn->$confkeys{$mn}][do_test] copy $configfiledir/$regconf to $mn successfully");
	
    my $diskfull_installation_flat_testcase="";
	my $diskless_installation_flat_testcase="";
	my $statelite_installation_flat_testcase="";
	my $sn_installation_hierarchy_testcase="";
	my $diskfull_installation_hierarchy_testcase="";
	my $diskless_installation_hierarchy_testcase="";
	my $statelite_installation_hierarchy_testcase="";
	my $batbundle="";
	if($mn =~ /sles/ || $mn =~ /rh/){
        $diskfull_installation_flat_testcase="reg_linux_diskfull_installation_flat";
        $diskless_installation_flat_testcase="reg_linux_diskless_installation_flat";
        $statelite_installation_flat_testcase="reg_linux_statelite_installation_flat";
        $sn_installation_hierarchy_testcase="reg_linux_SN_installation_hierarchy";
        $diskfull_installation_hierarchy_testcase="reg_linux_diskfull_installation_hierarchy";
        $diskless_installation_hierarchy_testcase="reg_linux_diskless_installation_hierarchy";
        $statelite_installation_hierarchy_testcase="reg_linux_statelite_installation_hierarchy";
		$batbundle="p_rhels_cmd.bundle" if($mn =~ /rhp/);
		$batbundle="x_rhels_cmd.bundle" if($mn =~ /rhx/);
		$batbundle="p_sles_cmd.bundle" if($mn =~ /slesp/);
		$batbundle="x_sles_cmd.bundle" if($mn =~ /slesx/);
        }elsif($mn =~ /ubuntux/){
		$diskfull_installation_flat_testcase="Ubuntu_full_installation_flat_x86_vm";
		$diskless_installation_flat_testcase="Ubuntu_diskless_installation_flat_x86_vm";
		$batbundle="x_ubuntu_cmd.bundle";
	}elsif($mn =~ /ubuntupel/){
		$diskfull_installation_flat_testcase="Ubuntu_full_installation_flat_p8le_vm";
                $diskless_installation_flat_testcase="Ubuntu_diskless_installation_flat_p8le_vm";
                $batbundle="p_ubunut.cmd.bundle";
	}
	
    if($mn !~ /ubuntu/){
		#diskfull installation in flat mode test
		send_msg(2, "[$mn->$confkeys{$mn}][do_test] doing diskfull installation test in flat mode in $mn.....");
		system("xdsh $confkeys{$mn} \"xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf  -t $diskfull_installation_flat_testcase\" >/dev/null 2>&1");
		send_msg(2, "[$mn->$confkeys{$mn}][do_test] do diskfull installation test in flat mode in $mn.....[done]");

		#commands bundle in flat mode test
		send_msg(2, "[$mn->$confkeys{$mn}][do_test] doing all commands test in flat mode in $mn.....");
		system("xdsh $confkeys{$mn} \"xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf  -b $batbundle\" >/dev/null 2>&1");
		send_msg(2, "[$mn->$confkeys{$mn}][do_test] do all commands test in flat mode in $mn.....[done]");

		#diskless installation in flat mode test		
		send_msg(2, "[$mn->$confkeys{$mn}][do_test] doing diskless installation test in flat mode in $mn.....");
		system("xdsh $confkeys{$mn} \"xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf  -t $diskless_installation_flat_testcase\" >/dev/null 2>&1");
		send_msg(2, "[$mn->$confkeys{$mn}][do_test] do diskless installation test in flat mode in $mn.....[done]");	
	
		#statelite installation in flat mode test
		send_msg(2, "[$mn->$confkeys{$mn}][do_test] doing statelite installation test in flat mode in $mn.....");
		system("xdsh $confkeys{$mn} \"xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf  -t $statelite_installation_flat_testcase\" >/dev/null 2>&1");
		send_msg(2, "[$mn->$confkeys{$mn}][do_test] do statelite installation test in flat mode in $mn.....[done]");

		#deploy SN
		send_msg(2, "[$mn->$confkeys{$mn}][do_test] setting up mysql for installing SN in $mn.....");
		$res = system("xdsh $confkeys{$mn} \"XCATMYSQLADMIN_PW=12345 XCATMYSQLROOT_PW=12345 mysqlsetup -i\" >/dev/null 2>&1");
		if ($res != 0){
			send_msg(0, "[$mn->$confkeys{$mn}][do_test] set up mysql for installing SN in $mn failed");
			return 1;
		}
		$res =  system ("xdsh $confkeys{$mn} \"echo \\\"GRANT ALL on xcatdb.* TO xcatadmin@\'%\'  IDENTIFIED BY \'12345\'\;\\\" | mysql -u root -p12345\" >/dev/null 2>&1");
		if ($res != 0){
			send_msg(0, "[$mn->$confkeys{$mn}][do_test] grant right to mysql for installing SN in $mn failed");
			return 1;
		}
		send_msg(2, "[$mn->$confkeys{$mn}][do_test] set up mysql for installing SN in $mn.....[done]");
		
		send_msg(2, "[$mn->$confkeys{$mn}][do_test] preparing xcat ball for installing SN in $mn.....");
		my $os = $confkeys{$mn."os"};
		my $arch = "ppc64";
		$arch="x86_64" if($mn !~ /aix/ && $mn =~ /x/);
		my $xcatballpath = "/install/post/otherpkgs/$os/$arch/xcat";
		$res = system("xdsh $confkeys{$mn} \"mkdir -p $xcatballpath/xcat-core && cp -r /xcat-core/build/* $xcatballpath/xcat-core && cp -r /xcat-dep $xcatballpath\"");
		if ($res != 0){
			send_msg(0, "[$mn->$confkeys{$mn}][do_test] prepare xcat ball for installing SN in $mn failed");
			return 1;
		}
		send_msg(2, "[$mn->$confkeys{$mn}][do_test] prepare xcat ball for installing SN in $mn.....[done]");
		
		if($mn =~ /rhp/){
		    my $ping_sn_on_rhp_pid = fork();
		    if( !defined($ping_sn_on_rhp_pid)){
		        send_msg(0, "fork process for trigger random number generator on redhat+ppc environment to install SN error");
		        exit;
		    }elsif($ping_sn_on_rhp_pid ==0){
		        send_msg(2, "....fork 1 process[pid=$$] for trigger random number generator on redhat+ppc environment to install SN..........");
		        my $snnodename = `cat $configfiledir/$regconf |grep SN|grep =|awk -F'=' '{print \$2}'`;
		        chomp($snnodename);
		        &runcmd("sleep 900");
		        my $timecnt=0;
		        while($timecnt < 180){
		            my $booted = system("xdsh $confkeys{$mn} \"lsdef -l $snnodename|grep status|grep booted\"");
		            last if($booted==0);
					
		            my $booting = system("xdsh $confkeys{$mn} \"lsdef -l $snnodename|grep status|grep booting\"");
		            system("xdsh $confkeys{$mn} \"xdsh $snnodename \"echo 1\"\"")  if($booting==0);
		
		            &runcmd("sleep 10");
		            $timecnt++;
			    }
			    send_msg(2, "....process[pid=$$] for trigger random number generator on redhat+ppc environment to install SN....[exit]");
			    exit 0;
		    }
		}
		
		send_msg(2, "[$mn->$confkeys{$mn}][do_test] installing SN.....");
		system("xdsh $confkeys{$mn} \"xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf  -t $sn_installation_hierarchy_testcase\" >/dev/null 2>&1");
		send_msg(2, "[$mn->$confkeys{$mn}][do_test] install SN.....[done]");
		
		#diskfull installation in hierarchy mode test
		send_msg(2, "[$mn->$confkeys{$mn}][do_test] doing diskfull installation test in hierarchy mode in $mn.....");
		system("xdsh $confkeys{$mn} \"xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf  -t $diskfull_installation_hierarchy_testcase\" >/dev/null 2>&1");
		send_msg(2, "[$mn->$confkeys{$mn}][do_test] do diskfull installation test in hierarchy mode in $mn.....[done]");
		
		#diskless installation in hierarchy mode test
		send_msg(2, "[$mn->$confkeys{$mn}][do_test] doing diskless installation test in hierarchy mode in $mn.....");
		system("xdsh $confkeys{$mn} \"xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf  -t $diskless_installation_hierarchy_testcase\" >/dev/null 2>&1");
		send_msg(2, "[$mn->$confkeys{$mn}][do_test] do diskless installation test in hierarchy mode in $mn.....[done]");

		#statelite installation in hierarchy mode test
		send_msg(2, "[$mn->$confkeys{$mn}][do_test] doing statelite installation test in hierarchy mode in $mn.....");
		system("xdsh $confkeys{$mn} \"xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf  -t $statelite_installation_hierarchy_testcase\" >/dev/null 2>&1");
		send_msg(2, "[$mn->$confkeys{$mn}][do_test] do statelite installation test in hierarchy mode in $mn.....[done]");	
		
    }else{
		#diskfull installation in flat mode test
		send_msg(2, "[$mn->$confkeys{$mn}][do_test] doing diskfull installation test in flat mode in $mn.....");
		system("ssh -t $confkeys{$mn} 'exec bash -l -i -c \"xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf  -t $diskfull_installation_flat_testcase\"' >/dev/null 2>&1");
		send_msg(2, "[$mn->$confkeys{$mn}][do_test] do diskfull installation test in flat mode in $mn.....[done]");

		#commands bundle in flat mode test
		send_msg(2, "[$mn->$confkeys{$mn}][do_test] doing all commands test in flat mode in $mn.....");
		system("ssh -t $confkeys{$mn} 'exec bash -l -i -c \"xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf  -b $batbundle\"' >/dev/null 2>&1");
		send_msg(2, "[$mn->$confkeys{$mn}][do_test] do all commands test in flat mode in $mn.....[done]");

		#diskless installation in flat mode test		
		send_msg(2, "[$mn->$confkeys{$mn}][do_test] doing diskless installation test in flat mode in $mn.....");
		system("ssh -t $confkeys{$mn} 'exec bash -l -i -c \"xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf  -t $diskless_installation_flat_testcase\"' >/dev/null 2>&1");
		send_msg(2, "[$mn->$confkeys{$mn}][do_test] do diskless installation test in flat mode in $mn.....[done]");
	}
	
    send_msg(2, "[$mn->$confkeys{$mn}][install_xcat] run regression test in $mn finished");
	
	$res = system("scp -r $confkeys{$mn}:/opt/xcat/share/xcat/tools/autotest/result/* $todayregresultdir/$mn/ >/dev/null 2>&1");
	if ($res != 0){
	    send_msg(0, "[$mn->$confkeys{$mn}][do_test] copy regression result to $todayregresultdir/$mn/ failed");
	    return 1;
	}
	send_msg(2, "[$mn->$confkeys{$mn}][do_test] copy regression result to $todayregresultdir/$mn/ successfully");

    return 0;

}

#######################################
# creat report
#######################################
sub creat_report{

    my $mailreport .= "======================================\n";
    $mailreport .= "       Regression Result Summary\n";
    $mailreport .= "======================================\n\n";
	# $gitupdateflag = 0 git update out of time
	# $gitupdateflag = 1 there isn't new code checkin in last one day
	# $gitupdateflag = 2 git update failed
	if($gitupdateflag==0){
	    $mailreport .= "Get the latest xcat code from git out of time, stop regression test today!!!\n\n\n";
		&runcmd("touch $mailfile && echo \"$mailreport\" > $mailfile");
		&runcmd("mv $commitinfofile $commitinfofile.old && echo \"\" > $commitinfofile");  		
		return 0;
	}elsif($gitupdateflag==1 && ! $forceregwithoutupdate){
	    $mailreport .= "There isn't new code checkin last one day, stop regression test today!!!\n\n\n";
		&runcmd("touch $mailfile && echo \"$mailreport\" > $mailfile"); 
		&runcmd("mv $commitinfofile $commitinfofile.old && echo \"\" > $commitinfofile");  
		return 0;
	}elsif($gitupdateflag==1 && $forceregwithoutupdate){
	    $mailreport .= "There isn't new code checkin last one day, But force to run regresson today!!!\n\n\n";
		&runcmd("mv $commitinfofile $commitinfofile.old && echo \"\" > $commitinfofile");  
	}elsif($gitupdateflag==2){
	    $mailreport .= "Get the latest xcat source code from git error, stop regression test today!!!\n\n\n";
		&runcmd("touch $mailfile && echo \"$mailreport\" > $mailfile"); 
		&runcmd("mv $commitinfofile $commitinfofile.old && echo \"\" > $commitinfofile");   
		return 0;	
	}
 
    while(my($k,$v) = each %pidrecord){
        my $scenario="[".$confkeys{$k."os"};
        if($k=~/p/){$scenario.="+PPC64] ";}else{$scenario.="+x86_64] ";}
        $mailreport .= "$scenario\n";
        my $batbundle="";
        my $totalcase=0;

 if($k=~ /sles/ || $k =~ /rh/){
                $batbundle="$xcatcoredir/xCAT-test/autotest/bundle/p_rhels_cmd.bundle" if($k =~ /rhp/);
                $batbundle="$xcatcoredir/xCAT-test/autotest/bundle/x_rhels_cmd.bundle" if($k =~ /rhx/);
                $batbundle="$xcatcoredir/xCAT-test/autotest/bundle/p_sles_cmd.bundle" if($k =~ /slesp/);
                $batbundle="$xcatcoredir/xCAT-test/autotest/bundle/x_sles_cmd.bundle" if($k =~ /slesx/);
                $totalcase=`cat $batbundle|wc -l`;
                $totalcase+=7;
                }elsif($k =~ /ubuntux/){
                $batbundle="/$xcatcoredir/xCAT-test/autotest/bundle/x_ubuntu_cmd.bundle";
                $totalcase=`cat $batbundle|wc -l`;
                $totalcase+=2;
                }elsif($k =~ /ubuntupel/){
                $batbundle="$xcatcoredir/xCAT-test/autotest/bundle/p_ubunut.cmd.bundle";
                $totalcase=`cat "$batbundle|wc -l`;
                $totalcase+=2;
                }


        if ($v == 0){ # whole regression test finished 
            my $h=int($consumptionrecord{$k}/3600);
			my $m=int(($consumptionrecord{$k} - $h*3600)/60);
			my $s=($consumptionrecord{$k} - $h*3600)%60;
			my $consumption="$h hours $m minutes $s seconds";
			$mailreport .= "\tTime consumption: $consumption\n";
			
			if($piderrrecord{$k} ne ""){
			    $mailreport .= "\tDeploy MN result: $piderrrecord{$k}\n";
                            $mailreport .= "\tTotalCase $totalcase TotalRun 0 Failed 0\n";
			    next;
			}
		
			my $totalcnt=0;
			my $failcnt=0;
			my $faillist="";
			opendir(DIR, "$todayregresultdir/$k");
			foreach my $file (readdir DIR)
			{
			    next if($file !~ /xcattest.log/);
			    my $cnt = `cat $todayregresultdir/$k/$file |grep -- "------END::"|wc -l`;
				$totalcnt+=int($cnt);
			    for(my $i=1;$i<$cnt+1;$i++){
			        my $line=`cat $todayregresultdir/$k/$file |grep -- "------END::"|sed -n ${i}p`;
					chomp($line);
					if($line =~ /------END::([a-zA-Z0-9_-]+)::([a-zA-Z0-9_-]+)::Time.+/){
					    my $failedcase=$1;
					    if($2 =~ /Failed/){
					        $failcnt++;
							$faillist.=$failedcase.", ";
						}
					}
			    }
			}

			closedir(DIR);


			$mailreport .= "\tTotalCase $totalcase TotalRun $totalcnt Failed $failcnt\n\tFailed cases: $faillist\n";
		}else{ # regression test out of time
			$mailreport .= "\tTime consumption: Out of 8 hours!!!!!!\n";
		}
    }
	
    &runcmd("touch $mailfile && echo \"$mailreport\" > $mailfile"); 
    return 0;
}

#######################################
# send email
#######################################
sub send_email{
	my $checkinlist="";
	my $cnt =`cat  $commitinfofile |grep '20[0-9][0-9] [-+]0[0-9]00'|awk '{print \$1}' |grep -v root|sort | uniq|wc -l`;
	my $out =`cat  $commitinfofile |grep '20[0-9][0-9] [-+]0[0-9]00'|awk '{print \$1}' |grep -v root|sort | uniq`;
	for(my $i=1;$i<$cnt+1;$i++){
        my $tmpline=`echo "$out"|sed -n ${i}p`;
		chomp($tmpline);
		$checkinlist.= "$tmpline ";
	}
	
	my $totalcnt=0;
	my $failcnt=0;
        my $totalcase=0;   
$cnt =`cat  $mailfile |grep "Total"|grep "Failed"|wc -l`;
	$out =`cat  $mailfile |grep "Total"|grep "Failed"`;
	for(my $i=1;$i<$cnt+1;$i++){
        my $tmpline=`echo "$out"|sed -n ${i}p`;
		chomp($tmpline);
		if($tmpline =~ /TotalCase (\d+) TotalRun (\d+) Failed (\d+)/){
                   $totalcase+=$1;
		    $totalcnt+=$2;
			$failcnt+=$3;
		}
	}
	
	# $gitupdateflag = 0 git update out of time
	# $gitupdateflag = 1 there isn't new code checkin in last one day
	# $gitupdateflag = 2 git update failed
	my $subject="";
	if($gitupdateflag==0 || $gitupdateflag==2){
	    $subject = "[$mailtitle] Git update failed, stop regression test today!!!";
	}elsif($gitupdateflag==1 && ! $forceregwithoutupdate){
	    $subject = "[$mailtitle] No code checkin, stop regression test today!!!";
	}elsif($gitupdateflag==1 && $forceregwithoutupdate){
	    $subject = "[$mailtitle] TotalCase:".$totalcase.", TotalRun:".$totalcnt.", Fail:".$failcnt.", Branch:".$branch.", Checkin: NO ONE!!";
	}else{
	    $subject = "[$mailtitle] TotalCase:".$totalcase.", TotalRun:".$totalcnt.", Fail:".$failcnt.", Branch:".$branch.", Checkin: ".$checkinlist;
	}
    my $mailreport .= "\n\n======================================\n";
    $mailreport .= "       Commit Codes Last Day\n";
    $mailreport .= "======================================\n\n";
	
	&runcmd("echo \"$mailreport\">> $mailfile");
    &runcmd("cat $commitinfofile >> $mailfile");  	
	
	my $attachfile="/tmp/failed_case_detail.txt";
	&runcmd("echo \"\" > $attachfile");
	if($failcnt > 0){
        opendir(DIR, "$todayregresultdir");
        foreach my $subdir (readdir DIR){
            next  if($subdir =~ /^\./);
            #print "subdir-> $subdir\n";
			
			my $scenario="[".$confkeys{$subdir."os"};
			if($subdir=~/p/){$scenario.="+PPC64] ";}else{$scenario.="+x86_64] ";}
            my $separator .= "\n###############################################################\n";
            $separator .= "                    $scenario\n";
            $separator .= "###############################################################\n\n";
			&runcmd("echo \"$separator\" >> $attachfile");
            opendir(SUBDIR, "$todayregresultdir/$subdir");
            foreach my $file (readdir SUBDIR){
                next if($file !~ /^failedcases/);
                next if(-z "$todayregresultdir/$subdir/$file");
                #print "\t$file\n";
                system("cat $todayregresultdir/$subdir/$file >> $attachfile");
            }
        }
	}
	
	if($mail_list){
	    if($failcnt > 0){
	        system("cat $mailfile"." | /bin/mail -s \"$subject\" -a $attachfile \"$mail_list\"");
		}else{
	        system("cat $mailfile"." | /bin/mail -s \"$subject\" \"$mail_list\"");		
		}
	}else{
	    if($failcnt > 0){
            system("cat $mailfile"." | /bin/mail -s \"$subject\" -a $attachfile \"$confkeys{mailgroup}\"");
	    }else{
            system("cat $mailfile"." | /bin/mail -s \"$subject\" \"$confkeys{mailgroup}\"");
		}
	}
    return 0;
}


#######################################
#git update
#return value:
#0 successful
#1 code is already at latest version
#2 someting error
#######################################
sub git_update{
	send_msg(2, "[git update] starting to update xcat source code from git"); 
	
    my $res = system("cd $xcatcoredir && git checkout $branch > /dev/nul 2>&1");
    if ($res != 0){
        send_msg(0, "[git update] change to branch $branch....[failed]");
        return 2;
    }
    send_msg(2, "[git update] change to branch $branch....[ok]");
	
	my $gitpulloutput = "/tmp/gitpulloutput";
	my $trytime=1;
    while($trytime < 4){	
        $res = system("cd $xcatcoredir && git pull > $gitpulloutput 2>&1");
        if ($res != 0){
            send_msg(0, "[git update] pull the latest code to control node....[$trytime failed]");
			$trytime++;
            &runcmd("sleep 120");  	
        }else{
		    last;
		}
	}
	
	if($trytime == 4){
        send_msg(0, "[git update] pull the latest code to control node....[finally failed]");
        return 2;		
	}
	
    $res = system("grep 'Already up-to-date' $gitpulloutput > /dev/nul 2>&1");
    if ($res == 0 && ! $forceregwithoutupdate) {
        send_msg(2, "[git update] code is already at latest version. exit regresson");
        return 1;
    }elsif($res == 0 && $forceregwithoutupdate){
        send_msg(2, "[git update] code is already at latest version. But force to run regresson");
        return 0;
	}
	send_msg(2, "[git update] pull the latest code to control node....[ok]");
	
    my $lastcommitfile="/tmp/lastcommitfile";
	my $lastcommitinfo;
	if (-f $lastcommitfile){
        $lastcommitinfo = `cat $lastcommitfile`;
		chomp($lastcommitinfo);
    }else{
	    $lastcommitinfo = "";
	}
	#print "lastcommitinfo=$lastcommitinfo\n";
	
	if($lastcommitinfo eq ""){
        $res = system("cd $xcatcoredir && git log --since=\"1 days ago\"  --pretty=format:\"%cn %ad %s\" --name-status > $commitinfofile");
        if ($res != 0  || (-f $commitinfofile and -z _)){
            send_msg(0, "[git update] get commit information....[failed]");
            return 2;
        }
        send_msg(0, "[git update] get commit information....[ok]");
	}else{
        $res = system("cd $xcatcoredir && git log $lastcommitinfo..HEAD  --pretty=format:\"%cn %ad %s\" --name-status > $commitinfofile");
        if ($res != 0 || (-f $commitinfofile and -z _)){
            send_msg(0, "[git update] get commit information....[failed]");
            return 2;
        }
        send_msg(2, "[git update] get commit information....[ok]");
	}
	
	$res = system("cd $xcatcoredir && git log -n 1|awk 'NR==1'|awk '{print \$2}' > $lastcommitfile");
    if ($res != 0){
        send_msg(0, "[git update] get latest commit number failed");
        exit;
    }

    return 0;
}

#######################################
# step 0. Parse input arguments
#######################################
send_msg(2, "step 1, Parse input arguments............");
=pod
if (
    !GetOptions("--help|h|?" => \$needhelp,
                "f=s" => \$configfile,
                "b=s" => \$branch,
                "m=s" => \$management_node,
                "V"   => \$verbose_mode,
                "U"   => \$updates_regression,
                "e=s" => \$send_msg)
 )
=cut
if (
    !GetOptions("--help|h|?" => \$needhelp,
                "f=s" => \$configfile,
                "b=s" => \$branch,
                "m=s" => \$management_node,
				"a"   => \$forceregwithoutupdate,
                "e=s" => \$mail_list)
)
{
    &usage;
    send_msg(0, "step 0, PARSE ARGUMENTS returns error, exit");
    exit;
}
if ($needhelp)
{
    &usage;
    exit 0;
}


xCATreg_init();

###############################################################
# Mainfunction
###############################################################
send_msg(2,"........................");
send_msg(2,"........................");
send_msg(2,".....ooooO..............");
send_msg(2,"....(....)....Ooooo.....");
send_msg(2,".....\\..(.....(....)....");
send_msg(2,"......\\__).....)../.....");
send_msg(2,"..............(_ /......");
send_msg(2,"........................");
send_msg(2,".........START .........");
send_msg(2,"........................");
#######################################
# step 1. Read configuration files
#######################################
send_msg(2, "step 2, Reading xCATreg conf............");
my $rst = read_conf();
unless (! $rst) {
    send_msg(0, "read xCATreg configuration failed...........exit");
    exit;
}

foreach my $k (keys %confkeys) {
print "$k = $confkeys{$k}\n";
}

# begin to find which mn should be used
if ($management_node) {
    my @all;
    if ($management_node =~ /,/) {
        @all = split /,/, $management_node;
    } else {
        push @all, $management_node;
    }
    foreach my $mn (@all) {
        foreach my $k (keys %confkeys) {
            if ($confkeys{$k} eq $mn) {
                $mns{$k} = $mn;
                send_msg(2, "specify mn :$mn");
            }
        }
    }
}

pipe  PREAD,CWRITE;
my $gitpid = fork();
if( !defined($gitpid)){
    send_msg(0, "fork process for git update error");
    exit;
}elsif($gitpid ==0){
    close PREAD;
    send_msg(2, "..........fork 1 process[pid=$$] for git update..........");
    my $res = git_update();
    if($res > 1){
		send_msg(0, "git_update......[failed]");
        syswrite CWRITE,"git update failed\n";
		close CWRITE;
        exit 1;
    }
	if($res == 1){
	    send_msg(2, "git_update......[not find new code]");
        syswrite CWRITE,"git update got nothing\n";
		close CWRITE;
        exit 1;
    }   
	send_msg(2, "git_update......[ok]");
    syswrite CWRITE,"git update successful\n";
    close CWRITE;
	exit 0;
}

{#main process
    close CWRITE;
    my $gitstarttime = time();
    my $gitprocess=1;
    my $gitselect = new IO::Select;
    $gitselect->add(\*PREAD);     
    while($gitprocess){
        my @hdls;
		if (@hdls = $gitselect->can_read(0)) {
	        my $hdl;
	        foreach $hdl (@hdls) {
	            if ($hdl == \*PREAD) {
	                my $line;
	                chomp($line=<PREAD>);
	                if ($line){
					    if($line =~ /successful/){
						    send_msg(2, "[[main]]: $line");
							$gitupdateflag=3;
						}
						if($line =~ /failed/){
						    send_msg(0, "[[main]]: $line");
                            $gitupdateflag=2;							
						}
						if($line =~ /nothing/){
						    send_msg(2, "[[main]]: there isn't new code checkin during last one day.");
							$gitupdateflag=1;
						}
	                    $gitprocess = 0;
	                }
	            } 
	        }
	    }

		if(time() - $gitstarttime > 7200) {
		#if(time() - $gitstarttime > 2) {
			send_msg(0, "[[main]] 2 hours is expired. git update out of time");
			last;
		}
		&runcmd("sleep 2");   
    }    
	close PREAD;
	
	&runcmd("kill $gitpid") if($gitprocess==1);
    send_msg(2, "[[main]]........git process[$gitpid] exit successful..............");
	
    # $gitupdateflag = 0 git update out of time
	# $gitupdateflag = 1 there isn't new code checkin in last one day
	# $gitupdateflag = 2 there is something wrong during git update
	if($gitupdateflag<3){
		send_msg(2, "[[main]]: creat regression result report...........");
		$rst = creat_report();
		if ($rst) {
		    send_msg(0, "[[main]]: creat regression result report...........[FAILED]");
		}else{
		    send_msg(2, "[[main]]: creat regression result report...........[OK]");
		
		    send_msg(2, "[[main]]: send report email...........");
		    $rst = send_email();
		    if ($rst) {
		        send_msg(0, "[[main]]: send report email...........[FAILED]");
		    }else{
		         send_msg(2, "[[main]]: send report email...........[OK]");
		    }
		}
		exit 1;
	}
	send_msg(2, "[[main]]: git update successful, start regression today........");

	#exit 0
}


my $platformfork = 0;
my $timingork = 0;
# begin child process
pipe CONTROLREAD,MNWRITE;
foreach my $m (keys %mns) {
    $platformfork += 1;
    my $mn = $mns{$m};
    my $pid = fork();
    if ( !defined($pid) ) {
	    send_msg(0, "fork process for $m error");
        exit;
    } elsif ( $pid == 0 ) { # child process
	    send_msg(2, "..........fork process[pid=$$] for $m [$confkeys{$m}]..........");
        close CONTROLREAD;
		my $res; 
      		
        #######################################
        # step 3. Install MNs,
        #######################################
        send_msg(2, "[$m]step 3, Running mn_install...............");
        $res = mn_install($m);
        if ($res) {
            syswrite MNWRITE,"REPORTFROM:$m->$mn: install $mn failed\n";
            exit 1;
        }
		send_msg(2, "[$m]step 3, Run mn_install...............[OK]");
 	   
        #######################################
        # step 4. Copy code to MNs
        #######################################
        send_msg(2, "[$m]step 4, Running copy_code...............");
        $res =  copy_code($m);
        if ($res) {
            syswrite MNWRITE,"REPORTFROM:$m->$mn: copy xcat source code to $mn failed\n";
            exit 1;
        }
		send_msg(2, "[$m]step 4, Run copy_code...............[OK]");
 
        #######################################
        # step 5. Build xcat code in MNs
        #######################################
        send_msg(2, "[$m]step 5, Running build_xcat...............");
        $res =  build_xcat($m);
        if ($res) {
            syswrite MNWRITE,"REPORTFROM:$m->$mn: build new xcat in $mn failed\n";
            exit 1;
        }
		send_msg(2, "[$m]step 5, Run build_xcat...............[OK]");

        #######################################
        # step 6. Read xCAT MN's configuration
        #######################################
        send_msg(2, "[$m]step 6, Running install_xcat...............");
        $res = install_xcat($m);
        if ($res) {
            syswrite MNWRITE,"REPORTFROM:$m->$mn: install new xcat in $mn failed\n";
            exit 1;
        }
		send_msg(2, "[$m]step 6, Run install_xcat...............[OK]");
	
        #######################################
        # step 7. Genrate local configuration file for xcattest
        #         Do test
        #         Write log
        #######################################
        send_msg(2, "[$m]step 7, Running do_test...............");
        $res = do_test($m);
        if ($res) {
            syswrite MNWRITE,"REPORTFROM:$m->$mn: run regression test cases in $mn failed\n";
            exit 1;
        }
        send_msg(2, "[$m]step 7,  Running do_test...............[OK]");	
		
        send_msg(2, "whole regression test for $mn finished");
		syswrite MNWRITE,"REPORTFROM:$m->$mn: whole regression test are successful\n";
        exit 0;
    } # end of child process
    &runcmd("mkdir -p $todayregresultdir/$m");
    $pidrecord{$m} = $pid;
    $piderrrecord{$m} = "";
} # end of foreach


{ # parent process
	close MNWRITE;
	my $rst;
	my $regstarttime = time();
	
	my $select = new IO::Select;
	$select->add(\*CONTROLREAD);
	while ($platformfork) {
	    my @hdls;
		if (@hdls = $select->can_read(0)) {
	        my $hdl;
	        foreach $hdl (@hdls) {
	            if ($hdl == \*CONTROLREAD) {
	                my $line;
	                chomp($line=<CONTROLREAD>);
	                if ($line){
						if($line =~ /successful/){
						    send_msg(2, "[[main]]: $line");
						    $line =~ s/^REPORTFROM:(.+)->.+/$1/g;
						    $pidrecord{$line} = 0;
						    $consumptionrecord{$line} = time() - $regstarttime;
						}
						if($line =~ /failed/){
						    send_msg(0, "[[main]]: $line");
							my $tmp = $line;
						    $line =~ s/^REPORTFROM:(.+)->.+/$1/g;
						    $pidrecord{$line} = 0;
						    $consumptionrecord{$line} = time() - $regstarttime;
							$tmp =~ s/REPORTFROM:.+->.+:(.+)/$1/g;
							$piderrrecord{$line} = $tmp;
						}
	                    $platformfork -= 1;
	                }
	            } 
	        }
	    }
		
		if(time() - $regstarttime > 28800) {
		#if($curtime - $regstarttime > 10) {
			send_msg(1, "[timing] 8 hours is expired");
			last;
		}
		&runcmd("sleep 1");  
	}
	
	close CONTROLREAD;
	if($platformfork){
        send_msg(0, "[[main]]: there are $platformfork platform's regression test return out of time");
        while(my($k,$v) = each %pidrecord){
            &runcmd("kill $v") if ($v != 0);
        }
	}else{
	    send_msg(2, "[[main]]: all platform's regression test return on time");
	}
	
	send_msg(2, "[[main]]: creat regression result report...........");
	$rst = creat_report();
	if ($rst) {
		send_msg(0, "[[main]]: creat regression result report...........[FAILED]");
	}else{
        send_msg(2, "[[main]]: creat regression result report...........[OK]");
		
		send_msg(2, "[[main]]: send report email...........");
		$rst = send_email();
		if ($rst) {
		    send_msg(0, "[[main]]: send report email...........[FAILED]");
		}else{
		    send_msg(2, "[[main]]: send report email...........[OK]");
		}
	}
	send_msg(2, "[[main]]: main process exit");
	exit 0;
}