#!/usr/bin/env perl
# IBM(c) 2010 EPL license http://www.eclipse.org/legal/epl-v10.html
#
#############################################################################################################################################
#
#   This script will be used to run xCAT daily regression
#
#   Usage:xCATreg - Run xCAT daily regression
#   xCATreg [-?|-h]
#   xCATreg [-b branch][-m MN][-f configure file][-a][-V]
#
#   -?/-h:  help information for the xCATreg usage.
#   -b: when -b is specified, the regression will be done against the specified branch. You can use 2.7, 2.8,master.The default value is master.
#   -m: when -m is specified, the regression will be done the specified mn. The default value is all the mn in configuration file.
#   -f :specify the configuration file,default value is /xCATreg/regresson.conf
#   -a: when -a is specified, run the regression even there is no code updates
#   -V: log and message in verbose mode
#
#   Example usage:
#   ./xCATreg   Run regressoins on master branch and on all platforms in configuration file
#   ./xCATreg -b 2.8 -m slesmn -f /root/regressoin.conf -a    Run regression on 2.8 branch on slesmn even when there is no code updates.
#
#############################################################################################################################################

$ENV{PATH} = "/opt/xcat/bin:/opt/xcat/sbin:/opt/xcat/share/xcat/tools:/usr/sbin:/usr/bin:/sbin:/bin";

use strict;
use warnings;
use Getopt::Long;
use Data::Dumper;
use Term::ANSIColor;
use Time::Local;
use IO::Select;

BEGIN
{
    $::XCATROOT = $ENV{'XCATROOT'} ? $ENV{'XCATROOT'} : -d '/opt/xcat' ? '/opt/xcat' : '/usr';
}
use lib "$::XCATROOT/lib/perl";

my $date;
my $regrootdir = "/home/autotest_code";
my $mailtitle  = "PPC64LE Cluster";

#my $regrootdir="/xCATreg";
#my $mailtitle="X Cluster";
my $logfiledir        = "$regrootdir/log";
my $logfile           = "";
my $configfiledir     = "$regrootdir/conf";
my $configfile        = "$regrootdir/conf/xCATreg.conf";
my $xcatcoredowndir   = "$regrootdir/xcatsource/xcat-core-build-down";
my $xcatcoredir       = "$regrootdir/xcatsource/xcat-core";
my $xcatdepdir        = "$regrootdir/xcatsource/xcat-dep";
my $osisodir          = "$regrootdir/iso";
my $regresultdir      = "$regrootdir/regresult";
my $todayregresultdir = "";
my $maildir           = "$regrootdir/mail";
my $mailfile          = "";
my $rootdir           = "$::XCATROOT/share/xcat/tools/autotest";
my %confkeys;
my $mail_list       = 0;
my $needhelp        = 0;
my $branch          = "master";
my $management_node = undef;

#my $verbose_mode = 0;
my $updates_regression = 0;
my %mns;
my %pidrecord;
my %piderrrecord;
my %consumptionrecord;
my $commitinfofile        = "/tmp/commitinfo";
my $gitupdateflag         = 0;
my $forceregwithoutupdate = 0;
my $depupdateflag         = 0;

#######################################
# runcmd
#######################################
sub runcmd {
    my ($cmd) = @_;
    my $rc = 0;
    $::RUNCMD_RC = 0;
    my $outref = [];
    @$outref = `$cmd 2>&1`;
    if ($?)
    {
        $rc          = $?;
        $::RUNCMD_RC = $rc;
    }
    chomp(@$outref);
    return @$outref;
}

#######################################
# clean_outdated_file
#######################################
sub is5daysago {
    my $targetdate  = shift;
    my $targetdatey = "";
    my $targetdatem = "";
    my $targetdated = "";
    my $datey       = "";
    my $datem       = "";
    my $dated       = "";
    if ($targetdate =~ /(\d{4})(\d{2})(\d{2})/) {
        $targetdatey = $1;
        $targetdatem = $2 - 1;
        $targetdated = $3;
    }
    if ($date =~ /(\d{4})(\d{2})(\d{2})/) {
        $datey = $1;
        $datem = $2 - 1;
        $dated = $3;
    }
    my $todays  = timelocal(0, 0, 0, $dated,       $datem,       $datey);
    my $targets = timelocal(0, 0, 0, $targetdated, $targetdatem, $targetdatey);
    if ($todays - $targets > 432000) {
        return 1;
    }
    return 0;
}

#######################################
# clean_outdated_file
#######################################
sub clean_outdated_file {
    opendir(DIR, "$logfiledir");
    foreach my $file (readdir DIR) {
        if ($file =~ /xCATreg(\d+).log/) {
            my $res = is5daysago($1);
            if ($res) {
                &runcmd("rm -f $logfiledir/$file");
            }
        }
    }
    closedir(DIR);

    opendir(DIR, "$maildir");
    foreach my $file (readdir DIR) {
        if ($file =~ /mailreport.(\d+)/) {
            my $res = is5daysago($1);
            if ($res) {
                &runcmd("rm -f $maildir/$file");
            }
        }
    }
    closedir(DIR);

    opendir(DIR, "$regresultdir");
    foreach my $file (readdir DIR) {
        if ($file =~ /(\d+)/) {
            my $res = is5daysago($1);
            if ($res) {
                &runcmd("rm -rf $regresultdir/$file");
            }
        }
    }
    closedir(DIR);

    return 0;
}

#######################################
# xCATreg_init
#######################################
sub xCATreg_init {

    #init important attributes

    $date = `date  +"%Y%m%d%H%M%S"`;
    chomp($date);

    &runcmd("mkdir -p $logfiledir")   if (!-e $logfiledir);
    &runcmd("mkdir -p $maildir")      if (!-e $maildir);
    &runcmd("mkdir -p $regresultdir") if (!-e $regresultdir);
    &runcmd("mkdir -p $xcatcoredir")  if (!-e $xcatcoredir);

    $logfile           = "xCATreg" . $date . ".log";
    $todayregresultdir = $regresultdir . "/" . $date;
    &runcmd("mkdir -p $todayregresultdir");
    $mailfile = $maildir . "/mailreport." . $date;

    my $cnt = `ps aux|grep xCATreg |grep -v grep|awk '{print \$2}'|wc -l`;
    my $out = `ps aux|grep xCATreg |grep -v grep|awk '{print \$2}'`;
    for (my $i = 1 ; $i < $cnt + 1 ; $i++) {
        my $pid = `echo "$out"|sed -n ${i}p`;
        chomp($pid);
        next if ($pid == $$);
        &runcmd("kill $pid");
    }

    clean_outdated_file();
    return 0;
}

#######################################
# usage for arguments
#######################################
sub usage {
    print "Usage:xCATreg - Run xcat autotest  test cases.\n
  Explanation for the options:
    -b: when -b is specified, the regression will be done against the specified branch. The default value is master.\n
    -m: when -m is specified, run regression on the specified mn. The default value is all the mn in configuration file.\n
    -f: specify the configuration file location.\n
    -a: when -a is specified, run the regression even there is no code updates.\n
    -e: <email_addr> send the test result to email_addr\n\n";
    print "  xCATreg [-?|-h]\n";
    print "  xCATreg [-f configure file] [-b branch] [-m mangement node] [-e mail list] [-a] Even code not updates, there will be regression.\n";
    print "  xCATreg [-f configure file] [-b branch] [-m mangement node] [-e mail list] if code updates there will be regression.\n";
    print "\n";
    return;
}

#######################################
# send messages
#######################################
sub send_msg {
    my $num = shift;
    my $msg = shift;
    my $content;
    if ($num == 0) {
        $content = "Fatal error:";
    } elsif ($num == 1) {
        $content = "Warning:";
    } elsif ($num == 2) {
        $content = "Notice:";
    }
    my $timestamp = `date +"%Y%m%d%H%M%S"`;
    chomp($timestamp);
    if (!open(LOGFILE, ">> $logfiledir/$logfile")) {
        return 1;
    }
    print LOGFILE "$content $timestamp $$: $msg.\n";
    close LOGFILE;
}


#######################################
# read_conf
#######################################
sub read_conf {
    my $myfile = undef;
    my $line   = undef;

    if (!open($myfile, "$configfile")) {
        send_msg(0, "Open $configfile failed");
        return 1;
    }
    while ($line = <$myfile>) {
        $line =~ s/\s//g;
        next if ($line =~ /^#/);
        next if ($line eq "");
        my @attr = split(/=/, $line);
        $confkeys{ $attr[0] } = $attr[1];
    }
    close($myfile);
    return 0;
}

#######################################
# mn_install to install all mn
#######################################
sub mn_install {
    my $mn = shift;

    &runcmd("lsdef|grep $confkeys{$mn}");
    if ($?) {
        send_msg(0, "[$mn->$confkeys{$mn}][mn_install] Didn't defined $confkeys{$mn} in current control node");
        return 1;
    }

    my $os   = $confkeys{ $mn . "os" };
    my $arch = "ppc64";
    if ($mn !~ /aix/ && $mn =~ /x/) {
        $arch = "x86_64";
    } elsif ($mn !~ /aix/ && $mn =~ /LE/) {
        $arch = "ppc64le";
    } elsif ($mn !~ /aix/ && $mn =~ /EL/) {
        $arch = "ppc64el";
    }
    my $osimage = "";

    if ($mn !~ /aix/) {
        $osimage = $os . "-" . $arch . "-install-compute";
    } else {
        $osimage = "aix";    #????????need to test for aix
    }

    #print "[$mn] : os= $os arch=$arch osimage=$osimage\n";

    &runcmd("lsdef -t osimage|grep $osimage");
    if ($?) {
        send_msg(0, "[$mn->$confkeys{$mn}][mn_install] Didn't defined $osimage in current control node");
        return 1;
    }

    send_msg(2, "[$mn->$confkeys{$mn}][mn_install] installing $osimage for node $confkeys{$mn}................");

    if ($arch =~ /x86_64/ || $arch =~ /ppc64le/ || $arch =~ /ppc64el/) {
        &runcmd("rmvm $confkeys{$mn} -f -p ");
        &runcmd("chdef $confkeys{$mn} mac= ");
        &runcmd("mkvm $confkeys{$mn} -s 40G");
    }

    &runcmd("makedhcp -n");
    &runcmd("makedns -n");
    &runcmd("chdef $confkeys{$mn} status=");
    if ($?) {
        send_msg(0, "[$mn->$confkeys{$mn}][mn_install] Failed to change MN status in current control node");
        return 1;
    }


    &runcmd("nodeset $confkeys{$mn} osimage=$osimage");
    if ($arch =~ /x86_64/ || $arch =~ /ppc64le/ || $arch =~ /ppc64el/) {
        &runcmd("rpower $confkeys{$mn} boot");
    } else {
        &runcmd("rnetboot $confkeys{$mn}");
    }

    &runcmd("sleep 600");
    &runcmd("a=0;while ! `lsdef -l $confkeys{$mn}|grep status|grep booted >/dev/null`; do sleep 30;((a++));if [ \$a -gt 100 ];then break;fi done");

    #	if($mn !~ /ubuntu/){
    #	    &runcmd("sleep 300");
    #	    &runcmd("a=0;while ! `lsdef -l $confkeys{$mn}|grep status|grep booted >/dev/null`; do sleep 10;((a++));if [ \$a -gt 200 ];then break;fi done");}
    #	else{
    #	    &runcmd("sleep 1800");
    #	}

    &runcmd("lsdef -l $confkeys{$mn}|grep status|grep booted >/dev/null");
    if ($?) {
        send_msg(0, "[$mn->$confkeys{$mn}][mn_install] install $osimage for node $confkeys{$mn} failed");
        return 1;
    }

    if ($mn =~ /ubuntu/) {
        system("xdsh $confkeys{$mn} \"rm -rf /bin/sh\" >/dev/null 2>&1");
        system("xdsh $confkeys{$mn} \"ln -s /bin/bash /bin/sh\"");
    }

    #	if($mn =~ /ubuntu/){
    #	    system("updatenode $confkeys{$mn} -P \"confignics -s\" >/dev/null 2>&1");
    #	    system("xdsh $confkeys{$mn} \"iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE\" >/dev/null 2>&1");
    #		system("xdsh $confkeys{$mn} \"iptables -A FORWARD -i eth1 -j ACCEPT\" >/dev/null 2>&1");
    #       system("xdsh $confkeys{$mn} \"iptables -A FORWARD -i eth1 -o eth0 -m state --state RELATED,ESTABLISHED -j ACCEPT\" >/dev/null 2>&1");
    #	}
    if ($os =~ /rhels7/) {
        system("xdsh $confkeys{$mn} \"yum -y install bzip2 > /var/log/xcat/install_xcat.log 2>&1\" >/dev/null 2>&1");
        system("xdsh $confkeys{$mn} \"yum -y install createrepo > /var/log/xcat/install_xcat.log 2>&1\" >/dev/null 2>&1");
    }
    return 0;
}

#######################################
# down dep
#######################################
sub down_dep {
    my $mn = shift;
    send_msg(2, "[$mn->$confkeys{$mn}][down xcatdep] starting to down the latest xcat dep");
    my $downxcatdepdir = "$regrootdir/curl/$mn";
    `mkdir -p $downxcatdepdir`;
    my $curllink = $confkeys{ $mn . "curllink" };
    my $deplink;
    if (!-f "$downxcatdepdir/curlhtml1") {
        send_msg(2, "it is first time to down xcatdep");
` curl  "$curllink/"  |grep "xcat-dep-"  > "$downxcatdepdir/curlhtml1" `;
        ` tail -1  "$downxcatdepdir/curlhtml1" >  "$downxcatdepdir/curlfile"`;
        $deplink = `cat "$downxcatdepdir"/"curlfile" |awk '{print \$6}' |sed 's/"/ /g' |awk '{print \$2}'  `;
        `mkdir -p $xcatdepdir/$mn`;
        ` wget  -P "$xcatdepdir/$mn" -c $curllink/$deplink `;
        if ($?) {
            send_msg(0, "download xcatdep failed");
            return 1;
        }
        $depupdateflag = 1;
    } else {
        `curl  "$curllink/"  |grep "xcat-dep-"  > "$downxcatdepdir/curlhtml2" `;
        ` diff  "$downxcatdepdir"/"curlhtml1"  "$downxcatdepdir"/"curlhtml2"`;
        if ($? == 0) {
            send_msg(2, "no update for  xcatdep ");
            $depupdateflag = 0;
        } else {
` tail -1  "$downxcatdepdir"/"curlhtml2"> "$downxcatdepdir"/"curlfile"`;
            $deplink = `cat "$downxcatdepdir"/"curlfile" |awk '{print \$6}' |sed 's/"/ /g' |awk '{print \$2}'  `;
            ` rm -rf  $xcatdepdir/$mn`;
            `mkdir -p $xcatdepdir/$mn`;
            ` wget  -P "$xcatdepdir/$mn" -c $curllink/$deplink `;
            if ($?) {
                send_msg(2, "download xcatdep failed");
                return 1;
            }
`mv  "$downxcatdepdir"/"curlhtml2"    "$downxcatdepdir"/"curlhtml1" -f `;
            $depupdateflag = 2;
        }
    }
    return ($depupdateflag);
}

#########################################
# down xcat-core build
###########################################
sub down_xcat_core_build {
    my $mn = shift;
    send_msg(2, "[$mn->$confkeys{$mn}][down xcatcore] starting to down the latest xcat core");
    my $downxcatcoredowndir = "$xcatcoredowndir/$mn";
    if (!-f "$xcatcoredowndir/$mn") {
        `mkdir -p $xcatcoredowndir/$mn`;
    }
    my $downxcatcoreip  = $confkeys{ $mn . "downxcatcoreip" };
    my $downxcatcoredir = $confkeys{ $mn . "downxcatcoredir" };
    my $latestxcatcoredir = `ssh  $downxcatcoreip ls -rt $downxcatcoredir / |tail -1  |awk '{printf \$0}'`;
    my $xcatcorename = $confkeys{ $mn . "xcatcorename" };
`scp -r $downxcatcoreip\:$downxcatcoredir/$latestxcatcoredir/$xcatcorename  $downxcatcoredowndir `;
    if ($? != 0) {
        send_msg(0, "[$mn->$confkeys{$mn}][down_xcat_core_build]  failed");
        return 1;
    }
}

#######################################
# copy code
#######################################
sub copy_code {
    my $mn = shift;

    send_msg(2, "[$mn->$confkeys{$mn}][copy_code] starting to copy the latest xcat code and dep");
    my $dnsserver = `cat /etc/hosts |grep $confkeys{$mn}|awk '{print \$1}'|head -n 1`;
    chomp($dnsserver);
    my $res = system("xdsh $confkeys{$mn} \"sed -i 's:nameserver .*:nameserver $dnsserver:g' /etc/resolv.conf\" >/dev/null 2>&1");
    if ($res != 0) {
        send_msg(0, "[$mn->$confkeys{$mn}][install_xcat] modify resolv.conf in $mn failed");
        return 1;
    }

    #copy xcat-dep tarball to MN:/
    my $depball = "";
    if ($mn =~ /aix/) {
        $depball = `ls -l $xcatdepdir/$mn|awk '{print \$9}'|grep aix|tail -n 1`;
    } elsif ($mn =~ /ubuntu/) {
        $depball = `ls -l $xcatdepdir/$mn|awk '{print \$9}'|grep ubuntu|tail -n 1`;
    } else {
        $depball = `ls -l $xcatdepdir/$mn|awk '{print \$9}'|grep 'xcat-dep'|tail -n 1`;
    }
    chomp($depball);
    print "depball is $depball\n";
    my $coreball = `ls -l $xcatcoredowndir/$mn|awk '{print \$9}'|tail -n 1`;
    chomp($coreball);
    if ($depball eq "") {
        send_msg(0, "[$mn->$confkeys{$mn}][copy_code] can't find dep tarball for $mn under dir $xcatdepdir");
        return 1;
    }
    send_msg(2, "[$mn->$confkeys{$mn}][copy_code] find dep tarball $depball for $mn");
    if ($coreball eq "") {
        send_msg(0, "[$mn->$confkeys{$mn}][copy_code] can't find core tarball for $mn under dir $coreball");
        return 1;
    }

    send_msg(2, "[$mn->$confkeys{$mn}][copy_code] find core tarball $coreball for $mn");

    $res = system("scp $xcatdepdir/$mn/$depball root\@$confkeys{$mn}:/  >/dev/null");
    if ($res) {
        send_msg(0, "[$mn->$confkeys{$mn}][copy_code] copy $depball to $mn failed");
        return 1;
    }
    send_msg(2, "[$mn->$confkeys{$mn}][copy_code] copy $depball to $mn successfully");

    $res = system("scp $xcatcoredowndir/$mn/$coreball root\@$confkeys{$mn}:/  >/dev/null");
    if ($res) {
        send_msg(0, "[$mn->$confkeys{$mn}][copy_code] copy $coreball to $mn failed");
        return 1;
    }
    send_msg(2, "[$mn->$confkeys{$mn}][copy_code] copy $coreball to $mn successfully");

    $res = system("xdsh $confkeys{$mn} 'cd / && tar xvf /$depball' >/dev/null 2>&1");
    if ($?) {
        send_msg(0, "[$mn->$confkeys{$mn}][copy_code] untar $depball in $mn failed");
        return 1;
    }
    send_msg(2, "[$mn->$confkeys{$mn}][copy_code] untar $depball in $mn successfully");
    $res = system("xdsh $confkeys{$mn} 'cd / && tar xvf /$coreball' >/dev/null 2>&1");
    if ($?) {
        send_msg(0, "[$mn->$confkeys{$mn}][copy_code] untar $coreball in $mn failed");
        return 1;
    }
    send_msg(2, "[$mn->$confkeys{$mn}][copy_code] untar $coreball in $mn successfully");

    #copy xcat-core source code to MN:/
    if ($mn =~ /ubuntux/) {

        #defect 107 Automation ENV replaces the link file to real file before running xcat build
        $res = system("xdsh $confkeys{$mn} 'mkdir -p /xcatbuild' && rsync -avz $xcatcoredir root\@$confkeys{$mn}:/xcatbuild/  >/dev/null");
    } else {

        #defect 107 Automation ENV replaces the link file to real file before running xcat build
        $res = system("rsync -avz $xcatcoredir root\@$confkeys{$mn}:/  >/dev/null");
    }
    if ($res) {
        send_msg(0, "[$mn->$confkeys{$mn}][copy_code] copy the latest xcat code to $mn failed");
        return 1;
    }
    send_msg(2, "[$mn->$confkeys{$mn}][copy_code] copy the latest xcat code to $mn successfully");

    #copy /etc/hosts file to MN
    $res = system("scp /etc/hosts root\@$confkeys{$mn}:/etc  >/dev/null");
    if ($?) {
        send_msg(0, "[$mn->$confkeys{$mn}][copy_code] copy /etc/hosts to $mn failed");
        return 1;
    }
    send_msg(2, "[$mn->$confkeys{$mn}][copy_code] copy /etc/hosts  to $mn successfully");

    my $os      = $confkeys{ $mn . "os" };
    my $regconf = $mn . ".conf";
    system("sed -i 's|os=\\S*|os=$os|g' $configfiledir/$regconf");

    my $arch = "ppc64";
    if ($mn !~ /aix/ && $mn =~ /x/) {
        $arch = "x86_64";
    } elsif ($mn !~ /aix/ && $mn =~ /LE/) {
        $arch = "ppc64le";
    } elsif ($mn !~ /aix/ && $mn =~ /EL/) {
        $arch = "ppc64el";
    }
    my $iso;

    #copy specific files to MN for each platform
    if ($mn =~ /rh/) {
        $res = system("scp /install/$os/$arch/RPM-GPG-KEY-redhat-release root\@$confkeys{$mn}:/  >/dev/null");
        if ($?) {
            send_msg(0, "[$mn->$confkeys{$mn}][copy_code] copy RPM-GPG-KEY-redhat-release to $mn failed");
            return 1;
        }
        send_msg(2, "[$mn->$confkeys{$mn}][copy_code] copy RPM-GPG-KEY-redhat-release to $mn successfully");

        $os =~ /(\D+)(\d+\.\d+)/;
        $iso = `ls -l $osisodir|awk '{print \$9}'|grep RHEL`;
        chomp($iso);

        #print "[$mn->$confkeys{$mn}][copy_code] iso=$iso\n";
        if ($iso eq "") {
            send_msg(0, "[$mn->$confkeys{$mn}][copy_code] can't find iso for $os and $arch under dir $osisodir");
            return 1;
        }
        send_msg(2, "[$mn->$confkeys{$mn}][copy_code] find $iso for os=$os and arch=$arch");

        $res = system("scp $osisodir/$iso root\@$confkeys{$mn}:/  >/dev/null");
        if ($?) {
            send_msg(0, "[$mn->$confkeys{$mn}][copy_code] copy ISO $iso to $mn failed");
            return 1;
        }
        send_msg(2, "[$mn->$confkeys{$mn}][copy_code] copy ISO $iso to $mn successfully");
        system("sed -i 's|^ISO=/\\S*|ISO=/$iso|g' $configfiledir/$regconf");
    }

    if ($mn =~ /sles/) {
        $os =~ /(\D+)(\d+)\.?(\d?)/;
        if ($3) {
            $iso = `ls -l $osisodir| awk '{print \$9}'|grep SLE|grep $arch |grep -- "-$2-SP$3-" |tail -n 1`;
        } else {
            $iso = `ls -l $osisodir| awk '{print \$9}'|grep SLE|grep $arch |grep -v "SP"|grep -- "-$2-"|tail -n 1`;
        }
        chomp($iso);

        #print "[$mn->$confkeys{$mn}][copy_code] iso=$iso\n";
        if ($iso eq "") {
            send_msg(0, "[$mn->$confkeys{$mn}][copy_code] can't find ISO for os=$os and arch=$arch under dir $osisodir");
            return 1;
        }
        send_msg(2, "[$mn->$confkeys{$mn}][copy_code] find $iso for os=$os and arch=$arch");

        $res = system("scp $osisodir/$iso root\@$confkeys{$mn}:/  >/dev/null");
        if ($?) {
            send_msg(0, "[$mn->$confkeys{$mn}][copy_code] copy ISO $iso to $mn failed");
            return 1;
        }
        send_msg(2, "[$mn->$confkeys{$mn}][copy_code] copy ISO $iso to $mn successfully");
        system("sed -i 's|^ISO=/\\S*|ISO=/$iso|g' $configfiledir/$regconf");

        if ($os =~ /sles11/) {
            $res = system("xdsh $confkeys{$mn}  \"mv /usr/lib/rpm/find-requires /usr/lib/rpm/find-requires.org\" && scp $osisodir/find-requires $confkeys{$mn}:/usr/lib/rpm/ > /dev/null");
            if ($?) {
                send_msg(0, "[$mn->$confkeys{$mn}][copy_code] For sles11, copy find-requires to $mn failed");
                return 1;
            }
            send_msg(2, "[$mn->$confkeys{$mn}][copy_code] For sles11, copy find-requires to $mn successfully");
        }
    }
    if ($mn =~ /ubuntu/) {
        $os =~ /(\D+)(\d+\.\d+\.?\d?)/;
        my $osversion = $2;
        if ($mn =~ /ubuntux/) {
            $iso = `ls -l $osisodir| awk '{print \$9}'|grep ubuntu|grep amd64 |grep $osversion |tail -n 1`;
        } elsif ($mn =~ /ubuntuEL/) {
            $iso = `ls -l $osisodir| awk '{print \$9}'|grep ubuntu|grep ppc64el |grep $osversion |tail -n 1`;
        }
        chomp($iso);

        #print "[$mn->$confkeys{$mn}][copy_code] iso=$iso\n";
        if ($iso eq "") {
            send_msg(0, "[$mn->$confkeys{$mn}][copy_code] can't find ISO for os=$os and arch=$arch under dir $osisodir");
            return 1;
        }
        send_msg(2, "[$mn->$confkeys{$mn}][copy_code] find $iso for os=$os and arch=$arch");

        $res = system("scp $osisodir/$iso root\@$confkeys{$mn}:/  >/dev/null");
        if ($?) {
            send_msg(0, "[$mn->$confkeys{$mn}][copy_code] copy ISO $iso to $mn failed");
            return 1;
        }
        send_msg(2, "[$mn->$confkeys{$mn}][copy_code] copy ISO $iso to $mn successfully");
        system("sed -i 's|^ISO=/\\S*|ISO=/$iso|g' $configfiledir/$regconf");
    }
    return 0;
}

#######################################
# config mn
#######################################
sub install_xcat {
    my $mn = shift;
    send_msg(2, "[$mn->$confkeys{$mn}][install_xcat] starting to install xcat in $confkeys{$mn}");

    my $os   = $confkeys{ $mn . "os" };
    my $arch = "ppc64";
    if ($mn !~ /aix/ && $mn =~ /x/) {
        $arch = "x86_64";
    } elsif ($mn !~ /aix/ && $mn =~ /LE/) {
        $arch = "ppc64le";
    } elsif ($mn !~ /aix/ && $mn =~ /EL/) {
        $arch = "ppc64el";
    }
    if ($mn =~ /ubuntu/) {
        system("xdsh $confkeys{$mn} \"echo \"nameserver 9.0.2.1\" >> /etc/resolv.conf\" >/dev/null 2>&1");
        system("xdsh $confkeys{$mn} 'apt-get -y install software-properties-common' >/dev/null 2>&1");
        if ($mn =~ /ubuntux/) {
            system("xdsh $confkeys{$mn} 'add-apt-repository \"deb http://archive.ubuntu.com/ubuntu \$(lsb_release -sc) main\"' >/dev/null 2>&1");
            system("xdsh $confkeys{$mn} 'add-apt-repository \"deb http://archive.ubuntu.com/ubuntu \$(lsb_release -sc)-updates main\"' >/dev/null 2>&1");
            system("xdsh $confkeys{$mn} 'add-apt-repository \"deb http://archive.ubuntu.com/ubuntu \$(lsb_release -sc) universe\"' >/dev/null 2>&1");
            system("xdsh $confkeys{$mn} 'add-apt-repository \"deb http://archive.ubuntu.com/ubuntu \$(lsb_release -sc)-updates universe\"' >/dev/null 2>&1");
        } elsif ($mn =~ /ubuntuEL/) {
            system("xdsh $confkeys{$mn} 'add-apt-repository \"deb http://ports.ubuntu.com/ubuntu-ports \$(lsb_release -sc) main\"' >/dev/null 2>&1");
            system("xdsh $confkeys{$mn} 'add-apt-repository \"deb http://ports.ubuntu.com/ubuntu-ports \$(lsb_release -sc)-updates main\"' >/dev/null 2>&1");
            system("xdsh $confkeys{$mn} 'add-apt-repository \"deb http://ports.ubuntu.com/ubuntu-ports \$(lsb_release -sc) universe\"' >/dev/null 2>&1");
            system("xdsh $confkeys{$mn} 'add-apt-repository \"deb http://ports.ubuntu.com/ubuntu-ports \$(lsb_release -sc)-updates universe\"' >/dev/null 2>&1");
        }

        system("xdsh $confkeys{$mn} \"wget -O - \"http://sourceforge.net/projects/xcat/files/ubuntu/apt.key/download\" | apt-key add -\" >/dev/null 2>&1");
        system("xdsh $confkeys{$mn} 'apt-get clean all' >/dev/null 2>&1");
        system("xdsh $confkeys{$mn} 'apt-get update' >/dev/null 2>&1");
        system("xdsh $confkeys{$mn} 'apt-get -y install build-essential dpkg-dev dh-make debhelper fakeroot gnupg lintian pbuilder quilt reprepro libsoap-lite-perl libdbi-perl' >/dev/null 2>&1");
    }

    if ($mn =~ /rh/) {
        $os =~ /(\D+)(\d+)\.?(\d?)/;
        system("xdsh $confkeys{$mn} \"cd /xcat-core && ./mklocalrepo.sh\" >/dev/null 2>&1");
        system("xdsh $confkeys{$mn} \"cd /xcat-dep/rh$2/$arch && ./mklocalrepo.sh\" >/dev/null 2>&1");
        system("xdsh $confkeys{$mn} \"rpm --import /RPM-GPG-KEY-redhat-release\" >/dev/null 2>&1");
        system("xdsh $confkeys{$mn} \"yum -y install xCAT xCAT-test > /var/log/xcat/install_xcat.log 2>&1\" >/dev/null 2>&1");
        if ($os =~ /rhels7/) {
            system("xdsh $confkeys{$mn} \"yum -y install mariadb-devel  mariadb-libs  mariadb-server mariadb-bench mariadb  perl-DBD-MySQL  mysql-connector-odbc unixODBC\" >/dev/null 2>&1");
        } elsif ($os =~ /rhels6/) {
            system("xdsh $confkeys{$mn} \"yum -y install mysql-server mysql mysql-bench mysql-devel mysql-connector-odbc\" >/dev/null 2>&1");
        }
        system("xdsh $confkeys{$mn} \"source /etc/profile.d/xcat.sh\" >/dev/null 2>&1");
        if ($mn =~ /rhx/) {
            system("xdsh $confkeys{$mn} \"yum install -y perl-Sys-Virt\" >/dev/null 2>&1");
        }
    } elsif ($mn =~ /sles/) {
        $os =~ /(\D+)(\d+)\.?(\d?)/;
        system("xdsh $confkeys{$mn} \"zypper ar file:///xcat-dep/sles$2/$arch xCAT-dep\" >/dev/null 2>&1");
        system("xdsh $confkeys{$mn} \"zypper ar file:///xcat-core xCAT-core\" >/dev/null 2>&1");
        system("xdsh $confkeys{$mn} \"zypper sl -U\" >/dev/null 2>&1");
        system("xdsh $confkeys{$mn} \"zypper --gpg-auto-import-keys search --match-exact -s screen\" >/dev/null 2>&1");
        system("xdsh $confkeys{$mn} \"zypper -n install xCAT xCAT-test > /var/log/xcat/install_xcat.log 2>&1\" >/dev/null 2>&1");
        system("xdsh $confkeys{$mn} \"zypper -n install createrepo\" >/dev/null 2>&1");
        system("xdsh $confkeys{$mn} \"zypper -n install mysql-client libmysqlclient_r15  libqt4-sql-mysql libmysqlclient15 perl-DBD-mysql mysql unixODBC\" >/dev/null 2>&1");
        system("xdsh $confkeys{$mn} \"source /etc/profile.d/xcat.sh\" >/dev/null 2>&1");

        if ($mn =~ /slesLE/) {
            system("xdsh  $confkeys{$mn} \"zypper -n install perl-Net-DNS-0.80-1.ppc64le\">/dev/null 2>&1");
        }
        if ($mn =~ /slesx/) {
            system("xdsh $confkeys{$mn} \"zypper -n install perl-Sys-Virt\" >/dev/null 2>&1");
        }
    } elsif ($mn =~ /ubuntux/) {
        system("xdsh $confkeys{$mn} \"/xcat-core/mklocalrepo.sh\" >/dev/null 2>&1");
        system("xdsh $confkeys{$mn} \"/xcat-dep/mklocalrepo.sh\" >/dev/null 2>&1");
        system("xdsh $confkeys{$mn} 'apt-get clean all' >/dev/null 2>&1");
        system("xdsh $confkeys{$mn} 'apt-get update' >/dev/null 2>&1");
        system("xdsh $confkeys{$mn} 'apt-get -y install xcat xCAT-test  > /var/log/xcat/install_xcat.log 2>&1' >/dev/null 2>&1");
    } elsif ($mn =~ /ubuntuEL/) {
        system("xdsh $confkeys{$mn} \"/xcat-core/mklocalrepo.sh\" >/dev/null 2>&1");
        system("xdsh $confkeys{$mn} \"/xcat-dep/mklocalrepo.sh\" >/dev/null 2>&1");
        system("xdsh $confkeys{$mn} 'apt-get clean all' >/dev/null 2>&1");
        system("xdsh $confkeys{$mn} 'apt-get update' >/dev/null 2>&1");
        system("xdsh $confkeys{$mn} 'apt-get -y install xcat xCAT-test  > /var/log/xcat/install_xcat.log 2>&1' >/dev/null 2>&1");
        system("scp $osisodir/mini.iso  $confkeys{$mn}:/  ");
    }

    #check if MN is installed successful
    my $lsdefout = `xdsh $confkeys{$mn} '/opt/xcat/bin/lsdef -v'`;
    chomp($lsdefout);
    if ($lsdefout !~ /lsdef - Version/) {
        send_msg(0, "[$mn->$confkeys{$mn}][install_xcat] install xcat in $mn failed");
        return 1;
    }

    send_msg(2, "[$mn->$confkeys{$mn}][install_xcat] install xcat in $mn successfully");
    return 0;
}

#######################################
# do test
#######################################
sub do_test {
    my $mn       = shift;
    my $os       = $confkeys{ $mn . "os" };
    my $casestop = 1;

    send_msg(2, "[$mn->$confkeys{$mn}][do_test] starting to run regression test in $confkeys{$mn}");

    my $regconf = $mn . ".conf";
    my $res = system("scp $configfiledir/$regconf root\@$confkeys{$mn}:/opt/xcat/share/xcat/tools/autotest/default.conf  >/dev/null");
    if ($res != 0) {
        send_msg(0, "[$mn->$confkeys{$mn}][do_test] copy $configfiledir/$regconf to $mn failed");
        return 1;
    }
    send_msg(2, "[$mn->$confkeys{$mn}][do_test] copy $configfiledir/$regconf to $mn successfully");
    if ($os =~ /rhels7/) {
        system("scp  /opt/xcat/share/xcat/install/rh/service.rhels7.ppc64.otherpkgs.pkglist root\@$confkeys{$mn}:/opt/xcat/share/xcat/install/rh  >/dev/null");
    }
    my $diskfull_installation_flat_testcase                  = "";
    my $diskless_installation_flat_testcase                  = "";
    my $statelite_installation_flat_testcase                 = "";
    my $sn_installation_hierarchy_testcase                   = "";
    my $diskfull_installation_hierarchy_testcase             = "";
    my $diskless_installation_hierarchy_testcase             = "";
    my $statelite_installation_hierarchy_by_ramdisk_testcase = "";
    my $statelite_installation_hierarchy_by_nfs_testcase     = "";
    my $setup_dockerhost_testcase                            = "";
    my $batbundle                                            = "";

    if ((($mn =~ /sles/) || ($mn =~ /rh/)) && ($mn !~ /E/)) {
        $diskfull_installation_flat_testcase = "reg_linux_diskfull_installation_flat";
        $diskless_installation_flat_testcase = "reg_linux_diskless_installation_flat";
        $statelite_installation_flat_testcase = "reg_linux_statelite_installation_flat";
        $sn_installation_hierarchy_testcase = "reg_linux_SN_installation_hierarchy";
        $diskfull_installation_hierarchy_testcase = "reg_linux_diskfull_installation_hierarchy";
        $diskless_installation_hierarchy_testcase = "reg_linux_diskless_installation_hierarchy";
        $statelite_installation_hierarchy_by_ramdisk_testcase = "reg_linux_statelite_installation_hierarchy_by_ramdisk";
        $statelite_installation_hierarchy_by_nfs_testcase = "reg_linux_statelite_installation_hierarchy_by_nfs";
        $batbundle = "p_rhels_cmd.bundle" if ($mn =~ /rhp/);
        $batbundle = "x_rhels_cmd.bundle" if ($mn =~ /rhx/);
        $batbundle = "p_sles_cmd.bundle" if ($mn =~ /slesp/);
        $batbundle = "x_sles_cmd.bundle" if ($mn =~ /slesx/);
    } elsif ($mn =~ /ubuntux/) {
        $diskfull_installation_flat_testcase = "Ubuntu_full_installation_flat_x86_vm";
        $diskless_installation_flat_testcase = "Ubuntu_diskless_installation_flat_x86_vm";
        $setup_dockerhost_testcase = "Full_installation_flat_docker";
        $batbundle                 = "x_ubuntu_cmd.bundle";
    } elsif ($mn =~ /E/) {
        $diskfull_installation_flat_testcase = "Full_installation_flat_p8_le";
        $diskless_installation_flat_testcase = "Diskless_installation_flat_p8_le";
        $batbundle = "le_rhels_cmd.bundle" if ($mn =~ /rhLE/);
        $batbundle = "le_sles_cmd.bundle" if ($mn =~ /slesLE/);
        $batbundle = "le_ubuntu_cmd.bundle" if ($mn =~ /ubuntuEL/);
    }
    if (($mn !~ /ubuntu/) && ($mn !~ /E/)) {
        {
            my $rc;

            #diskfull installation in flat mode test
            send_msg(2, "[$mn->$confkeys{$mn}][do_test] doing diskfull installation test in flat mode in $mn.....");
            $rc = system("xdsh $confkeys{$mn} \"xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf  -t $diskfull_installation_flat_testcase\" >/dev/null 2>&1");
            send_msg(2, "[$mn->$confkeys{$mn}][do_test] do diskfull installation test in flat mode in $mn.....[done]");
            last if ($rc != 0);

            #commands bundle in flat mode test
            send_msg(2, "[$mn->$confkeys{$mn}][do_test] doing all commands test in flat mode in $mn.....");
            $rc = system("xdsh $confkeys{$mn} \"xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf  -b $batbundle\" >/dev/null 2>&1");
            send_msg(2, "[$mn->$confkeys{$mn}][do_test] do all commands test in flat mode in $mn.....[done]");
            last if ($rc != 0);

            #diskless installation in flat mode test
            send_msg(2, "[$mn->$confkeys{$mn}][do_test] doing diskless installation test in flat mode in $mn.....");
            $rc = system("xdsh $confkeys{$mn} \"xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf  -t $diskless_installation_flat_testcase\" >/dev/null 2>&1");
            send_msg(2, "[$mn->$confkeys{$mn}][do_test] do diskless installation test in flat mode in $mn.....[done]");
            last if ($rc != 0);

            #statelite installation in flat mode test
            send_msg(2, "[$mn->$confkeys{$mn}][do_test] doing statelite installation test in flat mode in $mn.....");
            $rc = system("xdsh $confkeys{$mn} \"xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf  -t $statelite_installation_flat_testcase\" >/dev/null 2>&1");
            send_msg(2, "[$mn->$confkeys{$mn}][do_test] do statelite installation test in flat mode in $mn.....[done]");
            last if ($rc != 0);

            #deploy SN
            send_msg(2, "[$mn->$confkeys{$mn}][do_test] setting up mysql for installing SN in $mn.....");
            $res = system("ssh -t $confkeys{$mn} 'exec bash -l -i -c \"XCATMYSQLADMIN_PW=12345 XCATMYSQLROOT_PW=12345 /opt/xcat/bin/mysqlsetup -i\"' >/dev/null 2>&1");
            if ($res != 0) {
                send_msg(0, "[$mn->$confkeys{$mn}][do_test] set up mysql for installing SN in $mn failed");
                return 1;
            }
            $res = system("xdsh $confkeys{$mn} \"echo \\\"GRANT ALL on xcatdb.* TO xcatadmin@\'%\'  IDENTIFIED BY \'12345\'\;\\\" | mysql -u root -p12345\" >/dev/null 2>&1");
            if ($res != 0) {
                send_msg(0, "[$mn->$confkeys{$mn}][do_test] grant right to mysql for installing SN in $mn failed");
                return 1;
            }
            send_msg(2, "[$mn->$confkeys{$mn}][do_test] set up mysql for installing SN in $mn.....[done]");

            send_msg(2, "[$mn->$confkeys{$mn}][do_test] preparing xcat ball for installing SN in $mn.....");
            my $os   = $confkeys{ $mn . "os" };
            my $arch = "ppc64";
            if (($mn !~ /aix/) && ($mn =~ /x/)) {
                $arch = "x86_64";
            } elsif ($mn !~ /aix/ && $mn =~ /LE/) {
                $arch = "ppc64le";
            } elsif ($mn !~ /aix/ && $mn =~ /EL/) {
                $arch = "ppc64el";
            }
            my $xcatballpath = "/install/post/otherpkgs/$os/$arch/xcat";
            $res = system("xdsh $confkeys{$mn} \"mkdir -p $xcatballpath/xcat-core && cp -r /xcat-core/* $xcatballpath/xcat-core && cp -r /xcat-dep $xcatballpath\"");
            if ($res != 0) {
                send_msg(0, "[$mn->$confkeys{$mn}][do_test] prepare xcat ball for installing SN in $mn failed");
                return 1;
            }
            send_msg(2, "[$mn->$confkeys{$mn}][do_test] prepare xcat ball for installing SN in $mn.....[done]");

            if ($mn =~ /rhp/) {
                my $ping_sn_on_rhp_pid = fork();
                if (!defined($ping_sn_on_rhp_pid)) {
                    send_msg(0, "fork process for trigger random number generator on redhat+ppc environment to install SN error");
                    exit;
                } elsif ($ping_sn_on_rhp_pid == 0) {
                    send_msg(2, "....fork 1 process[pid=$$] for trigger random number generator on redhat+ppc environment to install SN..........");
                    my $snnodename = `cat $configfiledir/$regconf |grep SN|grep =|awk -F'=' '{print \$2}'`;
                    chomp($snnodename);
                    &runcmd("sleep 900");
                    my $timecnt = 0;
                    while ($timecnt < 180) {
                        my $booted = system("xdsh $confkeys{$mn} \"lsdef -l $snnodename|grep status|grep booted\"");
                        last if ($booted == 0);

                        my $booting = system("xdsh $confkeys{$mn} \"lsdef -l $snnodename|grep status|grep booting\"");
                        system("xdsh $confkeys{$mn} \"xdsh $snnodename \"echo 1\"\"") if ($booting == 0);

                        &runcmd("sleep 10");
                        $timecnt++;
                    }
                    send_msg(2, "....process[pid=$$] for trigger random number generator on redhat+ppc environment to install SN....[exit]");
                    exit 0;
                }
            }

            send_msg(2, "[$mn->$confkeys{$mn}][do_test] installing SN.....");
            $rc = system("xdsh $confkeys{$mn} \"xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf  -t $sn_installation_hierarchy_testcase\" >/dev/null 2>&1");
            send_msg(2, "[$mn->$confkeys{$mn}][do_test] install SN.....[done]");
            last if ($rc != 0);

            #diskfull installation in hierarchy mode test
            send_msg(2, "[$mn->$confkeys{$mn}][do_test] doing diskfull installation test in hierarchy mode in $mn.....");
            $rc = system("xdsh $confkeys{$mn} \"xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf  -t $diskfull_installation_hierarchy_testcase\" >/dev/null 2>&1");
            send_msg(2, "[$mn->$confkeys{$mn}][do_test] do diskfull installation test in hierarchy mode in $mn.....[done]");
            last if ($rc != 0);

            #diskless installation in hierarchy mode test
            send_msg(2, "[$mn->$confkeys{$mn}][do_test] doing diskless installation test in hierarchy mode in $mn.....");
            $rc = system("xdsh $confkeys{$mn} \"xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf  -t $diskless_installation_hierarchy_testcase\" >/dev/null 2>&1");
            send_msg(2, "[$mn->$confkeys{$mn}][do_test] do diskless installation test in hierarchy mode in $mn.....[done]");
            last if ($rc != 0);

            #statelite installation by ramdisk in hierarchy mode test
            send_msg(2, "[$mn->$confkeys{$mn}][do_test] doing statelite installation by ramdisk test in hierarchy mode in $mn.....");
            $rc = system("xdsh $confkeys{$mn} \"xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf  -t $statelite_installation_hierarchy_by_ramdisk_testcase\" >/dev/null 2>&1");
            send_msg(2, "[$mn->$confkeys{$mn}][do_test] do statelite installation by ramdisk test in hierarchy mode in $mn.....[done]");
            last if ($rc != 0);

            #statelite installation by nfs in hierarchy mode test
            send_msg(2, "[$mn->$confkeys{$mn}][do_test] doing statelite installation by nfs test in hierarchy mode in $mn.....");
            $rc = system("xdsh $confkeys{$mn} \"xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf  -t $statelite_installation_hierarchy_by_nfs_testcase\" >/dev/null 2>&1");
            send_msg(2, "[$mn->$confkeys{$mn}][do_test] do statelite installation by nfs test in hierarchy mode in $mn.....[done]");
            last if ($rc != 0);

            $casestop = 0;
        }
    } else {
        {
            my $rc = 0;

            #diskless installation in flat mode test
            send_msg(2, "[$mn->$confkeys{$mn}][do_test] doing diskless installation test in flat mode in $mn.....");
            $rc = system("ssh -t $confkeys{$mn} 'exec bash -l -i -c \"xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf  -t $diskless_installation_flat_testcase\"' >/dev/null 2>&1");
            send_msg(2, "[$mn->$confkeys{$mn}][do_test] do diskless installation test in flat mode in $mn.....[done]");
            last if ($rc != 0);

            #diskfull installation in flat mode test
            send_msg(2, "[$mn->$confkeys{$mn}][do_test] doing diskfull installation test in flat mode in $mn.....");
            $rc = system("ssh -t $confkeys{$mn} 'exec bash -l -i -c \"xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf  -t $diskfull_installation_flat_testcase\"' >/dev/null 2>&1");
            send_msg(2, "[$mn->$confkeys{$mn}][do_test] do diskfull installation test in flat mode in $mn.....[done]");
            last if ($rc != 0);

            if ($mn =~ /ubuntux/) {

                #set up dockerhosts
                send_msg(2, "[$mn->$confkeys{$mn}][do_test] setupdockerhost $mn.....");
                $rc = system("ssh -t $confkeys{$mn} 'exec bash -l -i -c \"xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf  -t $setup_dockerhost_testcase\"' >/dev/null 2>&1");
                send_msg(2, "[$mn->$confkeys{$mn}][do_test] setup dockerhosts $mn.....[done]");
                last if ($rc != 0);
            }

            #commands bundle in flat mode test
            send_msg(2, "[$mn->$confkeys{$mn}][do_test] doing all commands test in flat mode in $mn.....");
            $rc = system("ssh -t $confkeys{$mn} 'exec bash -l -i -c \"xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf  -b $batbundle\"' >/dev/null 2>&1");
            send_msg(2, "[$mn->$confkeys{$mn}][do_test] do all commands test in flat mode in $mn.....[done]");
            $casestop = 0;
        }
    }

    if ($casestop) {
        send_msg(2, "[$mn->$confkeys{$mn}][do_test] the regression job in $mn was broken by one case.");
    } else {
        send_msg(2, "[$mn->$confkeys{$mn}][do_test] run the whole regression test in $mn finished");
    }

    send_msg(2, "[$mn->$confkeys{$mn}][install_xcat] run regression test in $mn finished");

    $res = system("scp -r $confkeys{$mn}:/opt/xcat/share/xcat/tools/autotest/result/* $todayregresultdir/$mn/ >/dev/null 2>&1");
    if ($res != 0) {
        send_msg(0, "[$mn->$confkeys{$mn}][do_test] copy regression result to $todayregresultdir/$mn/ failed");
        return 1;
    }
    send_msg(2, "[$mn->$confkeys{$mn}][do_test] copy regression result to $todayregresultdir/$mn/ successfully");

    return 0;

}

#######################################
# creat report
#######################################
sub creat_report {

    my $mailreport .= "======================================\n";
    $mailreport    .= "       Regression Result Summary\n";
    $mailreport    .= "======================================\n\n";

    # $gitupdateflag = 0 git update out of time
    # $gitupdateflag = 1 there isn't new code checkin in last one day
    # $gitupdateflag = 2 git update failed
    if ($gitupdateflag == 0) {
        $mailreport .= "Get the latest xcat code from git out of time, stop regression test today!!!\n\n\n";
        &runcmd("touch $mailfile && echo \"$mailreport\" > $mailfile");
        &runcmd("mv $commitinfofile $commitinfofile.old && echo \"\" > $commitinfofile");
        return 0;
    } elsif ($gitupdateflag == 1 && !$forceregwithoutupdate) {
        $mailreport .= "There isn't new code checkin last one day, stop regression test today!!!\n\n\n";
        &runcmd("touch $mailfile && echo \"$mailreport\" > $mailfile");
        &runcmd("mv $commitinfofile $commitinfofile.old && echo \"\" > $commitinfofile");
        return 0;
    } elsif ($gitupdateflag == 1 && $forceregwithoutupdate) {
        $mailreport .= "There isn't new code checkin last one day, But force to run regresson today!!!\n\n\n";
        &runcmd("mv $commitinfofile $commitinfofile.old && echo \"\" > $commitinfofile");
    } elsif ($gitupdateflag == 2) {
        $mailreport .= "Get the latest xcat source code from git error, stop regression test today!!!\n\n\n";
        &runcmd("touch $mailfile && echo \"$mailreport\" > $mailfile");
        &runcmd("mv $commitinfofile $commitinfofile.old && echo \"\" > $commitinfofile");
        return 0;
    }

    while (my ($k, $v) = each %pidrecord) {
        my $scenario = "[" . $confkeys{ $k . "os" };
        if ($k =~ /p/) {
            $scenario .= "+PPC64] ";
        } elsif ($k =~ /E/) {
            $scenario .= "+PPC64LE] ";
        } elsif ($k =~ /x/) {
            $scenario .= "+x86_64] ";
        }
        $mailreport .= "$scenario\n";
        my $batbundle = "";
        my $totalcase = 0;

        if ($k =~ /slesp/ || $k =~ /rhx/ || $k =~ /slesx/ || $k =~ /rhp/) {
            $batbundle = "$xcatcoredir/xCAT-test/autotest/bundle/p_rhels_cmd.bundle" if ($k =~ /rhp/);
            $batbundle = "$xcatcoredir/xCAT-test/autotest/bundle/x_rhels_cmd.bundle" if ($k =~ /rhx/);
            $batbundle = "$xcatcoredir/xCAT-test/autotest/bundle/p_sles_cmd.bundle" if ($k =~ /slesp/);
            $batbundle = "$xcatcoredir/xCAT-test/autotest/bundle/x_sles_cmd.bundle" if ($k =~ /slesx/);
            $totalcase = `cat $batbundle|wc -l`;
            $totalcase += 8;
        } elsif ($k =~ /ubuntux/) {
            $batbundle = "/$xcatcoredir/xCAT-test/autotest/bundle/x_ubuntu_cmd.bundle";
            $totalcase = `cat $batbundle|wc -l`;
            $totalcase += 3;
        } elsif ($k =~ /E/) {
            $batbundle = "$xcatcoredir/xCAT-test/autotest/bundle/le_rhels_cmd.bundle" if ($k =~ /rhLE/);
            $batbundle = "$xcatcoredir/xCAT-test/autotest/bundle/le_sles_cmd.bundle" if ($k =~ /slesLE/);
            $batbundle = "$xcatcoredir/xCAT-test/autotest/bundle/le_ubuntu_cmd.bundle" if ($k =~ /ubuntuEL/);
            $totalcase = `cat $batbundle|wc -l`;
            $totalcase += 2;
        }


        if ($v == 0) {    # whole regression test finished
            my $h           = int($consumptionrecord{$k} / 3600);
            my $m           = int(($consumptionrecord{$k} - $h * 3600) / 60);
            my $s           = ($consumptionrecord{$k} - $h * 3600) % 60;
            my $consumption = "$h hours $m minutes $s seconds";
            $mailreport .= "\tTime consumption: $consumption\n";

            if ($piderrrecord{$k} ne "") {
                $mailreport .= "\tDeploy MN result: $piderrrecord{$k}\n";
                $mailreport .= "\tTotalCase $totalcase TotalRun 0 Failed 0\n";
                next;
            }

            my $totalcnt = 0;
            my $failcnt  = 0;
            my $faillist = "";
            opendir(DIR, "$todayregresultdir/$k");
            foreach my $file (readdir DIR)
            {
                next if ($file !~ /xcattest.log/);
                my $cnt = `cat $todayregresultdir/$k/$file |grep -- "------END::"|wc -l`;
                $totalcnt += int($cnt);
                for (my $i = 1 ; $i < $cnt + 1 ; $i++) {
                    my $line = `cat $todayregresultdir/$k/$file |grep -- "------END::"|sed -n ${i}p`;
                    chomp($line);
                    if ($line =~ /------END::([a-zA-Z0-9_-]+)::([a-zA-Z0-9_-]+)::Time.+/) {
                        my $failedcase = $1;
                        if ($2 =~ /Failed/) {
                            $failcnt++;
                            $faillist .= $failedcase . ", ";
                        }
                    }
                }
            }

            closedir(DIR);

            my $lastfile = `ls -lrt $todayregresultdir/$k|grep "xcattest.log"|tail -1|awk '{print \$9}'`;
            chomp($lastfile);
            my $lastline = `cat $todayregresultdir/$k/$lastfile |grep -- "------END::"|tail -1`;
            chomp($lastline);
            my $lastcase;
            $lastcase = $1 if ($lastline =~ /------END::([a-zA-Z0-9_-]+)::([a-zA-Z0-9_-]+)::Time.+/);

            $mailreport .= "\tTotalCase $totalcase TotalRun $totalcnt Failed $failcnt\n\tFailed cases: $faillist\n\tFinal environment stop at $lastcase\n";
        } else {    # regression test out of time
            $mailreport .= "\tTime consumption: Out of 9 hours!!!!!!\n";
        }
    }

    &runcmd("touch $mailfile && echo \"$mailreport\" > $mailfile");
    return 0;
}

#######################################
# send email
#######################################
sub send_email {
    my $checkinlist = "";
    my $cnt = `cat  $commitinfofile |grep '20[0-9][0-9] [-+]0[0-9]00'|awk '{print \$1}' |grep -v root|sort | uniq|wc -l`;
    my $out = `cat  $commitinfofile |grep '20[0-9][0-9] [-+]0[0-9]00'|awk '{print \$1}' |grep -v root|sort | uniq`;
    for (my $i = 1 ; $i < $cnt + 1 ; $i++) {
        my $tmpline = `echo "$out"|sed -n ${i}p`;
        chomp($tmpline);
        $checkinlist .= "$tmpline ";
    }

    my $totalcnt  = 0;
    my $failcnt   = 0;
    my $totalcase = 0;
    $cnt = `cat  $mailfile |grep "Total"|grep "Failed"|wc -l`;
    $out = `cat  $mailfile |grep "Total"|grep "Failed"`;
    for (my $i = 1 ; $i < $cnt + 1 ; $i++) {
        my $tmpline = `echo "$out"|sed -n ${i}p`;
        chomp($tmpline);
        if ($tmpline =~ /TotalCase (\d+) TotalRun (\d+) Failed (\d+)/) {
            $totalcase += $1;
            $totalcnt  += $2;
            $failcnt   += $3;
        }
    }

    # $gitupdateflag = 0 git update out of time
    # $gitupdateflag = 1 there isn't new code checkin in last one day
    # $gitupdateflag = 2 git update failed
    $depupdateflag = `cat $logfiledir/$logfile |grep "updated for xcat dep" `;
    my $subject = "";
    if ($gitupdateflag == 0 || $gitupdateflag == 2) {
        $subject = "[$mailtitle] Git update failed, stop regression test today!!!";
    } elsif ($gitupdateflag == 1 && !$forceregwithoutupdate) {
        $subject = "[$mailtitle] No code checkin, stop regression test today!!!";
    } elsif ($gitupdateflag == 1 && $forceregwithoutupdate) {
        `cat $logfiledir/$logfile |grep "updated for xcat dep" `;
        if ($? == 0) {
            $subject = "[$mailtitle] TotalCase:" . $totalcase . ", TotalRun:" . $totalcnt . ", Fail:" . $failcnt . ", Branch:" . $branch . ", xcatdep updated, Checkin: NO ONE !!";
        } else {
            $subject = "[$mailtitle] TotalCase:" . $totalcase . ", TotalRun:" . $totalcnt . ", Fail:" . $failcnt . ", Branch:" . $branch . ", xcatdep no updated, Checkin: NO ONE !!";
        }
    } else {
        `cat $logfiledir/$logfile |grep "updated for xcat dep" `;
        if ($? == 0) {
            $subject = "[$mailtitle] TotalCase:" . $totalcase . ", TotalRun:" . $totalcnt . ", Fail:" . $failcnt . ", Branch:" . $branch . ", xcatdep updated, Checkin: " . $checkinlist;
        } else {
            $subject = "[$mailtitle] TotalCase:" . $totalcase . ", TotalRun:" . $totalcnt . ", Fail:" . $failcnt . ", Branch:" . $branch . ", xcatdep no updated, Checkin: " . $checkinlist;
        }
    }
    my $mailreport .= "\n\n======================================\n";
    $mailreport    .= "       Commit Codes Last Day\n";
    $mailreport    .= "======================================\n\n";

    &runcmd("echo \"$mailreport\">> $mailfile");
    &runcmd("cat $commitinfofile >> $mailfile");

    my $attachfile = "/tmp/failed_case_detail.txt";
    &runcmd("echo \"\" > $attachfile");
    if ($failcnt > 0) {
        opendir(DIR, "$todayregresultdir");
        foreach my $subdir (readdir DIR) {
            next if ($subdir =~ /^\./);

            #print "subdir-> $subdir\n";

            my $scenario = "[" . $confkeys{ $subdir . "os" };
            if ($subdir =~ /p/) {
                $scenario .= "+PPC64] ";
            } elsif ($subdir =~ /x/) {
                $scenario .= "+x86_64] ";
            } elsif ($subdir =~ /E/) {
                $scenario .= "+ppc64le] ";
            }

            my $separator .= "\n###############################################################\n";
            $separator .= "                    $scenario\n";
            $separator .= "###############################################################\n\n";
            &runcmd("echo \"$separator\" >> $attachfile");
            opendir(SUBDIR, "$todayregresultdir/$subdir");
            foreach my $file (readdir SUBDIR) {
                next if ($file !~ /^failedcases/);
                next if (-z "$todayregresultdir/$subdir/$file");

                #print "\t$file\n";
                system("cat $todayregresultdir/$subdir/$file >> $attachfile");
            }
        }
    }

    if ($mail_list) {
        if ($failcnt > 0) {
            system("cat $mailfile" . " | /bin/mail -s \"$subject\" -a $attachfile \"$mail_list\"");
        } else {
            system("cat $mailfile" . " | /bin/mail -s \"$subject\" \"$mail_list\"");
        }
    } else {
        if ($failcnt > 0) {
            system("cat $mailfile" . " | /bin/mail -s \"$subject\" -a $attachfile \"$confkeys{mailgroup}\"");
        } else {
            system("cat $mailfile" . " | /bin/mail -s \"$subject\" \"$confkeys{mailgroup}\"");
        }
    }
    return 0;
}


#######################################
#git update
#return value:
#0 successful
#1 code is already at latest version
#2 someting error
#######################################
sub git_update {
    send_msg(2, "[git update] starting to update xcat source code from git");

    my $res = system("cd $xcatcoredir && git checkout $branch > /dev/nul 2>&1");
    if ($res != 0) {
        send_msg(0, "[git update] change to branch $branch....[failed]");
        return 2;
    }
    send_msg(2, "[git update] change to branch $branch....[ok]");

    my $gitpulloutput = "/tmp/gitpulloutput";
    my $trytime       = 1;
    while ($trytime < 4) {
        $res = system("cd $xcatcoredir && git pull > $gitpulloutput 2>&1");
        if ($res != 0) {
            send_msg(0, "[git update] pull the latest code to control node....[$trytime failed]");
            $trytime++;
            &runcmd("sleep 120");
        } else {
            last;
        }
    }

    if ($trytime == 4) {
        send_msg(0, "[git update] pull the latest code to control node....[finally failed]");
        return 2;
    }

    $res = system("grep 'Already up-to-date' $gitpulloutput > /dev/nul 2>&1");
    if ($res == 0 && !$forceregwithoutupdate) {
        send_msg(2, "[git update] code is already at latest version. exit regresson");
        return 1;
    } elsif ($res == 0 && $forceregwithoutupdate) {
        send_msg(2, "[git update] code is already at latest version. But force to run regresson");
        return 0;
    }
    send_msg(2, "[git update] pull the latest code to control node....[ok]");

    my $lastcommitfile = "/tmp/lastcommitfile";
    my $lastcommitinfo;
    if (-f $lastcommitfile) {
        $lastcommitinfo = `cat $lastcommitfile`;
        chomp($lastcommitinfo);
    } else {
        $lastcommitinfo = "";
    }

    #print "lastcommitinfo=$lastcommitinfo\n";

    if ($lastcommitinfo eq "") {
        $res = system("cd $xcatcoredir && git log --since=\"1 days ago\"  --pretty=format:\"%cn %ad %s\" --name-status > $commitinfofile");
        if ($res != 0 || (-f $commitinfofile and -z _)) {
            send_msg(0, "[git update] get commit information....[failed]");
            return 2;
        }
        send_msg(0, "[git update] get commit information....[ok]");
    } else {
        $res = system("cd $xcatcoredir && git log $lastcommitinfo..HEAD  --pretty=format:\"%cn %ad %s\" --name-status > $commitinfofile");
        if ($res != 0 || (-f $commitinfofile and -z _)) {
            send_msg(0, "[git update] get commit information....[failed]");
            return 2;
        }
        send_msg(2, "[git update] get commit information....[ok]");
    }

    $res = system("cd $xcatcoredir && git log -n 1|awk 'NR==1'|awk '{print \$2}' > $lastcommitfile");
    if ($res != 0) {
        send_msg(0, "[git update] get latest commit number failed");
        exit;
    }

    return 0;
}

#######################################
# step 0. Parse input arguments
#######################################
send_msg(2, "step 1, Parse input arguments............");

=pod
if (
    !GetOptions("--help|h|?" => \$needhelp,
                "f=s" => \$configfile,
                "b=s" => \$branch,
                "m=s" => \$management_node,
                "V"   => \$verbose_mode,
                "U"   => \$updates_regression,
                "e=s" => \$send_msg)
 )
=cut

if (
    !GetOptions("--help|h|?" => \$needhelp,
        "f=s" => \$configfile,
        "b=s" => \$branch,
        "m=s" => \$management_node,
        "a"   => \$forceregwithoutupdate,
        "e=s" => \$mail_list)
  )
{
    &usage;
    send_msg(0, "step 0, PARSE ARGUMENTS returns error, exit");
    exit;
}
if ($needhelp)
{
    &usage;
    exit 0;
}


xCATreg_init();

###############################################################
# Mainfunction
###############################################################
send_msg(2, "........................");
send_msg(2, "........................");
send_msg(2, ".....ooooO..............");
send_msg(2, "....(....)....Ooooo.....");
send_msg(2, ".....\\..(.....(....)....");
send_msg(2, "......\\__).....)../.....");
send_msg(2, "..............(_ /......");
send_msg(2, "........................");
send_msg(2, ".........START .........");
send_msg(2, "........................");
#######################################
# step 1. Read configuration files
#######################################
send_msg(2, "step 2, Reading xCATreg conf............");
my $rst = read_conf();
unless (!$rst) {
    send_msg(0, "read xCATreg configuration failed...........exit");
    exit;
}

foreach my $k (keys %confkeys) {
    print "$k = $confkeys{$k}\n";
}

# begin to find which mn should be used
if ($management_node) {
    my @all;
    if ($management_node =~ /,/) {
        @all = split /,/, $management_node;
    } else {
        push @all, $management_node;
    }
    foreach my $mn (@all) {
        foreach my $k (keys %confkeys) {
            if ($confkeys{$k} eq $mn) {
                $mns{$k} = $mn;
                send_msg(2, "specify mn :$mn");
            }
        }
    }
}

pipe PREAD, CWRITE;
my $gitpid = fork();
if (!defined($gitpid)) {
    send_msg(0, "fork process for git update error");
    exit;
} elsif ($gitpid == 0) {
    close PREAD;
    send_msg(2, "..........fork 1 process[pid=$$] for git update..........");
    my $res = git_update();
    if ($res > 1) {
        send_msg(0, "git_update......[failed]");
        syswrite CWRITE, "git update failed\n";
        close CWRITE;
        exit 1;
    }
    if ($res == 1) {
        send_msg(2, "git_update......[not find new code]");
        syswrite CWRITE, "git update got nothing\n";
        close CWRITE;
        exit 1;
    }
    send_msg(2, "git_update......[ok]");
    syswrite CWRITE, "git update successful\n";
    close CWRITE;
    exit 0;
}

{    #main process
    close CWRITE;
    my $gitstarttime = time();
    my $gitprocess   = 1;
    my $gitselect    = new IO::Select;
    $gitselect->add(\*PREAD);
    while ($gitprocess) {
        my @hdls;
        if (@hdls = $gitselect->can_read(0)) {
            my $hdl;
            foreach $hdl (@hdls) {
                if ($hdl == \*PREAD) {
                    my $line;
                    chomp($line = <PREAD>);
                    if ($line) {
                        if ($line =~ /successful/) {
                            send_msg(2, "[[main]]: $line");
                            $gitupdateflag = 3;
                        }
                        if ($line =~ /failed/) {
                            send_msg(0, "[[main]]: $line");
                            $gitupdateflag = 2;
                        }
                        if ($line =~ /nothing/) {
                            send_msg(2, "[[main]]: there isn't new code checkin during last one day.");
                            $gitupdateflag = 1;
                        }
                        $gitprocess = 0;
                    }
                }
            }
        }

        if (time() - $gitstarttime > 7200) {

            #if(time() - $gitstarttime > 2) {
            send_msg(0, "[[main]] 2 hours is expired. git update out of time");
            last;
        }
        &runcmd("sleep 2");
    }
    close PREAD;

    &runcmd("kill $gitpid") if ($gitprocess == 1);
    send_msg(2, "[[main]]........git process[$gitpid] exit successful..............");

    # $gitupdateflag = 0 git update out of time
    # $gitupdateflag = 1 there isn't new code checkin in last one day
    # $gitupdateflag = 2 there is something wrong during git update
    if ($gitupdateflag < 3) {
        send_msg(2, "[[main]]: creat regression result report...........");
        $rst = creat_report();
        if ($rst) {
            send_msg(0, "[[main]]: creat regression result report...........[FAILED]");
        } else {
            send_msg(2, "[[main]]: creat regression result report...........[OK]");

            send_msg(2, "[[main]]: send report email...........");
            $rst = send_email();
            if ($rst) {
                send_msg(0, "[[main]]: send report email...........[FAILED]");
            } else {
                send_msg(2, "[[main]]: send report email...........[OK]");
            }
        }
        exit 1;
    }
    send_msg(2, "[[main]]: git update successful, start regression today........");

    #exit 0
}


my $platformfork = 0;
my $timingork    = 0;

# begin child process
pipe CONTROLREAD, MNWRITE;
foreach my $m (keys %mns) {
    $platformfork += 1;
    my $mn  = $mns{$m};
    my $pid = fork();
    if (!defined($pid)) {
        send_msg(0, "fork process for $m error");
        exit;
    } elsif ($pid == 0) {    # child process
        send_msg(2, "..........fork process[pid=$$] for $m [$confkeys{$m}]..........");
        close CONTROLREAD;
        my $res;

        #######################################
        # step 3. download xcat core
        ########################################
        send_msg(2, "[$m]step 3, Running down_xcat_core...............");
        $res = down_xcat_core_build($m);
        if ($res) {
            syswrite MNWRITE, "REPORTFROM:$m->$mn: download xcat core for $mn failed\n";
            exit 1;
        }
        send_msg(2, "[$m]step 3, Run down_xcat_core...............[OK]");

        #######################################
        # step 4. Install MNs,
        #######################################
        send_msg(2, "[$m]step 4, Running mn_install...............");
        $res = mn_install($m);
        if ($res) {
            syswrite MNWRITE, "REPORTFROM:$m->$mn: install $mn failed\n";
            exit 1;
        }
        send_msg(2, "[$m]step 4, Run mn_install...............[OK]");
        #######################################
        # step 5. Down xcatdep to MNs
        #######################################
        send_msg(2, "[$m]step 5, Down xcatdep...............");
        $depupdateflag = down_dep($m);
        if ($depupdateflag == 2) {
            send_msg(2, "[$m]step 5,  updated for xcat dep");
        } else {
            send_msg(2, "[$m]step 5, no update for xcat dep");
        }
        #######################################
        # step 6. Copy code to MNs
        #######################################
        send_msg(2, "[$m]step 6, Running copy_code...............");
        $res = copy_code($m);
        if ($res) {
            syswrite MNWRITE, "REPORTFROM:$m->$mn: copy xcat source code to $mn failed\n";
            exit 1;
        }
        send_msg(2, "[$m]step 6, Run copy_code...............[OK]");

        ########################################
        # step 7. Read xCAT MN's configuration
        #######################################
        send_msg(2, "[$m]step 7, Running install_xcat...............");
        $res = install_xcat($m);
        if ($res) {
            syswrite MNWRITE, "REPORTFROM:$m->$mn: install new xcat in $mn failed\n";
            exit 1;
        }
        send_msg(2, "[$m]step 7, Run install_xcat...............[OK]");

        #######################################
        # step 8. Genrate local configuration file for xcattest
        #         Do test
        #         Write log
        #######################################
        send_msg(2, "[$m]step 8, Running do_test...............");
        $res = do_test($m);
        if ($res) {
            syswrite MNWRITE, "REPORTFROM:$m->$mn: run regression test cases in $mn failed\n";
            exit 1;
        }
        send_msg(2, "[$m]step 8,  Running do_test...............[OK]");

        send_msg(2, "whole regression test for $mn finished");
        syswrite MNWRITE, "REPORTFROM:$m->$mn: whole regression test are successful\n";
        exit 0;
    }    # end of child process
    &runcmd("mkdir -p $todayregresultdir/$m");
    $pidrecord{$m}    = $pid;
    $piderrrecord{$m} = "";
}    # end of foreach


{    # parent process
    close MNWRITE;
    my $rst;
    my $regstarttime = time();

    my $select = new IO::Select;
    $select->add(\*CONTROLREAD);
    while ($platformfork) {
        my @hdls;
        if (@hdls = $select->can_read(0)) {
            my $hdl;
            foreach $hdl (@hdls) {
                if ($hdl == \*CONTROLREAD) {
                    my $line;
                    chomp($line = <CONTROLREAD>);
                    if ($line) {
                        if ($line =~ /successful/) {
                            send_msg(2, "[[main]]: $line");
                            $line =~ s/^REPORTFROM:(.+)->.+/$1/g;
                            $pidrecord{$line}         = 0;
                            $consumptionrecord{$line} = time() - $regstarttime;
                        }
                        if ($line =~ /failed/) {
                            send_msg(0, "[[main]]: $line");
                            my $tmp = $line;
                            $line =~ s/^REPORTFROM:(.+)->.+/$1/g;
                            $pidrecord{$line}         = 0;
                            $consumptionrecord{$line} = time() - $regstarttime;
                            $tmp =~ s/REPORTFROM:.+->.+:(.+)/$1/g;
                            $piderrrecord{$line} = $tmp;
                        }
                        $platformfork -= 1;
                    }
                }
            }
        }

        if (time() - $regstarttime > 36000) {

            #if($curtime - $regstarttime > 10) {
            send_msg(1, "[timing] 8 hours is expired");
            last;
        }
        &runcmd("sleep 1");
    }

    close CONTROLREAD;
    if ($platformfork) {
        send_msg(0, "[[main]]: there are $platformfork platform's regression test return out of time");
        while (my ($k, $v) = each %pidrecord) {
            &runcmd("kill $v") if ($v != 0);
        }
    } else {
        send_msg(2, "[[main]]: all platform's regression test return on time");
    }

    send_msg(2, "[[main]]: creat regression result report...........");
    $rst = creat_report();
    if ($rst) {
        send_msg(0, "[[main]]: creat regression result report...........[FAILED]");
    } else {
        send_msg(2, "[[main]]: creat regression result report...........[OK]");

        send_msg(2, "[[main]]: send report email...........");
        $rst = send_email();
        if ($rst) {
            send_msg(0, "[[main]]: send report email...........[FAILED]");
        } else {
            send_msg(2, "[[main]]: send report email...........[OK]");
        }
    }
    send_msg(2, "[[main]]: main process exit");
    exit 0;
}