#!/usr/bin/env perl
# IBM(c) 2010 EPL license http://www.eclipse.org/legal/epl-v10.html
#
#############################################################################################################################################
#
#   This script will be used to run xCAT daily regression
#
#   Usage:xCATreg - Run xCAT daily regression
#   xCATreg [-?|-h]
#   xCATreg [-b branch][-m MN][-f configure file][-a][-V]
#
#   -?/-h:  help information for the xCATreg usage.
#   -b: when -b is specified, the regression will be done against the specified branch. You can use 2.7, 2.8,master.The default value is master.
#   -m: when -m is specified, the regression will be done the specified mn. The default value is all the mn in configuration file.
#   -f :specify the configuration file,default value is /xCATreg/regresson.conf
#   -a: when -a is specified, run the regression even there is no code updates
#   -V: log and message in verbose mode
#
#   Example usage:
#   ./xCATreg   Run regressoins on master branch and on all platforms in configuration file
#   ./xCATreg -b 2.8 -m slesmn -f /root/regressoin.conf -a    Run regression on 2.8 branch on slesmn even when there is no code updates.
#
#############################################################################################################################################


use strict;
use warnings;
use Getopt::Long;
use Data::Dumper;
use Term::ANSIColor;
use Time::Local;

BEGIN
{
    $::XCATROOT = $ENV{'XCATROOT'} ? $ENV{'XCATROOT'} : -d '/opt/xcat' ? '/opt/xcat' : '/usr';
}
use lib "$::XCATROOT/lib/perl";
my %confhash;
my $rootdir = "$::XCATROOT/share/xcat/tools/autotest";
my $needhelp  = 0;
my $branch = 0;
my $rhppc64configfile = "$rootdir/default.conf";
my $configfile = "/regression/rhppc64/default.conf";
my $MN = undef;
my $management_node = undef;
my $CN = undef;
my $output = undef;
my $verbose_mode = 0;
my $updates_regression = 0;
my $send_msg = 0;
my $res = 0;
my $string1 = undef;
my $resultdir = "$rootdir/result";
mkdir $resultdir unless -d $resultdir;
my $timestamp = `date +"%Y%m%d%H%M%S"`;
my %rhppc64config=();
my %slesppc64config=();
my %rhx8664config=();
my %slesx8664config=();
my %rhppc64mnconfig=();
my $fullinst=0;
my $dsklsinst=0;
my $bundlerun=1;
my $stateliteinst=0;
my $snfullinst=0;
my $dsklscnsninst=0;
my $fullcnsninst=0;
my $statelitecnsninst=0;
my %confkeys;
my %results;
my %mns;
#######################################
# usage for arguments
#######################################
sub usage
{
    print "Usage:xCATreg - Run xcat autotest  test cases.\n
           Explanation for the options:
                                    -b: when -b is specified, the regression will be done against the specified branch. The default value is master.\n
                                    -m: when -m is specified, run regression on the specified mn. The default value is all the mn in configuration file.\n
                                    -f: specify the configuration file location.\n
                                    -U: when -U is specified, only code updates can trigger the regression.\n
                                    -V: log and message in verbose mode.\n
                                    -e: <email_addr>   send the test result to email_addr\n";
    print "  xCATreg [-?|-h]\n";
    print "  xCATreg [-f configure file] [-b branch] [-m mangement node][-V][-e] install mn \n";
    print "  xCATreg [-f configure file] [-b branch] [-m mangement node ] [-U][-V][-e] if code updates there will be regression.\n";
    print "\n";
    return;
}

#######################################
# config for rhppc64env
#######################################
sub config_rhppc64 {
    send_msg("******************************");
    send_msg("Reading Configure");
    send_msg("******************************");
    #if(!(-e $configfile)){
    #send_msg("Warning: The xCAT test Configure file doesn't exist!");
    #send_msg("Warning: The xCAT test Configure file doesn't exist!");
    #return 0;
    #}
    my $type = "";
    my $sub_type = undef; # The string after $type_
    # Script-->
    #       Script_Prev
    #       Script_Post
    # Table--->
    #       Table_xxxxx
    # Object-->
    #       Object_xxxx
    # System---->
    # Custom---->

    my $name = undef;
    my $attr = undef;
    my $value = undef;
    my $c = 0;
    my $cmd = undef;

    my $mgt_name = undef;
    open(FILE, "$configfile") or die "can't to open $configfile";
    while(my $line = <FILE>) {
        $line = &trim($line);
        next if(length($line) == 0);
        #Table name can not contain "_"
        if($line =~ /\[\s*(\w+)\_(\w+)\s*\]/) {
            $type = $1;
            $sub_type = $2;
            $name = undef;
            $c = 0;
        }elsif($line =~ /\[\s*rhppc64System|Custom\s*\]/){

         $type = "rhppc64Varible";
        }elsif ($type eq "rhppc64Table") {
            ##TABLE BLOCK##
            if($line =~ /(\w+)\s*=\s*([\w\.\-]+)/) {
                $attr = $1;
                $value = $2;
                if($name&&($rhppc64config{table}{$sub_type}{$name}{__KEY__} ne $attr)){
                    $rhppc64config{table}{$sub_type}{$name}{$attr}=$value;
                } else {
                    $name = $value;
                    $rhppc64config{table}{$sub_type}{$name}{__KEY__}=$attr;
                }
            }
        }elsif ($type eq "rhppc64Object") {
            ##OBJECT BLOCK##
            if($line =~ /(\w+)\s*=\s*([\w\.\-]+)/) {
               $attr = $1;
               $value = $2;
               #print "rhppc64node attr is $attr\n";
               #print "rhppc64node value is $value\n";
               if($attr eq "Name"){
                   $name = $value;
               } elsif(!defined($name)){
                   print "Please give name for Object\n";
                   close FILE;
                   return 1;
               } else {
                   $rhppc64config{object}{$sub_type}{$name}{$attr}=$value;
               }
            }
        }elsif ($type eq "rhppc64Script") {
            ##SCRIPT_BLOCK##
            if($sub_type eq "Prev") {
                $rhppc64config{script_prev}->[$c] = $line;
                $c = $c + 1;
            }
            elsif ($sub_type eq "rhppc64Post") {
                $rhppc64config{script_post}->[$c] = $line;
                $c = $c + 1;
            }
        } elsif ($type eq "rhppc64Varible") {
            ##NODE_BLOCK##
            if($line =~ /(\w+)\s*=\s*([\w\.\-\/]+)/) {
                $rhppc64config{var}{$1} = $2;
                print "var $1,$2\n";
            }
        }
    }
    if(exists $rhppc64config{object}){
        foreach my $type (keys %{$rhppc64config{object}}){
            foreach my $name (keys %{$rhppc64config{object}{$type}}){
                send_msg("OBJECT:$name,TYPE:$type");
                &runcmd( "  echo [Object_$type]>>default.conf");
                &runcmd( "  echo Name=$name>>default.conf");
                #print "$name,TYPE:$type \n";
                foreach my $attr (keys %{$rhppc64config{object}{$type}{$name}}){
                    send_msg("    $attr = $rhppc64config{object}{$type}{$name}{$attr};");
                    &runcmd( "  echo $attr=$rhppc64config{object}{$type}{$name}{$attr}>>default.conf");
                }
            }
        }
    }
    if(exists $rhppc64config{table}){
        foreach my $type (keys %{$rhppc64config{table}}){
            send_msg("TABLE:$type");
            &runcmd( "  echo [Table_$type]>>default.conf");
            #&runcmd( "  echo key=$type>>default.conf");
            #&runcmd( "  echo [Table_site]>>default.conf");
            #&runcmd( "  echo key=$type>>default.conf");
            foreach my $name (keys %{$rhppc64config{table}{$type}}){
               send_msg("    $rhppc64config{table}{$type}{$name}{__KEY__} = $name");
               &runcmd( "  echo $rhppc64config{table}{$type}{$name}{__KEY__}=$name>>default.conf");
               foreach my $attr (keys %{$rhppc64config{table}{$type}{$name}}){
                   if($attr ne '__KEY__'){
                       send_msg("    $attr = $rhppc64config{table}{$type}{$name}{$attr}");
                       &runcmd( " echo $attr=$rhppc64config{table}{$type}{$name}{$attr}>>default.conf");
                   }
                }
                send_msg("\n");
            }
        }
    }
    if(exists $rhppc64config{script_prev}){
         send_msg("Script_Prev:");
         foreach $cmd (@{$rhppc64config{script_prev}}){
            send_msg("    $cmd");
         }
    }

    if(exists $rhppc64config{script_post}){
        send_msg("Script_Post:");
        foreach $cmd (@{$rhppc64config{script_post}}){
            send_msg("    $cmd");
        }
    }
    if (exists $rhppc64config{var}){
    #my $MN=$rhppc64config{var}{MN};
    #my $MNIP=$rhppc64config{var}{MNIP};
    #&runcmd( "echo $MN $MN.$DOMAIN $MNIP>>/etc/hosts");
    #print "MN is $MN\n";}
        send_msg("Varible:");
        &runcmd( "  echo [System]>>default.conf");
        foreach my $varname (keys %{$rhppc64config{var}}){
            send_msg("    $varname = $rhppc64config{var}{$varname}");
            &runcmd( "  echo $varname=$rhppc64config{var}{$varname}>>default.conf");
            #print "var is $rhppc64config{var}\n";
        }
    }
 
    close FILE;
    return 0;

}
#######################################
# git update
#######################################
sub git_update {
    send_msg("begin to do git update");
    
    my $gitup;
    $gitup="/tmp/gitup";
    
    #Do checkout in git repo
    #$res = system("cd $confkeys{srcdir}");
    #if ($res !=0) {
    #    send_msg("no source code directory,exit");
    #
    #    exit;
    #}
    
    $res = system("git checkout $branch");
    if ($res != 0){
        send_msg("git checkout failed");
        exit 1;
    }
    
    $res = system("git pull >$gitup");
    if ($res != 0){
        send_msg("git pull failed");
        exit 1;
    }
    
    $res = system("grep 'Already up-to-date' $gitup");
    if (($res == 0)&&( $updates_regression == 1)) {
      send_msg("code is already at latest version. exit regresson\n");
      exit 1;
    }
    return 0;
}

#######################################
# copy code
#######################################
sub copy_code {
    my $mn = shift;
    my $codedir = $confhash{srcdir};
    send_msg("src code directory is $confhash{srcdir}"); 
    
    ##will modify to $rhppc64config{var}{MNIP} 
    #my $mn = $management_node;
    #send_msg("mn is $mn");
    
    #install dep for buildlocal
    send_msg("begin to install build required packages on mn $mn");
    $res = system("xdsh $mn yum install -y rpm-build perl-Time-HiRes perl-DBI createrepo");
    
    
    #need to copy /etc/hosts to mn
    send_msg("copy /etc/hosts to mn $mn");
    system("scp /etc/hosts $mn:/etc");
    
    send_msg("begin to copy code to $mn");
    $res = system("scp -r $codedir root\@$mn:/");
    if ($res != 0){
        send_msg("code copy failed $mn");
        exit 1;
    }
    return 0;
}



#######################
# build xcat 
#######################
sub build_xcat {
    my $mn = shift;
    #for temp usage
    send_msg("========= began build xcat on mn ========");
    #will changed /code/xcat-core to $confkeys{srcdir}  
    $res = system("xdsh $mn /xcat-core/buildlocal.sh CURDIR=/xcat-core");
    if ($res != 0){
              send_msg("build failed on mn $mn");
              exit 1;
          }
    
    send_msg("====================build done============================");
    sleep 20;
    return 0;
}  

#######################################
# config mn
#######################################
sub config_mn {
    my $mn = shift;
    send_msg("begin read configuration file for mn");
    mkdir $resultdir unless -d $resultdir;
    $timestamp = `date +"%Y%m%d%H%M%S"`;
    my @osname = &runcmd("uname -a");
        #if ( $osname [0]  =~ /^Linux\s*/  &&  -f "/etc/redhat-release" && $osname [0] =~ /ppc64/){
        #print "ppc64 redhat env\n";
        #$os="rhels6.4";
        #$arch="ppc64";
        #print "os is $os,arch is $arch\n";
        $res = &config_rhppc64();
    if ($res) {
        send_msg("CONFIGURE MN returns error, exit");
        exit 1;
    }
    #}
    send_msg("step 6 : reading mn configuration done=====");
    return 0;
}      
#######################################
# install xcat and init rhppc64 env
#######################################
sub init
{
    my $mn = shift;
    if (exists $rhppc64config{var}){
        #my $MN=$rhppc64config{var}{MN};
        my $MN=$mn;
        my $MNIP=$rhppc64config{var}{MNIP};
        my $CN=$rhppc64config{var}{CN};
        my $CNIP=$rhppc64config{var}{CNIP};
        my $SN=$rhppc64config{var}{SN};
        my $SNIP=$rhppc64config{var}{SNIP};
        my $SNCN=$rhppc64config{var}{SNCN};
        my $SNCNIP=$rhppc64config{var}{SNCNIP};
        my $HMC=$rhppc64config{var}{HMC};
        my $HMCIP=$rhppc64config{var}{HMCIP};
        my $MOUNTIP=$rhppc64config{var}{MOUNTIP};
        my $DOMAIN=$rhppc64config{var}{DOMAIN};
        my $nodedir=$rhppc64config{var}{nodedir};
        system("xdsh $MN mkdir -p /iso/mountpoint");
        print "--prepareing redhat iso file.......\n";
        print "--mount NF .......";
#        system ("scp -r /iso/*.iso $MN:/iso");
#       system("xdsh $MN mount -o loop /iso/RHEL6.4-20130130.0-Server-ppc64-DVD1.iso /iso/mountpoint");
       system ("scp -r /etc/hosts $MN:/etc/hosts");
       system ("scp -r /etc/resolv.conf $MN:/etc/resolv.conf");
       print "[OK]\n";
       print "--get the latest XCAT tarball.......\n";
       system("xdsh $MN  rm -rf /etc/yum.repos.d/*");
#       $res =  system("scp -r $nodedir/xcat-dep $MN:/");
        &repo();
       system("scp -r  rhel6.4.repo $MN:/etc/yum.repos.d/rhel6.4.repo");   
       #system("scp -r default.conf $MN:$rhppc64configfile");
       #system("xdsh $MN perl $nodedir/xcatbuild/xcat-core/mklocalrepo.sh");
       system("xdsh $MN perl /xcat-dep/rh6/ppc64/mklocalrepo.sh");
       print "--install XCAT .......\n";

       system("xdsh $MN yum -y install createrepo");
       system("xdsh $MN createrepo /xcat-core/build");
       system("xdsh $MN cp /xcat-core/build/xCAT-core.repo /etc/yum.repos.d/");
       system("xdsh $MN yum clean metadata");
       system("xdsh $MN rpm --import /iso/mountpoint/RPM-GPG-KEY-redhat-release");
       system("xdsh $MN yum -y install perl-xCAT xCAT-client xCAT-server xCAT");
       print "--install XCATTEST .......\n";
       system("xdsh $MN yum -y install xCAT-test");
       system("scp -r default.conf $MN:$rhppc64configfile");

        print "--install createrepo .......\n";
        #system("xdsh $MN yum -y install createrepo");
        system("xdsh $MN yum -y install screen");
        system("xdsh $MN yum -y install mysql-server mysql mysql-bench mysql-devel mysql-connector-odbc");
        system("xdsh $MN mkdir -p /autotest/result");
        #system("xdsh $MN sysctl -n net.ipv4.ip_forward=1");
        $res =  system("xdsh $MN source /etc/profile.d/xcat.sh");
        #if ($res != 0){
        #send_msg("install xCAT failed on rhpmn");
        #exit 1;
        #}

        send_msg( " rhppc64 env is ready\n");
    }

    return 0;


}
#######################################
# do test
#######################################
sub do_test {
    my $mn = shift;
    # step 7.1 Install xcat and init mn
    send_msg("began to install xCAT and initialize mn");
    $res = &init($mn);
    if ($res != 0){
        exit 1;
    }
    
    send_msg("Begin to do test");
    $res = &do_test1($mn);
    if ($res) {
        send_msg("DO TEST returns error, exit");
        exit 1;
    }
    return 0;

}
#######################################
# run all test
#######################################
sub  do_test1
{
    my $mn = shift;
    #my $MN=$rhppc64config{var}{MN};
    my $MN=$mn;
    my $nodedir=$rhppc64config{var}{nodedir};
    print "copy config file ";
    system("scp -r default.conf $MN:$rhppc64configfile");
    
    print "Start to run diskless installation $MN ...\n";
    send_msg("******************************");
    send_msg("start diskless test");
    send_msg("******************************");
    #if($dsklsinst){
#        system("xdsh $MN perl /opt/xcat/bin/xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf -t Linux_diskless_installation_flat_ppc64");
#        system("xdsh $MN mv /opt/xcat/share/xcat/tools/autotest/result/* /autotest/result/");
    #   exit 1;
    #}
    #if($bundlerun){
    #     print "Start to run the automation test bucket ....\n";
    #    system("xdsh $MN mkdir -p /autotest/result");
    #     system("xdsh $MN /opt/xcat/bin/xcattest -b /opt/xcat/share/xcat/tools/autotest/bundle/bat.bundle");
    #     system("xdsh $MN mv /opt/xcat/share/xcat/tools/autotest/result/* /autotest/result/");
    #    system("xdsh $MN cp /autotest/result/xcattest.log.$timestamp /autotest/result/log/xcattest.log.$timestamp.current");
    #    $output = ("xdsh $MN tail /autotest/result/xcattest.log.$timestamp");
    #    if($output =~ /Total: (\d+) , Failed: (\d+)/){
    #        send_msg{command}{total} = $1;
    #        send_msg{command}{fail} = $2;
    #        send_msg{command}{timestamp} = $timestamp;
    #        if(send_msg{command}{fail} != 0){
    #            send_msg{command}{failcase} = "|                         |        |  Failed cases:"."\n";
    #            $output = (xdsh $MN cat /autotest/result/failedcases.$timestamp | grep END);
    #            while($output =~ /END::(\w+)/g){
    #                send_msg{command}{failcase} = $send_msg{command}{failcase}."|                         |        |    ".$1."\n";
    #                print "$msg{command}{failcase}";
    #            }
    #            print "$send_msg{command}{failcase}";
    #        }
    #    }
    #    }
    #if($stateliteinst){
    #    system("xdsh $MN /opt/xcat/bin/xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf  -t Linux_statelite_installation_flat_ppc64");
    #    system("xdsh $MN mv /opt/xcat/share/xcat/tools/autotest/result/* /autotest/result/");
    #   system("xdsh $MN cp /autotest/result/xcattest.log.$timestamp /autotest/result/log/xcattest.log.$timestamp.current");
    #   $output = system(" xdsh $MN tail /autotest/result/xcattest.log.$timestamp");
    #   if($output =~ /Failed: (\d+)/){
    #       if($1 != 0){
    #         send_msg{linux_statelite_installation_flat}{pass} = 0;
    #         send_msg{linux_statelite_installation_flat}{timestamp} = $timestamp;
    #      } else {
    #          send_msg{linux_statelite_installation_flat}{pass} = 1;
    #      }
    #   }
    #}
    #if($fullinst){
        system("xdsh $MN /opt/xcat/bin/xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf -t Linux_full_installation_flat_ppc64");
        system("xdsh $MN mv /opt/xcat/share/xcat/tools/autotest/result/* /autotest/result/");
    #   system("xdsh $MN cp /autotest/result/xcattest.log.$timestamp /autotest/result/log/xcattest.log.$timestamp.current");
    #   $output = xdsh $MN tail /autotest/result/xcattest.log.$timestamp;
    #   if($output =~ /Failed: (\d+)/){
    #       if($1 != 0){
    #           send_msg{linux_full_installation_flat}{pass} = 0;
    #           send_msg{linux_full_installation_flat}{timestamp} = $timestamp;
    #       } else {
    #           send_msg{linux_full_installation_flat}{pass} = 1;
    #      }
    #   }
    #}
    #if($snfullinst){
    #     system("xdsh $MN /opt/xcat/bin/xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf -t Linux_sn_installation_flat_x86_vm");
    #     system("xdsh $MN mv /opt/xcat/share/xcat/tools/autotest/result/* /autotest/result/");
    #    system("xdsh $MN cp /autotest/result/xcattest.log.$timestamp /autotest/result/log/xcattest.log.$timestamp.current");
    #    $output = xdsh $MN tail /autotest/result/xcattest.log.$timestamp;
    #    if($output =~ /Failed: (\d+)/){
    #       if($1 != 0){
    #           send_msg{linux_sn_installation_flat}{pass} = 0;
    #           send_msg{linux_sn_installation_flat}{timestamp} = $timestamp;
    #       } else {
    #           send_msg{linux_sn_installation_flat}{pass} = 1;
    #       }
    #    }
    #}
    #if($dsklscnsninst){
    #     system("xdsh $MN /opt/xcat/bin/xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf  -t Linux_cn_with_sn_diskless_installation_flat_x86_vm");
    #     system("xdsh $MN mv /opt/xcat/share/xcat/tools/autotest/result/* /autotest/result/");
    #    system("xdsh $MN cp /autotest/result/xcattest.log.$timestamp /autotest/result/log/xcattest.log.$timestamp.current");
    #    if($output =~ /Failed: (\d+)/){
    #       if($1 != 0){
    #          send_msg{linux_cn_with_sn_diskless_installation_flat}{pass} = 0;
    #          send_msg{linux_cn_with_sn_diskless_installation_flat}{timestamp} = $timestamp;
    #       } else {
    #           send_msg{linux_cn_with_sn_diskless_installation_flat}{pass} = 1;
    #       }
    #    }
    #}
    #if($statelitecnsninst){
    #    system("xdsh $MN /opt/xcat/bin/xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf -t Linux_cn_with_sn_statelite_installation_flat_x86_vm");
    #    system(" xdsh $MN mv /opt/xcat/share/xcat/tools/autotest/result/* /autotest/result/");
    #   system("xdsh $MN cp /autotest/result/xcattest.log.$timestamp /autotest/result/log/xcattest.log.$timestamp.current");
    #}
    #if($fullcnsninst){
    #    system("xdsh $MN /opt/xcat/bin/xcattest -f /opt/xcat/share/xcat/tools/autotest/default.conf -t Linux_cn_with_sn_full_installation_flat_x86_vm");
    #    system("xdsh $MN mv /opt/xcat/share/xcat/tools/autotest/result/* /autotest/result/");
    #   system("xdsh $MN cp /autotest/result/xcattest.log.$timestamp /autotest/result/log/xcattest.log.$timestamp.current");
    #}
    #system("mkdir -p $nodedir/result");
    # system("scp -r $MN:/autotest/result /regression/rhppc64");
}

#######################################
# mn_install to install all mn 
#######################################
sub mn_install {
    my $osimage="management_1408a";
    my $mn="the_name_of_mn";

    `nodeset $mn osimage=$osimage`;

    $output = system("rpower $mn off");
    do {
         sleep 5;
         $output = system("rpower $mn state");
    } while ($output =~ /off/);

    $output = system("rpower $mn on");
    do {
         sleep 5;
         $output = system("rpower $mn state");
    } while ($output =~ /on/);

    do { 
         sleep 300;
         $output = system("xdsh $mn date");
    } while ($output =~ /$mn/);
}

#######################################
# runcmd
#######################################
sub runcmd {
    my ($cmd) = @_;
    my $rc = 0;
    $::RUNCMD_RC = 0;
    my $outref = [];
    @$outref = `$cmd 2>&1`;
    if ($?)
    {
        $rc = $? ;
    $::RUNCMD_RC = $rc;
    }
    chomp(@$outref);
    return @$outref;
}

#######################################
# trim
#######################################
sub trim {

   my $str = shift @_;
   if($str){
       #$str =~ s/\#/__wellnumber__/g;
       $str =~ s/^\s+|#.+|\s+$//g;
       #$str =~ s/__wellnumber__/#/g;
   }
   return $str;
}

#####################################
#repo
####################################
sub repo{
my @osname = &runcmd("uname -a");
if ( $osname [0]  =~ /^Linux\s*/  &&  -f "/etc/redhat-release" && $osname [0] =~ /ppc64/){
   send_msg ("ppc64 redhat env\n");
   my $os="rhels6.4";
   my $arch="ppc64";
    send_msg ("os is $os,arch is $arch\n");
    &runcmd( "  echo [rhe-6.4-server]>>rhel6.4.repo");
    &runcmd( "  echo name=RHEL 6.4 SERVER packages>>rhel6.4.repo");
    &runcmd("   echo baseurl=file:///iso/mountpoint/Server/>>rhel6.4.repo");
    &runcmd("   echo enabled=1>>rhel6.4.repo");
    &runcmd("   echo gpgcheck=1>>rhel6.4.repo");
   # system("scp -r  rhel6.4.repo $MN:/etc/yum.repos.d/rhel6.4.repo");
    }

}

#######################################
# send messages
#######################################
sub send_msg {
    my $msg = shift;
    my $data = `date`;
    open (LOGFILE, ">> /home/xcatreg.log");
    print LOGFILE "$timestamp: $msg.\n";
    close LOGFILE;
}
#######################################
# read_conf
#######################################
sub read_conf{
    my $keys;
    if (!open($keys, "<$configfile")) {
        send_msg("Open configuration file error");
    }
    my $line;
    while ($line = <$keys>) {
        if ($line =~ /end/) {
            last;
        }
        if ($line =~ /^\s*log\s*=\s*(\S*)\s*/) {
            $confkeys{log} = $1;
        }
        if ($line =~ /^\s*mailgroup\s*=\s*(\S*)\s*/) {
            $confkeys{mailgroup} = $1;
        }
        if ($line =~ /^\s*srcdir\s*=\s*(\S*)\s*/) {
            $confkeys{srcdir} = $1;
        }
        if ($line =~ /^\s*rhpdir\s*=\s*(\S*)\s*/) {
            $confkeys{rhpdir} = $1;
        }
        if ($line =~ /^\s*slespdir\s*=\s*(\S*)\s*/) {
            $confkeys{slespdir} = $1;
        }
        if ($line =~ /^\s*aixdir\s*=\s*(\S*)\s*/) {
            $confkeys{aixdir} = $1;
        }
        if ($line =~ /^\s*rhxdir\s*=\s*(\S*)\s*/) {
            $confkeys{rhxdir} = $1;
        }
        if ($line =~ /^\s*slesxdir\s*=\s*(\S*)\s*/) {
            $confkeys{slesxdir} = $1;
        }
        if ($line =~ /^\s*ubuntudir\s*=\s*(\S*)\s*/) {
            $confkeys{ubuntudir} = $1;
        }
        if ($line =~ /^\s*xcattestconf\s*=\s*(\S*)\s*/) {
            $confkeys{xcattestconf} = $1;
        }
        if ($line =~ /^\s*rhpmn\s*=\s*(\S*)\s*/) {
            $confkeys{rhpmn} = $1;
            $mns{rhpmn} = $confkeys{rhpmn} if ($confkeys{rhpmn});
        }
        if ($line =~ /^\s*slespmn\s*=\s*(\S*)\s*/) {
            $confkeys{slespmn} = $1;
            $mns{slespmn} = $confkeys{slespmn} if ($confkeys{slespmn});
        }
        if ($line =~ /^\s*aixmn\s*=\s*(\S*)\s*/) {
            $confkeys{aixmn} = $1;
	    $mns{aixmn} = $confkeys{aixmn}if ($confkeys{aixmn});
        }
        if ($line =~ /^\s*rhxmn\s*=\s*(\S*)\s*/) {
            $confkeys{rhxmn} = $1;
	    $mns{rhxmn} = $confkeys{rhxmn}if ($confkeys{rhxmn});
        }
        if ($line =~ /^\s*slesxmn\s*=\s*(\S*)\s*/) {
            $confkeys{slesxmn} = $1;
	    $mns{slesxmn} = $confkeys{slesxmn} if ($confkeys{slesxmn});
        }
        if ($line =~ /^\s*ubuntumn\s*=\s*(\S*)\s*/) {
            $confkeys{ubuntumn} = $1;
       	    $mns{ubuntumn} = $confkeys{ubuntumn} if ($confkeys{ubuntumn});
        }
    }
    send_msg("finish reading global vars");
    return %confkeys;
}

#######################################
# pro_result 
#######################################
sub pro_result{

return 0;

}



###############################################################
# Mainfunction
###############################################################


#######################################
# step 0. Parse input arguments
#######################################
send_msg("\n\n\n== Running step 0...............");
if (
    !GetOptions("h|?"  => \$needhelp,
                "f=s" => \$configfile,
                "b=s" => \$branch,
                "m=s" => \$management_node,
                "V" => \$verbose_mode,
                "U" => \$updates_regression,
                "e=s" => \$send_msg)
 )
{
    &usage;
    send_msg("PARSE ARGUMENTS returns error, exit");
    exit 1;
}
if ($needhelp)
{
    &usage;
    exit 0;
}

# begin to find which mn should be used
if ($management_node) {
   my @all;
   if ($management_node =~ /,/) {
       @all = split /,/, $management_node;
   } else {
       push @all, $management_node;
   }
   foreach my $mn (@all) {
       foreach my $k (keys %confkeys) {
           if ($confkeys{$k} eq $mn) {
               $mns{$k} = $mn;
               send_msg("specify mn $mn");
           }       
        }   
    }
}
#######################################
# step 1. Read configuration files
#######################################
send_msg("\n\n\n== Running read_conf...............");
%confhash = read_conf();
unless (%confhash) {
    send_msg("READ CONF returns error, exit");
    exit 1;
}


#######################################
# step 2. git update  
#######################################
send_msg("\n\n\n== Running git_update...............");
$res = git_update();
if ($res) {
    send_msg("GIT UPDATE returns error, exit");
    exit 1;
}
#######################################
# begin child process, until return log
####################################### 
unless ($ENV{'REGDEBUG'}) { #used for debug.
    pipe CREAD,PWRITE;
    foreach my $m (keys %mns) {
        my $mn = $mns{$m};
        my $pid = fork();
        if ( !defined($pid) ) {
            send_msg("Fork error: $!");
            exit 1;
        } elsif ( $pid == 0 ) { # child process
            close CREAD;    
            #######################################
            # step 3. Install MNs, 
            ####################################### 
            #send_msg("\n\n\n== Running mn_install...............");            
            #$res = mn_install();
            #if ($res) {
            #    send_msg("INSTALL MNS returns error, exit");
            #    exit 1;
            #}
            #######################################
            # step 4. Copy code to MNs
            #######################################
            send_msg("\n\n\n== Running copy_code...............");
#            $res = copy_code($mn);
            if ($res) {
                send_msg("COPY CODE returns error, exit");
                exit 1;
            }    
            #######################################
            # step 5. Build xcat code in MNs
            #######################################
            send_msg("\n\n\n== Running build_xcat...............");
#            $res = build_xcat($mn);
            if ($res) {
                send_msg("BUILD XCAT returns error, exit");
                exit 1;
            }        
            #######################################
            # step 6. Read xCAT MN's configuration
            #######################################
            send_msg("\n\n\n== Running config_mn...............");
            $res = config_mn($mn);
            if ($res) {
                send_msg("CONFIGURE MN returns error, exit");
                exit 1;
            }    
            #######################################
            # step 7. Genrate local configuration file for xcattest
            #         Do test
            #         Write log
            #######################################
            exit 1;
            send_msg("\n\n\n== Running do_test...............");
            $res = do_test($mn);
            if ($res) {
                send_msg("DO TEST returns error, exit");
                exit 1;
            }
            syswrite PWRITE,"$mn succeed\n";
            exit 0;
        } # end of child process
	} # end of foreach	
    close PWRITE;
    my $time = time();
    while (1) {
        while(<CREAD>){
             chomp;
             my $result = $_;
             if ($result =~ /(\w*) succeed/){
                 $results{$1} = 1;
             }
        }         
        last if((keys %results) == keys %mns);
        last if(time() - $time > 28800); #wait 8 hours at most    
    }
} else {
    foreach my $m (keys %mns) {
        my $mn = $mns{$m};
        #######################################
        # step 3. Install MNs, 
        #######################################   
        #send_msg("\n\n\n== Running mn_install...............");                       
        #$res = mn_install();
        #if ($res) {
        #    send_msg("INSTALL MNS returns error, exit");
        #    exit 1;
        #}
        #######################################
        # step 4. Copy code to MNs
        #######################################
        send_msg("\n\n\n== Running copy_code...............");
#        $res = copy_code($mn);
        if ($res) {
            send_msg("COPY CODE returns error, exit");
            exit 1;
        }    
        #######################################
        # step 5. Build xcat code in MNs
        #######################################
        send_msg("\n\n\n== Running build_xcat...............");
#        $res = build_xcat($mn);
        if ($res) {
            send_msg("BUILD XCAT returns error, exit");
            exit 1;
        }        
        #######################################
        # step 6. Read xCAT MN's configuration
        #######################################
        send_msg("\n\n\n== Running config_mn...............");
        $res = config_mn($mn);
        if ($res) {
            send_msg("CONFIGURE MN returns error, exit");
            exit 1;
        }
        #######################################
        # step 7. Genrate local configuration file for xcattest
        #         Do test
        #         Write log
        #######################################
        send_msg("\n\n\n== Running do_test...............");
        $res = do_test($mn);
        if ($res) {
            send_msg("DO TEST returns error, exit");
            exit 1;
        }
    }# end of foreach mn, begin parent process
}# end of debug 
   
#######################################
# step 8. process result 
#######################################
send_msg("\n\n\n== Running pro_result...............");
$res = pro_result();
if ($res) {
    send_msg("PROCESS RESULT returns error, exit");
    exit 1;
}
exit 0;