From 93e42d9908f2c82b056dfcc68976d005be444875 Mon Sep 17 00:00:00 2001
From: hu-weihua <huweihua@cn.ibm.com>
Date: Tue, 20 Sep 2016 02:39:47 -0400
Subject: [PATCH 1/3] Restructure osdeplay code logic

---
 xCAT-probe/lib/perl/LogParse.pm              |  684 +++++++
 xCAT-probe/lib/perl/probe_global_constant.pm |   47 +
 xCAT-probe/lib/perl/probe_utils.pm           |   33 +-
 xCAT-probe/subcmds/osdeploy                  | 1674 +++++++++---------
 4 files changed, 1562 insertions(+), 876 deletions(-)
 create mode 100644 xCAT-probe/lib/perl/LogParse.pm
 create mode 100644 xCAT-probe/lib/perl/probe_global_constant.pm

diff --git a/xCAT-probe/lib/perl/LogParse.pm b/xCAT-probe/lib/perl/LogParse.pm
new file mode 100644
index 000000000..77141fc45
--- /dev/null
+++ b/xCAT-probe/lib/perl/LogParse.pm
@@ -0,0 +1,684 @@
+package LogParse;
+
+# IBM(c) 2016 EPL license http://www.eclipse.org/legal/epl-v10.html
+
+BEGIN { $::XCATROOT = $ENV{'XCATROOT'} ? $ENV{'XCATROOT'} : -d '/opt/xcat' ? '/opt/xcat' : '/usr'; }
+use lib "$::XCATROOT/probe/lib/perl";
+use probe_global_constant;
+use probe_utils;
+
+use strict;
+use Data::Dumper;
+use File::Path;
+use File::Copy;
+use Time::Local;
+use File::Basename;
+
+#------------------------------------------
+
+=head3
+    Description:
+        The constructor of class 'LogParse'
+    Arguments:
+        Public attributes:
+            $self->{verbose}:scalar, Offer verbose information, used for handling feature logic
+        private attributes:
+            $self->{log_open_info}: reference of a hash, used to save the log file operating information. 
+            $self->{current_ref_year}: scalar, the year information of current time. such as 2016. Used for log time parsing
+            $self->{current_ref_time}:  scalar, the epoch format of current time, such as 1472437250. Used for log time parsing
+
+            $self->{debug}: Scalar, offer debug information, used for hanlde code logic, used by developer to debug function running 
+            $self->{debuglogpath}: Scalar, the path of debug log files
+            $self->{debuglogfd}: File descriptor of debug log files
+
+            The structure of "log_open_info" hash is:
+            $self->{log_open_info}->{<logfileshortname>}{openfd}  : The file descriptor of sepecific openning log file       
+            $self->{log_open_info}->{<logfileshortname>}{rotate_file_list} : Array, all rotate file about related log file
+            $self->{log_open_info}->{<logfileshortname>}{openning_file_index} : scalar, the index of openning file in rotate_file_list 
+            $self->{log_open_info}->{<logfileshortname>}{next_read_point} : scalar, the read point of one log file, used by 'seek' function
+            $self->{log_open_info}->{<logfileshortname>}{filetype} : scalar, the type of current log file, $::LOGTYPE_RSYSLOG or $::LOGTYPE_HTTP 
+            $self->{log_open_info}->{<logfileshortname>}{next_start_time} : scalar, the next read time
+
+    Returns:
+       The instance of class
+=cut
+
+#------------------------------------------
+sub new {
+    my @args  = @_;
+    my $self  = {};
+    my $class = shift;
+    $self->{verbose} = shift;
+
+    my %log_open_info;
+    $self->{log_open_info} = \%log_open_info;
+
+    my ($sec, $min, $hour, $day, $mon, $year, $wday, $yday, $isdst) = localtime(time());
+    $self->{current_ref_year} = $year;
+    $self->{current_ref_time} = time();
+
+    $self->{debug} = 0;
+    if ($self->{debug}) {
+        my $logfiledir = "/tmp/xcatprobedebug/";
+        mkpath("$logfiledir") unless (-d "$logfiledir");
+        $self->{debuglogpath} = $logfiledir;
+        $self->{debuglogfd};
+    }
+
+    bless($self, ref($class) || $class);
+    return $self;
+}
+
+#------------------------------------------
+
+=head3
+    Description:
+        Public functuon. Calculate the possible host name of current server in rsyslog files.
+    Arguments:
+        NULL
+    Returns:
+        A array which contains the possible host names of current server
+        Such as ("xxxxxx", "xxxxx.domain",....)
+=cut
+
+#------------------------------------------
+sub obtain_candidate_mn_hostname_in_log {
+    my $self = shift;
+
+    my $svr_hostname_short = `hostname -s`;
+    chomp($svr_hostname_short);
+    my $svr_hostname_domain = `hostname -d`;
+    chomp($svr_hostname_domain);
+
+    my @candidate_svr_hostname_inlog;
+    push(@candidate_svr_hostname_inlog, $svr_hostname_short);
+    push(@candidate_svr_hostname_inlog, "$svr_hostname_short.$svr_hostname_domain");
+    if ($self->{debug}) {
+        my $tmpstr = "";
+        $tmpstr .= "$_ " foreach (@candidate_svr_hostname_inlog);
+        probe_utils->send_msg("stdout", "d", "The candidate MN hostname in log are $tmpstr");
+    }
+    return @candidate_svr_hostname_inlog;
+}
+
+
+
+#------------------------------------------
+
+=head3
+    Description:
+        Public function. Specify the log files scope which will be scaned.
+    Arguments:
+        NULL
+    Returns:
+        A hash which save all candidate log file information
+        $candidate_log{<label>}{file}
+        $candidate_log{<label>}{type}
+        <label> : The short name of log file 
+        file:  The log file name, including full path
+        type: The valid type are $::LOGTYPE_HTTP and $::LOGTYPE_RSYSLOG, refer to 'probe_global_constant' for more information.
+=cut
+
+#------------------------------------------
+sub obtain_log_file_list {
+    my $self = shift;
+    my %candidate_log;
+
+    my @candidate_log_set;
+    push @candidate_log_set, "/var/log/messages" if (-e "/var/log/messages");
+    push @candidate_log_set, "/var/log/xcat/cluster.log" if (-e "/var/log/xcat/cluster.log");
+    push @candidate_log_set, "/var/log/xcat/computes.log" if (-e "/var/log/xcat/computes.log");
+
+    my $filename;
+    foreach my $log (@candidate_log_set) {
+        $filename = basename($log);
+        $filename =~ s/(\w+)\.(\w+)/$1/g;
+        $candidate_log{$filename}{file} = $log;
+        $candidate_log{$filename}{type} = $::LOGTYPE_RSYSLOG;
+    }
+
+    my $httplog;
+    if (-e "/var/log/httpd/access_log") {
+        $httplog = "/var/log/httpd/access_log";
+    } elsif (-e "/var/log/apache2/access_log") {
+        $httplog = "/var/log/apache2/access_log";
+    } elsif (-e "/var/log/apache2/access.log") {
+        $httplog = "/var/log/apache2/access.log";
+    }
+
+    $candidate_log{http}{file} = $httplog;
+    $candidate_log{http}{type} = $::LOGTYPE_HTTP;
+
+    if ($self->{debug}) {
+        my $exist_log_file_list;
+        $exist_log_file_list .= "$candidate_log{$_}{file} " foreach (keys %candidate_log);
+        probe_utils->send_msg("stdout", "d", "The log files to be scaned are $exist_log_file_list");
+    }
+    return \%candidate_log;
+}
+
+#------------------------------------------
+
+=head3
+    Description:
+        Public function. Obtian logs which belong to a specific second from candidate log files.
+    Arguments:
+        the_time_to_load:(input attribute) the specific second.
+        valid_one_second_log_set_ref: (output attribute) the space which save the vaild logs. A array of reference of hash. 
+                                      Refer to fucnction "obtain_log_content" for the hash structure of one log.
+        
+    Returns:
+        1: success
+        0: failed
+=cut
+
+#------------------------------------------
+sub obtain_one_second_logs {
+    my $self                         = shift;
+    my $the_time_to_load             = shift;
+    my $valid_one_second_log_set_ref = shift;
+
+    if ($self->{debug}) {
+        my $logfile = "$self->{debuglogpath}/obtain_one_second_logs.$the_time_to_load";
+        my $fd;
+        open($fd, "> $logfile");
+        $self->{debuglogfd} = $fd;
+    }
+
+    #read log for the first time
+    #need to find vaild log files and load them
+    if (!%{ $self->{log_open_info} }) {
+
+        #obtain all exist log files in current server
+        #such as "/var/log/message, /var/log/xcat/cluster.log......."
+        my $candidate_log_ref = $self->obtain_log_file_list();
+        foreach my $loglabel (keys %$candidate_log_ref) {
+
+            #obtain all rotate log file for one kind of log
+            #such as "/var/log/message, /var/log/message-20160506, /var/log/message-20160507......."
+            my $rotate_file_list_ref = $self->obtain_rotate_log_list($candidate_log_ref->{$loglabel}{file});
+
+            #depending on target log time, find out should read from which rotate log file and which point of this log file
+            my $read_start_point_ref = $self->obtain_valid_log_start_point($the_time_to_load, $rotate_file_list_ref, $candidate_log_ref->{$loglabel}{type});
+
+            $self->{log_open_info}->{$loglabel}{rotate_file_list} = $rotate_file_list_ref;
+            $self->{log_open_info}->{$loglabel}{openning_file_index} = $read_start_point_ref->{log_index};
+            $self->{log_open_info}->{$loglabel}{next_read_point} = $read_start_point_ref->{log_start_point};
+            $self->{log_open_info}->{$loglabel}{filetype} = $candidate_log_ref->{$loglabel}{type};
+        }
+    }
+
+    if ($self->{debug}) {
+        $self->debuglogger("------------Dumper self->{log_open_info}---------------");
+        foreach my $loglabel (keys %{ $self->{log_open_info} }) {
+            $self->debuglogger("[$loglabel]");
+            my $rotate_file_list = join(" ", @{ $self->{log_open_info}->{$loglabel}{rotate_file_list} });
+            $self->debuglogger("rotate_file_list: $rotate_file_list");
+            $self->debuglogger("openning_file_index: $self->{log_open_info}->{$loglabel}{openning_file_index}");
+            $self->debuglogger("next_read_point: $self->{log_open_info}->{$loglabel}{next_read_point}");
+            $self->debuglogger("filetype: $self->{log_open_info}->{$loglabel}{type}");
+            $self->debuglogger("openfd: $self->{log_open_info}->{$loglabel}{openfd}");
+        }
+        $self->debuglogger("--------------------------------------------------------\n");
+    }
+
+    foreach my $loglabel (keys %{ $self->{log_open_info} }) {
+        my $fd;
+        if (!exists($self->{log_open_info}->{$loglabel}{openfd})) {
+            if (!open($fd, "$self->{log_open_info}->{$loglabel}{rotate_file_list}->[$self->{log_open_info}->{$loglabel}{openning_file_index}]")) {
+                print "[error] open $self->{log_open_info}->{$loglabel}{rotate_file_list}->[$self->{log_open_info}->{$loglabel}{openning_file_index}] failed\n";
+                return 1;
+            }
+        } else {
+            $fd = $self->{log_open_info}->{$loglabel}{openfd};
+        }
+
+        if ($fd) {
+            $self->debuglogger("[read $self->{log_open_info}->{$loglabel}{rotate_file_list}->[$self->{log_open_info}->{$loglabel}{openning_file_index}]]");
+            my $next_read_point = $self->{log_open_info}->{$loglabel}{next_read_point};
+            my $next_start_time = 0;
+            my $read_eof        = 1;
+
+            my $i = 0;
+            for ($i = $self->{log_open_info}->{$loglabel}{openning_file_index} ; $i < @{ $self->{log_open_info}->{$loglabel}{rotate_file_list} } ; $i++) {
+                if ($i == $self->{log_open_info}->{$loglabel}{openning_file_index}) {
+                    seek($fd, $next_read_point, 0);
+                } else {
+                    open($fd, $self->{log_open_info}->{$loglabel}{rotate_file_list}->[$i]);
+                    $next_read_point = 0;
+                }
+
+                while (<$fd>) {
+                    chomp;
+                    $self->debuglogger("[$loglabel]read: $_");
+                    my $log_content_ref = $self->obtain_log_content($self->{log_open_info}->{$loglabel}{filetype}, $_);
+
+                    #if read the log whoes time bigger than target time, stop to read
+                    $self->debuglogger("\t$log_content_ref->{time}   $the_time_to_load");
+                    if ($log_content_ref->{time} > $the_time_to_load) {
+                        $next_start_time = $log_content_ref->{time};
+                        $read_eof        = 0;
+                        $self->debuglogger("\tlast");
+                        last;
+                    } else {
+
+                        #if read the log whoes time is equal the target time, save it
+                        push @$valid_one_second_log_set_ref, $log_content_ref;
+
+                        #adjust the next read point to next line
+                        my $len = length($_) + 1;
+                        $next_read_point += $len;
+                        $self->debuglogger("\tnext_read_point + $len");
+                    }
+                }
+
+                if ($read_eof) {
+
+                    #reach the end of current openning file, maybe there are vaild log in next rotate file,
+                    #prepare to search next rotate file
+                    close($fd);
+                    $self->debuglogger("\tread_eof");
+                } else {
+
+                    #have found all vaild log in current openning fd
+                    $self->{log_open_info}->{$loglabel}{openfd} = $fd;
+                    $self->{log_open_info}->{$loglabel}{openning_file_index} = $i;
+                    $self->{log_open_info}->{$loglabel}{next_read_point} = $next_read_point;
+                    $self->{log_open_info}->{$loglabel}{next_start_time} = $next_start_time;
+                    $self->debuglogger("\tfound all vaild log");
+                    last;
+                }
+            }    #end ratate_files loop
+
+            if ($i == @{ $self->{log_open_info}->{$loglabel}{rotate_file_list} } && $read_eof == 1) {
+                $self->{log_open_info}->{$loglabel}{openfd} = 0;
+                $self->{log_open_info}->{$loglabel}{next_start_time} = 9999999999;
+                $self->debuglogger("read out all rotate files");
+            }
+        }    #end $fd
+    }    #end log_open_info loop
+
+    #delete duplicate logs
+    $self->delete_duplicate_log($valid_one_second_log_set_ref);
+
+    if ($self->{debug}) {
+        close($self->{debuglogfd});
+    }
+
+    return 0;
+}
+
+#------------------------------------------
+
+=head3
+    Description:
+        Private function. Depending on target log file name, obtain all related rotated log files.
+        For example, if the target file is '/var/log/messages', the function should return all rotated '/var/log/messages' files.
+        Such as '/var/log/messages-20160101', '/var/log/messages-20160102'.........
+        The all related log files should save into a array in order, the latest log file should be in the tail and the oldest one is in the head of array.
+        For example, the array should look like ("/var/log/messages-20160103", "/var/log/messages-20160104", "/var/log/messages-20160105", "/var/log/messages");
+        Finally, this function return the reference of the array.
+
+    Arguments:
+        file_name: (input attribute) the target log file name, including full path.
+        rotate_file_list_ref: (output attibute) the reference of the array which include all related log files name.
+    Returns:
+        0: success
+        1: failed
+=cut
+
+#------------------------------------------
+sub obtain_rotate_log_list {
+    my $self      = shift;
+    my $file_name = shift;
+
+    my @files     = ();
+    my $file_path = dirname($file_name);
+
+    my @files_all = glob("$file_path/*");
+
+    my @files_grep = grep /$file_name.+\d$/, @files_all;
+
+    @files = sort { -M "$b" <=> -M "$a" } @files_grep;
+
+    push @files, $file_name;
+
+    return \@files;
+}
+
+#------------------------------------------
+
+=head3
+    Description:
+       Private function. Depending on the reference start time. go through all rotate file list, find out where should start to read log.
+       I.e. start from which point of which log file.
+
+    Arguments:
+        start_time: (input attribute) the target start time.
+        rotate_file_list_ref: (input attibute) the reference of the array which include all related log files name.
+        log_type: (input attribute) The type of log file
+
+    Returns:
+        start_point: a hash which include the index of start file and the start point of the file
+                     %start_point{log_index}: the index of start file in the rotate file list
+                     %start_point{log_start_point}: where start to read log from. this value will be used by "seek" function
+=cut
+
+#------------------------------------------
+sub obtain_valid_log_start_point {
+    my $self                 = shift;
+    my $start_time           = shift;
+    my $rotate_file_list_ref = shift;
+    my $log_type             = shift;
+
+    my %start_point;
+    my @files      = @{$rotate_file_list_ref};
+    my $list_index = @files - 1;
+
+    my $fd;
+    my $file;
+    my $filetype;
+    my $line;
+
+    $start_point{log_index}       = 0;
+    $start_point{log_start_point} = 0;
+    while (@files) {
+        $file     = pop(@files);
+        $filetype = `file $file 2>&1`;
+        chomp($filetype);
+        if (!open($fd, "$file")) {
+            print "open $file failed\n";
+            $list_index--;
+            next;
+        }
+
+        if ($line = <$fd>) {
+            my %log_content = %{ $self->obtain_log_content($log_type, $line, 0) };
+            if ($start_time <= $log_content{time}) {
+                $list_index--;
+                next;
+            } else {
+                $start_point{log_index} = $list_index;
+                seek($fd, 0, 2);
+                my $tail           = tell;
+                my $head           = 0;
+                my $lasttail       = $tail;
+                my $file_tail      = $tail;
+                my $tmp_start_time = $start_time - 1;
+                my $historynum     = 0;
+
+                while ($head <= $tail) {
+                    my $middle = int(($tail - $head) / 2) + $head;
+                    seek($fd, $middle, 0);
+                    $line = <$fd>;
+                    $middle += length($line);
+                    last unless ($line = <$fd>);
+                    my %log_content = %{ $self->obtain_log_content($log_type, $line, 0) };
+                    if ($tmp_start_time == $log_content{time}) {
+                        $historynum = $middle;
+                        last;
+                    } elsif ($tmp_start_time < $log_content{time}) {
+                        $tail = $middle;
+                        last if ($tail == $lasttail);
+                        $lasttail = $tail;
+                    } else {
+                        $head = $middle;
+                    }
+                }
+
+                $historynum = $head unless ($historynum);
+
+                while ($historynum < $file_tail) {
+                    seek($fd, $historynum, 0);
+                    $line = <$fd>;
+                    my %log_content = %{ $self->obtain_log_content($log_type, $line, 0) };
+                    if ($start_time <= $log_content{time}) {
+                        last;
+                    } elsif ($start_time > $log_content{time}) {
+                        $historynum += length($line);
+                    }
+                }
+                $start_point{log_start_point} = $historynum;
+                last;
+            }
+        }
+    }
+    return \%start_point;
+
+}
+
+
+#------------------------------------------
+
+=head3
+    Description:
+        Convert one line original log which comes from log file to a hash. 
+
+    Arguments:
+        log_type     :(input attribute) valid log type are $::LOGTYPE_RSYSLOG and $::LOGTYPE_HTTP 
+        original_log :(input attribute) one line log in real log file
+
+    Returns:
+        log_content_ref: (output attribute) the reference of a hash structure
+
+        The hash structure which contain log message is
+        %log_content
+        $log_content{time}   : the timestamp of log, the format is epoch_seconds
+        $log_content{sender} : the sender of log
+        $log_content{label}  : the label of log, such as $::LOGLABEL_DHCPD, $::LOGLABEL_TFTP.... refer to "probe_golbal_constant" for all kinds vaild labels 
+        $log_content{msg}    : the main message of log
+=cut
+
+#------------------------------------------
+sub obtain_log_content {
+    my $self         = shift;
+    my $log_type     = shift;
+    my $original_log = shift;
+
+    my %log_content = ();
+    my $sender;
+    my @split_line = split(/\s+/, $original_log);
+
+    if ($log_type == $::LOGTYPE_RSYSLOG) {
+        if ($split_line[0] =~ /(\d+)-(\d+)-(\d+)T(\d+):(\d+):(\d+)(.+)-(.+)/) {
+            $log_content{time} = $self->convert_to_epoch_seconds($split_line[0]);
+            $log_content{sender} = $split_line[1];
+            if ($split_line[2] =~ /dhcpd/i) {
+                $log_content{label} = $::LOGLABEL_DHCPD;
+            } elsif ($split_line[2] =~ /in.tftpd/i) {
+                $log_content{label} = $::LOGLABEL_TFTP;
+            } elsif ($split_line[2] =~ /^xcat/i) {
+                $log_content{label} = $::LOGLABEL_XCAT;
+            } else {
+                $log_content{label} = $::LOGLABEL_UNDEF;
+            }
+            $log_content{msg} = join(" ", @split_line[ 3 .. @split_line - 1 ]);
+        } else {
+            my $timestamp = join(" ", @split_line[ 0 .. 2 ]);
+            $log_content{time}   = $self->convert_to_epoch_seconds($timestamp);
+            $log_content{sender} = $split_line[3];
+            if ($split_line[4] =~ /dhcpd/i) {
+                $log_content{label} = $::LOGLABEL_DHCPD;
+            } elsif ($split_line[4] =~ /in.tftpd/i) {
+                $log_content{label} = $::LOGLABEL_TFTP;
+            } elsif ($split_line[4] =~ /^xcat/i) {
+                $log_content{label} = $::LOGLABEL_XCAT;
+            } else {
+                $log_content{label} = $::LOGLABEL_UNDEF;
+            }
+            $log_content{msg} = join(" ", @split_line[ 5 .. @split_line - 1 ]);
+        }
+    } elsif ($log_type == $::LOGTYPE_HTTP) {
+        $split_line[3] =~ s/^\[(.+)/$1/g;
+        $log_content{time}   = $self->convert_to_epoch_seconds($split_line[3]);
+        $log_content{sender} = $split_line[0];
+        $log_content{label}  = $::LOGLABEL_HTTP;
+        $log_content{msg}    = join(" ", @split_line[ 5 .. @split_line - 1 ]);
+    }
+    return \%log_content;
+}
+
+#------------------------------------------
+
+=head3
+    Description:
+        Convert input time format to the number of non-leap seconds since whatever time the system considers to be the epoch
+    Arguments:
+        timestr: the time format need to be converted
+    Returns:
+        the number of non-leap seconds since whatever time the system considers to be the epoch
+=cut
+
+#------------------------------------------
+sub convert_to_epoch_seconds {
+    my $self    = shift;
+    my $timestr = shift;
+
+    my $yday;
+    my $mday;
+    my $dday;
+    my $h;
+    my $m;
+    my $s;
+    my $epoch_seconds = -1;
+    my %monthsmap = ("Jan" => 0, "Feb" => 1, "Mar" => 2, "Apr" => 3, "May" => 4, "Jun" => 5, "Jul" => 6, "Aug" => 7, "Sep" => 8, "Oct" => 9, "Nov" => 10, "Dec" => 11);
+
+    # The time format looks like "2016-08-29T03:30:18.259287-04:00"
+    if ($timestr =~ /(\d+)-(\d+)-(\d+)T(\d+):(\d+):(\d+)(.+)-(.+)/) {
+        ($yday, $mday, $dday, $h, $m, $s) = ($1 + 0, $2 - 1, $3 + 0, $4 + 0, $5 + 0, $6 + 0);
+        $epoch_seconds = timelocal($s, $m, $h, $dday, $mday, $yday);
+
+        # The time format looks like "Aug 15 02:43:31"
+    } elsif ($timestr =~ /(\w+)\s+(\d+)\s+(\d+):(\d+):(\d+)/) {
+        ($mday, $dday, $h, $m, $s) = ($1, $2, $3, $4, $5);
+        $yday = $self->{current_ref_year};
+        $epoch_seconds = timelocal($s, $m, $h, $dday, $monthsmap{$mday}, $yday);
+        if ($epoch_seconds > $self->{current_ref_time}) {
+            $yday -= 1;
+            $epoch_seconds = timelocal($s, $m, $h, $dday, $monthsmap{$mday}, $yday);
+        }
+
+        # The time format looks like "15/Aug/2016:01:10:24"
+    } elsif ($timestr =~ /(\d+)\/(\w+)\/(\d+):(\d+):(\d+):(\d+)/) {
+        $epoch_seconds = timelocal($6, $5, $4, $1, $monthsmap{$2}, ($3 - 1900));
+    }
+    return $epoch_seconds;
+}
+
+
+#------------------------------------------
+
+=head3
+    Description:
+        Private function. After obtain one second logs. deleted these duplicate logs.
+    Arguments:
+        vaild_log_set_ref:(input/output attribute) The reference of the log set which will be scaned to delete duplicate log
+    Returns:
+        NULL
+=cut
+
+#------------------------------------------
+sub delete_duplicate_log {
+    my $self              = shift;
+    my $vaild_log_set_ref = shift;
+
+    my $log_index = @$vaild_log_set_ref - 1;
+    my @delete_index;
+    for (my $i = $log_index ; $i > 0 ; $i--) {
+        for (my $j = 0 ; $j < $i ; $j++) {
+            if (($vaild_log_set_ref->[$i]->{time} == $vaild_log_set_ref->[$j]->{time}) &&
+                ($vaild_log_set_ref->[$i]->{label} == $vaild_log_set_ref->[$j]->{label}) &&
+                ($vaild_log_set_ref->[$i]->{sender} eq $vaild_log_set_ref->[$j]->{sender}) &&
+                ($vaild_log_set_ref->[$i]->{msg} eq $vaild_log_set_ref->[$j]->{msg})) {
+                if (!grep { $_ eq $i } @delete_index) {
+                    push @delete_index, $i;
+                    last;
+                }
+            }
+        }
+    }
+
+    foreach (@delete_index) {
+        my @new_list = ();
+        splice(@$vaild_log_set_ref, $_, 1, @new_list);
+    }
+
+    if ($self->{debug}) {
+        $self->debuglogger("------------After delete duplicate logs---------------");
+        foreach my $log_ref (@{$vaild_log_set_ref}) {
+            $self->debuglogger("$log_ref->{msg}");
+        }
+    }
+}
+
+#------------------------------------------
+
+=head3
+    Description:
+        Public function. Obtain the next second when should get logs from
+    Arguments:
+        NULL
+    Returns:
+        The second when should get logs from
+=cut
+
+#------------------------------------------
+sub obtain_next_second {
+    my $self = shift;
+
+    my $next_start_time = 9999999999;
+    foreach my $loglabel (keys %{ $self->{log_open_info} }) {
+        if ($self->{log_open_info}->{$loglabel}{next_start_time} < $next_start_time) {
+            $next_start_time = $self->{log_open_info}->{$loglabel}{next_start_time};
+        }
+    }
+    return $next_start_time;
+}
+
+#------------------------------------------
+
+=head3
+    Description:
+        The destructor of class 'LogParse'
+    Arguments:
+        NULL
+    Returns:
+        NULL
+=cut
+
+#------------------------------------------
+sub destory {
+    my $self = shift;
+    foreach my $loglabel (keys %{ $self->{log_open_info} }) {
+        if ($self->{log_open_info}->{$loglabel}{openfd}) {
+            close($self->{log_open_info}->{$loglabel}{openfd});
+        }
+    }
+
+    if ($self->{debuglogfd}) {
+        close($self->{debuglogfd});
+    }
+}
+
+#------------------------------------------
+
+=head3
+    Description:
+        Private function. Used for log debug information to disk. 
+    Arguments:
+        $msg : The massage which will be logged into log file. 
+    Returns:
+        NULL
+=cut
+
+#------------------------------------------
+sub debuglogger {
+    my $self = shift;
+    my $msg  = shift;
+    if ($self->{debug}) {
+        my $fd = $self->{debuglogfd};
+        print $fd "$msg\n";
+    }
+}
+1;
+
diff --git a/xCAT-probe/lib/perl/probe_global_constant.pm b/xCAT-probe/lib/perl/probe_global_constant.pm
new file mode 100644
index 000000000..52b16e014
--- /dev/null
+++ b/xCAT-probe/lib/perl/probe_global_constant.pm
@@ -0,0 +1,47 @@
+#!/usr/bin/env perl
+# IBM(c) 2007 EPL license http://www.eclipse.org/legal/epl-v10.html
+package probe_golbal_constant;
+
+#The type of log
+$::LOGTYPE_RSYSLOG = 0;    #rsyslog
+$::LOGTYPE_HTTP    = 1;    #apache log
+
+#The lable of specific line log
+$::LOGLABEL_DHCPD = 0;
+$::LOGLABEL_TFTP  = 1;
+$::LOGLABEL_HTTP  = 2;
+$::LOGLABEL_XCAT  = 3;
+$::LOGLABEL_UNDEF = 4;
+
+#The important stage of provision process
+$::STATE_POWER_ON       = 1;
+$::STATE_POWERINGON     = 2;
+$::STATE_DHCP           = 3;
+$::STATE_BOOTLODER      = 4;
+$::STATE_KERNEL         = 5;
+$::STATE_INITRD         = 6;
+$::STATE_KICKSTART      = 7;
+$::STATE_INSTALLING     = 8;
+$::STATE_INSTALLRPM     = 9;
+$::STATE_POSTSCRIPT     = 10;
+$::STATE_BOOTING        = 11;
+$::STATE_POSTBOOTSCRIPT = 12;
+$::STATE_COMPLETED      = 13;
+
+#The description of every important stage of provision process
+%::STATE_DESC = (
+    $::STATE_POWER_ON       => "rpower_to_install",
+    $::STATE_POWERINGON     => "powering_on",
+    $::STATE_DHCP           => "got_ip_from_dhcp",
+    $::STATE_BOOTLODER      => "download_bootloder",
+    $::STATE_KERNEL         => "download_kernel",
+    $::STATE_INITRD         => "download_initrd",
+    $::STATE_KICKSTART      => "download_kickstart",
+    $::STATE_INSTALLING     => "installing",
+    $::STATE_INSTALLRPM     => "start_to_install_os_package",
+    $::STATE_POSTSCRIPT     => "running_postscripts",
+    $::STATE_BOOTING        => "booting",
+    $::STATE_POSTBOOTSCRIPT => "running_postbootscripts",
+    $::STATE_COMPLETED      => "complete",
+);
+1;
diff --git a/xCAT-probe/lib/perl/probe_utils.pm b/xCAT-probe/lib/perl/probe_utils.pm
index 00a3a59f7..d3c211de7 100644
--- a/xCAT-probe/lib/perl/probe_utils.pm
+++ b/xCAT-probe/lib/perl/probe_utils.pm
@@ -16,13 +16,19 @@ use Socket;
         Format is [<flag>] : <message>
         The valid <flag> are debug, warning, failed, info and ok
     Arguments:
-        output: where should output the message 
-        num:  the number of <flag>
+        output: where should the message be output 
+              The vaild value are:
+              stdout : print message to STDOUT
+              a file name: print message to the file which "file name" specify 
+        tag:  the type of message, the valid value are:
               d: debug
               w: warning
               f: failed
               o: ok
               i: info
+
+              if tag is NULL, that means output message without tag
+             
         msg:  the information need to output
      Returns:
         1 : Failed 
@@ -35,7 +41,7 @@ sub send_msg {
     $output = shift if (($output) && ($output =~ /probe_utils/));
     my $tag = shift;
     my $msg = shift;
-    my $flag;
+    my $flag="";
 
     if ($tag eq "d") {
         $flag = "[debug]  :";
@@ -516,4 +522,25 @@ sub convert_to_epoch_seconds {
     }
     return $epoch_seconds;
 }
+
+
+#------------------------------------------
+
+=head3
+    Description:
+        Convert node range in Regular Expression to a node name array
+    Arguments:
+        noderange : the range of node
+    Returns:
+        A array which contain each node name
+=cut
+
+#------------------------------------------
+sub parse_node_range {
+    my $noderange = shift;
+    $noderange= shift if (($noderange) && ($noderange =~ /probe_utils/));
+    my @nodeslist = `nodels $noderange`;
+    chomp($_) foreach(@nodeslist);
+    return @nodeslist;
+}
 1;
diff --git a/xCAT-probe/subcmds/osdeploy b/xCAT-probe/subcmds/osdeploy
index 969941453..2420b46fa 100755
--- a/xCAT-probe/subcmds/osdeploy
+++ b/xCAT-probe/subcmds/osdeploy
@@ -4,8 +4,9 @@
 BEGIN { $::XCATROOT = $ENV{'XCATROOT'} ? $ENV{'XCATROOT'} : -d '/opt/xcat' ? '/opt/xcat' : '/usr'; }
 
 use lib "$::XCATROOT/probe/lib/perl";
-
 use probe_utils;
+use LogParse;
+use probe_global_constant;
 use xCAT::NetworkUtils;
 use File::Basename;
 use IO::Select;
@@ -13,160 +14,226 @@ use Time::Local;
 use Data::Dumper;
 use Getopt::Long qw(:config no_ignore_case);
 
-my $program_name = basename("$0");
-my $help;
-my $test;
-my $maxwaittime = 60;         #unit is minute
-my $output      = "stdout";
-my $verbose     = 0;
-my $rst         = 0;
-my $noderange;
-my $terminal = 0;
-my $installnic;
-my $monitor = 0;
-my $replaylog;                ##used by feature replay deploymen log
+#---------------------------------------------
+#             Global attributes
+#---------------------------------------------
 
-my %rawdata;
-
-#-%rawdata structure-------
-# $rawdata{nodename}{"history"}       #array, the log history of current node
-# $rawdata{nodename}{"state"}         #the latest status of current node, used for State Machine
-# $rawdata{nodename}{"statehistory"}  #array, the history status of current node, used for State Machine
-#--------------------------
-
-my %macmap;
-
-#-%macmap structure-------
+#-------MAC to IP/node map list-------
 # $macmap{mac_addr}{"ip"}="x.x.x.x"
 # $macmap{mac_addr}{"node"}="nodename"
-#-------------------------
+#-------------------------------------
+my %macmap;
 
+#------IP to node map list-------
+# $ipnodemap{ip_addr}="nodename"
+#--------------------------------
 my %ipnodemap;
 
-#-%ipnodemap structure-------
-# $ipnodemap{ip_addr}="nodename"
-#---------------------------
+#The NIC of provision network
+my $installnic;
 
-my %monitor_nodes;
+#Used by customer or developer, to obtain more output information
+my $verbose = 0;
 
-#- %monitor_nodes structure-------
-# $monitor_nodes{nodename}{"status"}  #useless now
-# $monitor_nodes{nodename}{"rst"}
-#---------------------------------
-
-# provision state machine
-my %state_set = (
-    "unknown"                  => 0,
-    "server_reboot"             => 1,
-    "loaded_kernel_and_initrd" => 2,
-    "kernel_and_initrd_got_ip" => 3,
-    "install_os_packages"      => 4,
-    "run_postscript"           => 5,
-    "run_postbootscript"       => 6,
-    "done"                     => 7,
-);
-
-my %state_set_reverse = (
-    "0" => "unknown",
-    "1" => "server_reboot",
-    "2" => "loaded_kernel_and_initrd",
-    "3" => "kernel_and_initrd_got_ip",
-    "4" => "install_os_packages",
-    "5" => "run_postscript",
-    "6" => "run_postbootscript",
-    "7" => "done",
-);
-
-my %valid_process;
-$valid_process{1}{process} = [ $state_set{unknown}, $state_set{done} ];
-$valid_process{1}{type} = "reboot";
-
-$valid_process{2}{process} = [ $state_set{unknown}, $state_set{server_reboot}, $state_set{done} ];
-$valid_process{2}{type} = "reboot";
-
-$valid_process{3}{process} = [ $state_set{unknown}, $state_set{server_reboot}, $state_set{run_postbootscript}, $state_set{done} ];
-$valid_process{3}{type} = "reboot";
-
-$valid_process{4}{process} = [ $state_set{unknown}, $state_set{server_reboot}, $state_set{loaded_kernel_and_initrd}, $state_set{kernel_and_initrd_got_ip}, $state_set{install_os_packages}, $state_set{run_postscript}, $state_set{server_reboot}, $state_set{run_postbootscript}, $state_set{done} ];
-$valid_process{4}{type} = "deploy";
-
-$valid_process{5}{process} = [ $state_set{unknown}, $state_set{loaded_kernel_and_initrd}, $state_set{kernel_and_initrd_got_ip}, $state_set{install_os_packages}, $state_set{run_postscript}, $state_set{server_reboot}, $state_set{run_postbootscript}, $state_set{done} ];
-$valid_process{5}{type} = "deploy";
-
-sub reset_state {
-    my $state_ref  = shift;
-    my $condition  = shift;
-    my $reset_flag = 1;
-
-    if ($$state_ref == $state_set{unknown} && $condition eq "dhcp") {
-        $$state_ref = $state_set{server_reboot};
-    } elsif ($$state_ref == $state_set{unknown} && $condition eq "booted") {
-        $$state_ref = $state_set{done};
-    } elsif (($$state_ref == $state_set{unknown} && $condition eq "tftp") || ($$state_ref == $state_set{unknown} && $condition eq "http")) {
-        $$state_ref = $state_set{loaded_kernel_and_initrd};
-    } elsif ($$state_ref == $state_set{server_reboot} && $condition eq "booted") {
-        $$state_ref = $state_set{done};
-    } elsif ($$state_ref == $state_set{server_reboot} && $condition eq "postscript") {
-        $$state_ref = $state_set{run_postbootscript};
-    } elsif (($$state_ref == $state_set{server_reboot} && $condition eq "tftp") || ($$state_ref == $state_set{server_reboot} && $condition eq "http")) {
-        $$state_ref = $state_set{loaded_kernel_and_initrd};
-    } elsif ($$state_ref == $state_set{loaded_kernel_and_initrd} && $condition eq "dhcp") {
-        $$state_ref = $state_set{kernel_and_initrd_got_ip};
-    } elsif ($$state_ref == $state_set{kernel_and_initrd_got_ip} && $condition eq "http") {
-        $$state_ref = $state_set{install_os_packages};
-    } elsif ($$state_ref == $state_set{install_os_packages} && $condition eq "postscript") {
-        $$state_ref = $state_set{run_postscript};
-    } elsif ($$state_ref == $state_set{run_postscript} && $condition eq "dhcp") {
-        $$state_ref = $state_set{server_reboot};
-    } elsif ($$state_ref == $state_set{run_postbootscript} && $condition eq "booted") {
-        $$state_ref = $state_set{done};
-    } elsif ($$state_ref == $state_set{done} && $condition eq "dhcp") {
-        $$state_ref = $state_set{server_reboot};
-    } elsif ($$state_ref == $state_set{done} && $condition eq "poweron") {
-        $$state_ref = $state_set{server_reboot};
-    } else {
-        $reset_flag = 0;
-    }
-
-    return $reset_flag;
-}
-
-
-my @candidate_svr_hostname_inlog;
-my $svr_hostname_short = `hostname -s`;
-chomp($svr_hostname_short);
-my $svr_hostname_domain = `hostname -d`;
-chomp($svr_hostname_domain);
-push(@candidate_svr_hostname_inlog, $svr_hostname_short);
-push(@candidate_svr_hostname_inlog, "$svr_hostname_short.$svr_hostname_domain");
+#if no specific instruction, do monitor by default
+my $monitor = 1;
 
+#used by developer, to debug the detail information about function running
+my $debug = 0;
 
+#---------------------------------------------
+#            Command Usage
+#---------------------------------------------
+my $program_name = basename("$0");
 $::USAGE = "Usage:
     $program_name -h
-    $program_name -T
-    $program_name -n <node_range> [-V]
-    $program_name -n <node_range> -r <xxhxxm> [-V]
+    $program_name -n <node_range> [-t <max_waiting_time]> [-V]
+    $program_name -n <node_range> -r <roll_back_duration> [-V]
 
 Description:
-    Probe for OS provision process. Realtime monitor or replay history of OS provision process.
-    If realtime monitor, run this before 'rpower' node.
-    Currently, hierarchial structure is not supported.
+    Probe operating system provision process. Support two modes, they are 'Realtime monitor' and 'Replay history'.
+    Realtime monitor: Used during doing provision, trigger 'Realtime monitor' before triggering target node reboot to do provision. This tool will monitor the provision state of node. If without specific instruction, do 'Realtime monitor' by default. 
+    Replay history: Used after finish provision, to probe the provision happened some while ago.
+
+    [NOTE] Currently, hierarchial structure is not supported.
 
 Options:
     -h : Get usage information of $program_name
-    -T : Verify if $program_name can work, reserved option for probe framework
-    -V : Output more information for debug
-    -n : The range of nodes for monitor or replay log.
-    -t : The maximum time in minutes to wait when doing monitor, default is 60.
-    -r : Replay history log for probe provisioniong. Input a start time when probe should begin. 
-         Supported time formats are xxhxxm, xxh, or xxm. If units not specified, hour will be used by default.
+    -V : Output more information 
+    -n : The range of nodes to be monitored or replayed.
+    -t : The maximum time to wait when doing monitor, unit is minute. default is 60.
+    -r : Trigger 'Replay history' mode. follow the duration of rolling back. Unit are 'h' (hour) or 'm' (minute)
+         Supported formats look like 3h30m (3 hours and 30 minutes ago), 2h (2 hours ago), 40m (40 minutes ago) and 3 (3 hours ago).
+         If unit is not specified, hour will be used by default.
 ";
 
+
+#----------------------------------------------
+#               Main process
+#----------------------------------------------
+
+# parse command line arguments
+my $help        = 0;
+my $test        = 0;
+my $maxwaittime = 60;    #unit is minute, the max wait time of monitor
+my $rollforward_time_of_replay;    #used by feature replay provision log
+my $noderange;
+if (
+    !GetOptions("--help|h|?" => \$help,
+        "T"   => \$test,
+        "V"   => \$verbose,
+        "t=s" => \$maxwaittime,
+        "r=s" => \$rollforward_time_of_replay,
+        "n=s" => \$noderange))
+{
+    probe_utils->send_msg("stdout", "f", "Invalid parameter for $program_name");
+    probe_utils->send_msg("stdout", "",  "$::USAGE");
+    exit 1;
+}
+
+if ($help) {
+    probe_utils->send_msg("stdout", "", "$::USAGE");
+    exit 0;
+}
+
+if ($test) {
+    probe_utils->send_msg("stdout", "o", "Probe for OS provision process, realtime monitor of OS provision process.");
+    exit 0;
+}
+
+unless ($noderange) {
+    probe_utils->send_msg("stdout", "f", "A noderange is required");
+    probe_utils->send_msg("stdout", "",  "$::USAGE");
+    exit 1;
+}
+
+if ($rollforward_time_of_replay) {
+    if (($rollforward_time_of_replay !~ /(\d+)h(\d+)m/i) && ($rollforward_time_of_replay !~ /^(\d+)h*$/i) && ($rollforward_time_of_replay !~ /^(\d+)m$/i)) {
+        probe_utils->send_msg("stdout", "f", "Unsupported time format for option '-r'");
+        probe_utils->send_msg("stdout", "", "$::USAGE");
+        exit 1;
+    }
+}
+
+my $rst = do_pre_check($noderange);
+if ($debug) {
+    print "Dumper macmap--------\n";
+    print Dumper \%macmap;
+    print "Dumper ipnodemap--------\n";
+    print Dumper \%ipnodemap;
+    print "installnic = $installnic\n";
+}
+
+#if failed to pass pre-check, exit directly
+exit $rst if ($rst);
+
+
+if ($rollforward_time_of_replay) {
+    $monitor = 0;
+
+    my $start_time_of_replay = time();
+    my $end_time_of_replay   = $start_time_of_replay;
+    if ($rollforward_time_of_replay =~ /(\d+)h(\d+)m/i) {
+        $start_time_of_replay -= ($1 * 3600 + $2 * 60)
+    } elsif ($rollforward_time_of_replay =~ /^(\d+)h*$/i) {
+        $start_time_of_replay -= $1 * 3600;
+    } elsif ($rollforward_time_of_replay =~ /^(\d+)m$/) {
+        $start_time_of_replay -= $1 * 60;
+    }
+
+    $rst = do_replay($noderange, $start_time_of_replay, $end_time_of_replay);
+    exit $rst;
+}
+
+#if no specific instruction, do monitor by default
+$rst = do_monitor($noderange, $maxwaittime);
+
+exit $rst;
+
 #------------------------------------------
 
 =head3
     Description:
-        Check if all nodes are valid
+        Pre-check some static configuration. such as privision network interface, the definition of node
+    Arguments:
+        noderange:  The range of node
+    Returns:
+        0: pass
+        1: failed
+=cut
+
+#------------------------------------------
+sub do_pre_check {
+    my $noderange = shift;
+
+    my $rst = 0;    #the return value of current function
+
+    my @error = ();
+    my $sub_func_rst = obtain_install_nic(\$installnic, \@error);
+    if ($sub_func_rst) {
+        probe_utils->send_msg("stdout", "f", "Obtain install NIC in current server failed");
+        probe_utils->send_msg("stdout", "d", "$_") foreach (@error);
+    } else {
+        probe_utils->send_msg("stdout", "i", "The install NIC in current server is $installnic");
+    }
+    $rst |= $sub_func_rst;
+
+    @error = ();
+    $sub_func_rst = check_noderange($noderange, \@error);
+    if ($sub_func_rst) {
+        probe_utils->send_msg("stdout", "f", "There is something wrong in node definition");
+        probe_utils->send_msg("stdout", "d", "$_") foreach (@error);
+    } else {
+        probe_utils->send_msg("stdout", "o", "All nodes which will be deployed are valid");
+    }
+    $rst |= $sub_func_rst;
+
+    return $rst;
+}
+
+#------------------------------------------
+
+=head3
+    Description:
+        Obtain the network interface of provision
+    Arguments: 
+        installnic_ref: (output attribute) the reference of scalar which save the provision network interface.
+        return_error_ref: (output attribute) the reference of array which save the error information.
+    Returns:
+        0: success 
+        1: failed
+=cut
+
+#------------------------------------------
+sub obtain_install_nic {
+    my $installnic_ref   = shift;
+    my $return_error_ref = shift;
+
+    my $master_ip_in_site = `lsdef -t site -i master -c |awk -F"=" '{print \$2}'`;
+    chomp($master_ip_in_site);
+    unless ($master_ip_in_site) {
+        push @$return_error_ref, "'master' isn't configured in 'site' table";
+        return 1;
+    }
+
+    $$installnic_ref = `ip addr |grep -B2 $master_ip_in_site|awk -F" " '/mtu/{gsub(/:/,"",\$2); print \$2}'`;
+    chomp($$installnic_ref);
+    if (!$$installnic_ref) {
+        push @$return_error_ref, "The value of 'master' in 'site' table is $master_ip_in_site, can't get corresponding network interface in current server";
+        return 1;
+    }
+    return 0;
+}
+
+
+#------------------------------------------
+
+=head3
+    Description:
+        Check if the definition of all nodes are valid
+        If all nodes definition are valid, initialize below global attributes which will be used as a map in "Monitor" or "Replay" 
+        %macmap, %ipnodemap
+                
     Arguments:
         node_range: node range
     Returns:
@@ -177,71 +244,62 @@ Options:
 #------------------------------------------
 sub check_noderange {
     my $node_range = shift;
-    my @cmdoutput  = `lsdef $node_range -i ip,mac -c 2>&1`;
-    my $rst        = 0;
+    my $error_ref  = shift;
+
+    my @cmdoutput = `lsdef $node_range -i ip,mac -c 2>&1`;
+    my $rst       = 0;
     my %nodecheckrst;
 
     foreach (@cmdoutput) {
         chomp($_);
         $_ =~ s/^\s+|\s+$//g;
         if ($_ =~ /^Error: Could not find an object named '(\w+)' .+/i) {
-            $nodecheckrst{$1}{"error"} = "Could not find node definition";
+            $rst = 1;
+            push @$error_ref, "Node $1 without node definition";
         } elsif ($_ =~ /(\S+):\s+mac=(.*)/i) {
             my $node = $1;
             my $mac  = $2;
             if ($mac) {
                 $nodecheckrst{$node}{"mac"} = $mac;
             } else {
-                $nodecheckrst{$node}{"error"} = "Node $1 doesn't have MAC address";
+                $rst = 1;
+                push @$error_ref, "Node $node without 'mac' attribute definition";
             }
         } elsif ($_ =~ /(\S+):\s+ip=(.*)/i) {
             my $node = $1;
             my $ip   = $2;
             if ($ip) {
                 $nodecheckrst{$node}{"ip"} = $ip;
-            }
-        }
-    }
-
-    foreach my $node (keys %nodecheckrst) {
-        if (exists($nodecheckrst{$node}{error})) {
-            probe_utils->send_msg("$output", "d", "$node : $nodecheckrst{$node}{error}");
-            $rst = 1;
-            next;
-        }
-        my $noerror=1;
-        my @macs = split(/\|/, $nodecheckrst{$node}{"mac"});
-        foreach my $mac (@macs) {
-
-            #[NOTE] don't support 2 adapters in the same network now. TODO
-
-            if ($mac =~ /\!\*NOIP\*/) {
-                $mac =~ s/\!\*NOIP\*//g;
-                $macmap{$mac}{"ip"}   = "NOIP";
-                $macmap{$mac}{"node"} = $node;
             } else {
-                $macmap{$mac}{"node"} = $node;
-                if($nodecheckrst{$node}{"ip"}){
-                    $macmap{$mac}{"ip"} = $nodecheckrst{$node}{"ip"};
-                }else{
-                    my $nodeip = xCAT::NetworkUtils->getipaddr($node);
-                    if($nodeip){
-                        $macmap{$mac}{"ip"} = $nodeip;
-                    }else{
-                        $noerror = 0;
-                        $rst = 1;
-                        probe_utils->send_msg("$output", "f", "$node : can't be resolved to an IP address"); 
-                    }
+                my $nodeip = xCAT::NetworkUtils->getipaddr($node);
+                if ($nodeip) {
+                    $nodecheckrst{$node}{"ip"} = $nodeip;
+                } else {
+                    $rst = 1;
+                    push @$error_ref, "Node $node without ip definition";
                 }
             }
         }
-
-        $monitor_nodes{$node}{"rst"} = 0 if($noerror);
     }
 
-    unless (%monitor_nodes) {
-        probe_utils->send_msg("$output", "d", "There is no valid node to handle");
-        $rst = 1;
+    if (!$rst) {
+        foreach my $node (keys %nodecheckrst) {
+            my @macs = split(/\|/, $nodecheckrst{$node}{"mac"});
+            foreach my $mac (@macs) {
+
+                #[NOTE] don't support 2 adapters in the same network now. TODO
+
+                if ($mac =~ /\!\*NOIP\*/) {
+                    $mac =~ s/\!\*NOIP\*//g;
+                    $macmap{$mac}{"ip"}   = "NOIP";
+                    $macmap{$mac}{"node"} = $node;
+                } else {
+                    $macmap{$mac}{"node"} = $node;
+                    $macmap{$mac}{"ip"}   = $nodecheckrst{$node}{"ip"};
+                    $ipnodemap{ $nodecheckrst{$node}{"ip"} } = $node;
+                }
+            }
+        }
     }
 
     return $rst;
@@ -251,249 +309,183 @@ sub check_noderange {
 
 =head3
     Description:
-       Handle one line log come from dhcp log file
+        Initailize a very important hash "%node_state" which will save the state information of every node 
     Arguments:
-       msg: one line http log
-       installnic: target network interfaces
+        noderange: (input attribute) The range of node
+        node_state_ref: (output attribute) the reference of hash "%node_state" 
+            The strucuture of  hash "%node_state" are :
+                $node_state{<node>}{statehistory}  Array.  save the latest loop provision states 
+                $node_state{<node>}{done}          Scalar.  the flag of if the node have finished the provision                
+                $node_state{<node>}{allstatehistory}   Array. save the history states before the latest loop provision. Used in debug mode.  
+                $node_state{<node>}{log}           Array. save all related logs of node. Used in debug mode. 
     Returns:
-        0 : pass
-        1 : failed
+        NULL
 =cut
 
 #------------------------------------------
-sub handle_dhcp_msg {
-    my $msg = shift;
+sub init_node_state {
+    my $noderange      = shift;
+    my $node_state_ref = shift;
 
-    if ($msg =~ /.+DHCPDISCOVER\s+from\s+(.+)\s+via\s+([^:]+)(.*)/i) {
-        my $mac = $1;
-        my $nic = $2;
+    my @nodes = probe_utils->parse_node_range($noderange);
+    foreach my $node (@nodes) {
+        @{ $node_state_ref->{$node}{statehistory} } = ();
+        $node_state_ref->{$node}{done} = 0;
+    }
+}
 
-        if (exists $macmap{$mac}) {
-            my $node   = $macmap{$mac}{"node"};
-            my $record = "Receive DHCPDISCOVER via $nic";
-            probe_utils->send_msg("$output", "d", "[$node] $record") if ($monitor);
-            push(@{ $rawdata{$node}{"history"} }, $record);
-        }
-    } elsif ($msg =~ /.+DHCPOFFER\s+on\s+(.+)\s+to\s+(.+)\s+via\s+(.+)/i) {
-        my $ip  = $1;
-        my $mac = $2;
-        my $nic = $3;
 
-        if (exists $macmap{$mac}) {
-            my $node   = $macmap{$mac}{"node"};
-            my $record = "Send DHCPOFFER on $ip back to $mac via $nic";
-            probe_utils->send_msg("$output", "d", "[$node] $record") if ($monitor);
-            push(@{ $rawdata{$node}{"history"} }, $record);
-        }
+#------------------------------------------
 
-    } elsif ($msg !~ /unknown lease/ && $msg !~ /ignored/ && $msg =~ /.+DHCPREQUEST\s+for\s+(.+)\s\((.+)\)\s+from\s+(.+)\s+via\s+(.+)/) {
-        my $ip     = $1;
-        my $server = $2;
-        my $mac    = $3;
-        my $nic    = $4;
+=head3
+    Description:
+        Implement the replay feature.
+    Arguments:
+        noderange : the range of node
+        start_time_of_replay: the start time point of scaning log
+        end_time_of_replay: the end time point of scaning log
+   
+    Returns:
+        0: success 
+        1: failed
+=cut
 
-        if (exists $macmap{$mac}) {
-            my $node   = $macmap{$mac}{"node"};
-            my $record = "Receive DHCPREQUEST from $mac for $ip via $nic";
-            probe_utils->send_msg("$output", "d", "[$node] $record") if ($monitor);
-            push(@{ $rawdata{$node}{"history"} }, $record);
-        }
-    } elsif ($msg =~ /.+DHCPACK\s+on\s+(.+)\s+to\s+(.+)\s+via\s+(.+)/) {
-        my $ip  = $1;
-        my $mac = $2;
-        my $nic = $3;
+#------------------------------------------
+sub do_replay {
+    my $noderange            = shift;
+    my $start_time_of_replay = shift;
+    my $end_time_of_replay   = shift;
 
-        if (exists $macmap{$mac}) {
-            my $node   = $macmap{$mac}{"node"};
-            my $record = "Send DHCPACK on $ip back to $mac via $nic";
-            probe_utils->send_msg("$output", "d", "[$node] $record") if ($monitor);
-            push(@{ $rawdata{$node}{"history"} }, $record);
-            $ipnodemap{$ip} = $node;
-            push(@{ $rawdata{$node}{statehistory} }, $rawdata{$node}{state}) if (reset_state(\$rawdata{$node}{state}, "dhcp"));
+    my $rc = 0;
 
-            if ($macmap{$mac}{"ip"} != "NOIP" and $macmap{$mac}{"ip"} != $ip) {
-                my $warn_msg = "The ip($ip) assigned to $mac via DHCP is different from the ip($macmap{$mac}{'ip'}) in node definition.";
-                probe_utils->send_msg("$output", "w", "$warn_msg") if ($monitor);
-                push(@{ $rawdata{$node}{"history"} }, $warn_msg);
-            }
-        }
-    } elsif ($msg =~ /.+BOOTREQUEST\s+from\s+(.+)\s+via\s+([^:]+)(.*)/) {
-        my $mac = $1;
-        my $nic = $2;
-        if (exists $macmap{$mac}) {
-            my $node   = $macmap{$mac}{"node"};
-            my $record = "Receive BOOTREQUEST from $mac via $nic";
-            probe_utils->send_msg("$output", "d", "[$node] $record") if ($monitor);
-            push(@{ $rawdata{$node}{"history"} }, $record);
-        }
-    } elsif ($msg =~ /.+BOOTREPLY\s+for\s+(.+)\s+to\s+.+(\w\w:\w\w:\w\w:\w\w:\w\w:\w\w).+via\s+(.+)/) {
-        my $ip  = $1;
-        my $mac = $2;
-        my $nic = $3;
+    my $timestr = scalar(localtime($start_time_of_replay));
+    probe_utils->send_msg("stdout", "d", "Start to scan logs which are later than '$timestr', please waiting for a while.............");
 
-        if (exists $macmap{$mac}) {
-            my $node   = $macmap{$mac}{"node"};
-            my $record = "Send BOOTREPLY on $ip back to $mac via $nic";
-            probe_utils->send_msg("$output", "d", "[$node] $record") if ($monitor);
-            push(@{ $rawdata{$node}{"history"} }, $record);
-            $ipnodemap{$ip} = $node;
-            push(@{ $rawdata{$node}{statehistory} }, $rawdata{$node}{state}) if (reset_state(\$rawdata{$node}{state}, "dhcp"));
-
-            if ($macmap{$mac}{"ip"} != "NOIP" and $macmap{$mac}{"ip"} != $ip) {
-                my $warn_msg = "The ip($ip) assigned to $mac via DHCP is different from the ip($macmap{$mac}{'ip'}) in node definition.";
-                probe_utils->send_msg("$output", "w", "$warn_msg") if ($monitor);
-                push(@{ $rawdata{$node}{"history"} }, $warn_msg);
-            }
-        }
+    my %node_state;
+    init_node_state($noderange, \%node_state);
+    if ($debug) {
+        print "Dumper node_state-------\n";
+        print Dumper \%node_state;
     }
 
-    return 0;
+    my $log_parse = LogParse->new($verbose);
+    my @candidate_mn_hostname_in_log = $log_parse->obtain_candidate_mn_hostname_in_log();
+
+    while ($start_time_of_replay < $end_time_of_replay) {
+        my @valid_one_second_log_set;
+        my $rst = $log_parse->obtain_one_second_logs($start_time_of_replay, \@valid_one_second_log_set);
+        if ($rst) {
+            probe_utils->send_msg("stdout", "d", "Obtain logs failed from log files");
+            $rc = 1; 
+            last;
+        }
+
+        foreach my $log_ref (@valid_one_second_log_set) {
+            dispatch_log_to_handler($log_ref, \@candidate_mn_hostname_in_log, \%node_state);
+        }
+
+        $start_time_of_replay = $log_parse->obtain_next_second();
+    }
+    $log_parse->destory();
+
+    conclusion_report(\%node_state);
+    return $rc;
 }
 
 #------------------------------------------
 
 =head3
     Description:
-       Handle one line log come from tftp log file
+        Calculate the provision of every node. offer a report to customer
     Arguments:
-       msg: one line tftp log
+        node_state_ref: The reference of hash "%node_state". refer to function "init_node_state" for the structure of "%node_state"
     Returns:
-        0 : pass
-        1 : failed
+        0: success
+        1: failed
 =cut
 
 #------------------------------------------
-sub handle_tftp_msg {
-    my $msg = shift;
+sub conclusion_report {
+    my $node_state_ref = shift;
 
-    if ($msg =~ /RRQ\s+from\s+(.+)\s+filename\s+(.+)/i) {
-        my $ip     = $1;
-        my $file   = $2;
-        my $record = "Via TFTP download $file";
-        if (exists($rawdata{"$ipnodemap{$ip}"})) {
-            probe_utils->send_msg("$output", "d", "[$ipnodemap{$ip}] $record") if ($monitor);
-            push(@{ $rawdata{ $ipnodemap{$ip} }{"history"} }, $record);
-            push(@{ $rawdata{ $ipnodemap{$ip} }{statehistory} }, $rawdata{ $ipnodemap{$ip} }{state}) if (reset_state(\$rawdata{ $ipnodemap{$ip} }{state}, "tftp"));
-        }
+    probe_utils->send_msg("stdout", "", "==================conclusion_report=================");
+
+    if ($debug) {
+        print "---->the result of %node_state<------\n";
+        print Dumper $node_state_ref;
     }
-}
 
-#------------------------------------------
-
-=head3
-    Description:
-       Handle one line log come from http log file
-    Arguments:
-       msg: one line http log
-    Returns:
-        0 : pass
-        1 : failed
-=cut
-
-#------------------------------------------
-sub handle_http_msg {
-    my $msg = shift;
-    if ($msg =~ /(\d+\.\d+.\d+.\d+)\s.+GET\s+(.+)\s+HTTP.+/) {
-        my $ip     = $1;
-        my $file   = $2;
-        my $record = "Via HTTP get $file";
-
-        if (exists($rawdata{"$ipnodemap{$ip}"})) {
-            probe_utils->send_msg("$output", "d", "[$ipnodemap{$ip}] $record") if ($monitor);
-            push(@{ $rawdata{ $ipnodemap{$ip} }{"history"} }, $record);
-            push(@{ $rawdata{ $ipnodemap{$ip} }{statehistory} }, $rawdata{ $ipnodemap{$ip} }{state}) if (reset_state(\$rawdata{ $ipnodemap{$ip} }{state}, "http"));
-        }
-    }
-    return 0;
-}
-
-#------------------------------------------
-
-=head3
-    Description:
-       Handle one line log come from cluster.log
-    Arguments:
-       msg: one line log
-    Returns:
-        0 : pass
-        1 : failed
-=cut
-
-#------------------------------------------
-sub handle_cluster_msg {
-    my $line = shift;
-    my $node = "";
-    my $msg;
-    my $status;
-
-    my @splitline = split(/\s+/, $line);
-    if (($splitline[4] =~ /^xcat/i) || ($splitline[5] =~ /^xcat/i)) {
-
-        #log like: Aug  7 22:30:31 c910f02c01p09 xcat: c910f02c04p04 status: booted statustime: 08-07-2016 22:30:31
-        if (($splitline[6] =~ /^status:$/i) && ($splitline[8] =~ /^statustime:$/)) {
-            $node   = $splitline[5];
-            $status = $splitline[7];
-            if (exists($rawdata{$node})) {
-                my $record = "Node status is changed to $status";
-                probe_utils->send_msg("$output", "d", "[$node] $record") if ($monitor);
-                push(@{ $rawdata{$node}{"history"} }, $record);
-
-                #one node finish deployment
-                if ($status eq "booted") {
-                    $monitor_nodes{$node}{"rst"} = 1 if (defined($monitor_nodes{$node}));
-                    push(@{ $rawdata{$node}{statehistory} }, $rawdata{$node}{state}) if (reset_state(\$rawdata{$node}{state}, "booted"));
-                }
-
-                if ($status eq "powering-on") {
-                    push(@{ $rawdata{$node}{statehistory} }, $rawdata{$node}{state}) if (reset_state(\$rawdata{$node}{state}, "poweron"));
-                }
+    if ($verbose) {
+        probe_utils->send_msg("stdout", "d", "----------node history state----------");
+        foreach my $node (keys %$node_state_ref) {
+            my $allhistorystate;
+            my $historystate;
+            if (@{ $node_state_ref->{$node}{allstatehistory} }) {
+                $allhistorystate .= "$::STATE_DESC{$_} " foreach (@{ $node_state_ref->{$node}{allstatehistory} });
+                probe_utils->send_msg("stdout", "d", "[$node] the history before last loop provision: $allhistorystate");
             }
 
+            $historystate .= "$::STATE_DESC{$_} " foreach (@{ $node_state_ref->{$node}{statehistory} });
+            probe_utils->send_msg("stdout", "d", "[$node] last loop provision history: $historystate");
         }
+        probe_utils->send_msg("stdout", "d", "--------------------------------------");
     }
-    return 0;
-}
 
-#-----------------------------------------
+    my %failed_node;
+    foreach my $node (keys %$node_state_ref) {
+        my $stop_stage = 0;
 
-=head3
-    Description:
-       Handle one line log come from computes.log
-    Arguments:
-       msg: one line compute log
-    Returns:
-        0 : pass
-        1 : failed
-=cut
+        my $start_rpower   = 0;
+        my $power_on       = 0;
+        my $isntalling     = 0;
+        my $postbootscript = 0;
 
-#-----------------------------------------
-sub handle_compute_msg {
-    my $line = shift;
-    my $sender;
-    my $node;
-    my $msg;
+        #calculate node provision result
+        #the max value of all state is the final stop stage
+        foreach (@{ $node_state_ref->{$node}{statehistory} }) {
+            $stop_stage     = $_ if ($stop_stage < $_);
+            $start_rpower   = 1  if ($_ == $::STATE_POWER_ON);
+            $power_on       = 1  if ($_ == $::STATE_POWERINGON);
+            $isntalling     = 1  if ($_ == $::STATE_INSTALLING);
+            $postbootscript = 1  if ($_ == $::STATE_POSTBOOTSCRIPT);
+        }
 
-    if ($line =~ /.+\d{2}:\d{2}:\d{2}\s+(.+)\s+(xcat.+)/i) {
-        $sender = $1;
-        $msg    = $2;
+        # Cover limited non-privision error
+        # 1 if xcatd receive reboot command to do provision (such like rpower, rnetboot, rinstall...) but the node status didn't change to "powering-on"
+        #   that means reboot target node failed.
+        # 2 if power on target node successfully and there is 'running postbootscript' in node state history, but without "installing" state,
+        #   It is very possible to just do reboot process
 
-        if (!xCAT::NetworkUtils->isIpaddr($sender)) {
-            $node = $sender;
+        if ($start_rpower && !$power_on) {
+            $failed_node{$node}{non_provision_prediction} = "Trigger target node reboot failed";
+        } elsif ($start_rpower && $power_on && !$isntalling && $postbootscript) {
+            $failed_node{$node}{non_provision_prediction} = "Target node just reboot from disk";
         } else {
-            $node = $ipnodemap{$sender};
-        }
-
-        if ($node ne "" && exists($rawdata{$node})) {
-            probe_utils->send_msg("$output", "d", "[$node] $msg") if ($monitor);
-            push(@{ $rawdata{$node}{"history"} }, $msg);
-
-            #node start to run postscript or postbootscript
-            if ($msg =~ /Running postscript/) {
-                push(@{ $rawdata{$node}{statehistory} }, $rawdata{$node}{state}) if (reset_state(\$rawdata{$node}{state}, "postscript"));
+            if ($stop_stage != $::STATE_COMPLETED) {
+                $failed_node{$node}{provision_stop_point} = $stop_stage;
             }
         }
     }
 
+
+    if (%failed_node) {
+        my $failed_node_num = keys %failed_node;
+        if ($failed_node_num > 1) {
+            probe_utils->send_msg("stdout", "d", "There are $failed_node_num node provision failed");
+        } elsif ($failed_node_num == 1) {
+            probe_utils->send_msg("stdout", "d", "There is $failed_node_num node provision failed");
+        }
+
+        foreach my $node (keys %failed_node) {
+            if ($failed_node{$node}{non_provision_prediction}) {
+                probe_utils->send_msg("stdout", "f", "$node : $failed_node{$node}{non_provision_prediction}");
+            } else {
+                probe_utils->send_msg("stdout", "f", "$node : stop at stage '$::STATE_DESC{$failed_node{$node}{provision_stop_point}}'");
+            }
+        }
+    } else {
+        probe_utils->send_msg("stdout", "o", "All nodes provision successfully");
+    }
     return 0;
 }
 
@@ -501,20 +493,178 @@ sub handle_compute_msg {
 
 =head3
     Description:
-       Test if all nodes have finished job
+        Implement the monitor feature
     Arguments:
-       One golble attribute %monitor_nodes;
+        noderange : the range of node
+        maxwaittime:  the max waiting time for monitor. 
     Returns:
-       1: all nodes finished
-       0: not all nodes finished
+        0: success
+        1: failed
+=cut
+
+#------------------------------------------
+sub do_monitor {
+    my $noderange   = shift;
+    my $maxwaittime = shift;
+
+    my $rst      = 0;
+    my $terminal = 0;
+
+    $SIG{TERM} = $SIG{INT} = sub {
+        $terminal = 1;
+    };
+
+    my $startline =
+      "-------------------------------------------------------------
+Start capturing every message during OS provision process....
+-------------------------------------------------------------
+";
+    probe_utils->send_msg("stdout", "", "$startline");
+
+    my @openfilepids;
+    my @openfilefds;
+    my %fd_filetype_map;
+
+    {    #a very important brace to hold a code block
+        my $log_parse         = LogParse->new($verbose);
+        my $candidate_log_ref = $log_parse->obtain_log_file_list();
+
+        #open candidate log file to obtain realtime log
+        if (%$candidate_log_ref) {
+            foreach my $logfile (keys %$candidate_log_ref) {
+                my $pid;
+                my $fd;
+                if (!($pid = open($fd, "tail -f -n 0 $candidate_log_ref->{$logfile}{file} 2>&1 |"))) {
+                    probe_utils->send_msg("stdout", "f", "Can't open $candidate_log_ref->{$logfile}{file} to get logs");
+                    $rst = 1;
+                    last;
+                } else {
+                    push @openfilepids, $pid;
+                    push @openfilefds,  $fd;
+                    $fd_filetype_map{$fd} = $candidate_log_ref->{$logfile}{type};
+                }
+            }
+        } else {
+            probe_utils->send_msg("stdout", "f", "There isn't any vaild log file to be scaned");
+            $rst = 1;
+        }
+
+        last if ($rst);
+
+        my %node_state;
+        init_node_state($noderange, \%node_state);
+
+        my $select = new IO::Select;
+        $select->add(\*$_) foreach (@openfilefds);
+        $| = 1;
+
+        my @hdls;
+        my $starttime = time();
+        my @candidate_mn_hostname_in_log = $log_parse->obtain_candidate_mn_hostname_in_log();
+
+        #read log realtimely, then handle each log
+        for (; ;) {
+            if (@hdls = $select->can_read(0)) {
+                foreach my $hdl (@hdls) {
+                    my $line = "";
+                    chomp($line = <$hdl>);
+                    my $log_content_ref = $log_parse->obtain_log_content($fd_filetype_map{$hdl}, $line);
+                    dispatch_log_to_handler($log_content_ref, \@candidate_mn_hostname_in_log, \%node_state);
+                }
+            }
+
+            # stop reading log at below 3 scenarios
+            # 1 receive terminal signal from customer
+            if ($terminal) {
+                probe_utils->send_msg("stdout", "d", "Get INT or TERM signal from STDIN");
+                last;
+
+                # 2 all node have finished the provision
+            } elsif (all_monitor_node_done(\%node_state)) {
+                probe_utils->send_msg("stdout", "o", "All nodes specified to monitor, have finished OS provision process");
+                last;
+
+                # 3 exceed the max waiting time
+            } elsif (time() - $starttime > ($maxwaittime * 60)) {
+                probe_utils->send_msg("stdout", "i", "$maxwaittime minutes have expired, stop monitoring");
+                last;
+            } else {
+                sleep 0.01;
+            }
+        }
+
+        conclusion_report(\%node_state);
+        $log_parse->destory();
+    }
+
+    # close all running sub process
+    my $existrunningpid = 0;
+    $existrunningpid = 1 if (@openfilepids);
+    my $trytime = 0;
+    while ($existrunningpid) {
+
+        #send terminal signal to all running process at same time
+        if ($try < 5) {    #try INT 5 up to 5 times
+            foreach my $pid (@openfilepids) {
+                kill 'INT', $pid if ($pid);
+            }
+        } elsif ($try < 10) {    #try TERM 5 up to 5 times
+            foreach my $pid (@openfilepids) {
+                kill 'TERM', $pid if ($pid);
+            }
+        } else {                 #try KILL 1 time
+            foreach my $pid (@openfilepids) {
+                kill 'KILL', $pid if ($pid);
+            }
+        }
+        ++$try;
+        sleep 1;
+
+        #To check how many process exit, set the flag of exited process to 0
+        for (my $i = 0 ; $i <= $#openfilepids ; $i++) {
+            $openfilepids[$i] = 0 if (waitpid($openfilepids[$i], WNOHANG));
+        }
+
+        #To check if there are processes still running, if there are, try kill again in next loop
+        $existrunningpid = 0;
+        $existrunningpid |= $_ foreach (@openfilepids);
+
+        #just try 10 times, if still can't kill some process, give up
+        if ($try > 10) {
+            my $leftpid;
+            foreach my $pid (@openfilepids) {
+                $leftpid .= "$pid " if ($pid);
+            }
+            probe_utils->send_msg("stdout", "d", "Can't stop process $leftpid, please handle manually.");
+            last;
+        }
+    }
+
+    # close all openning file descriptors
+    close($_) foreach (@openfilefds);
+
+    return $rst;
+}
+
+#------------------------------------------
+
+=head3
+    Description:
+        Check if all node have been finished the provision process
+    Arguments:
+        node_state_ref: The reference of hash "%node_state". refer to function "init_node_state" for the structure of "%node_state"
+    Returns:
+        0: success
+        1: failed
 =cut
 
 #------------------------------------------
 sub all_monitor_node_done {
-    my $done = 1;
+    my $node_state_ref = shift;
+    my $done           = 1;
 
-    foreach my $node (keys %monitor_nodes) {
-        if ($monitor_nodes{$node}{"rst"} == 0) {
+    foreach my $node (keys %$node_state_ref) {
+        if ($node_state_ref->{$node}{done} == 0) {
             $done = 0;
             last;
         }
@@ -527,134 +677,176 @@ sub all_monitor_node_done {
 
 =head3
     Description:
-       Dump monitor history,  categorised by mac address.
+        Handle one line DHCP log
     Arguments:
-       NULL
+        log_ref: (input attribute) the reference of hash which save one line dhcp log. 
+        node_state_ref: (output attribute), the reference of hash "%node_state". refer to function "init_node_state" for the structure of "%node_state" 
     Returns:
+        NULL
 =cut
 
 #------------------------------------------
-sub dump_history {
+sub handle_dhcp_msg {
+    my $log_ref        = shift;
+    my $node_state_ref = shift;
+    if ($log_ref->{msg} =~ /DHCPDISCOVER\s+from\s+(.+)\s+via\s+([^:]+)(.*)/i) {
+        my $mac = $1;
+        my $nic = $2;
 
-    my $title = "
-=============================================================
-=  The summary of os provision:
-=============================================================
-";
-    print "$title\n";
-
-    foreach $node (keys %rawdata) {
-
-        if ($verbose) {
-            print "[$node]\n";
-
-            my $httphit = 0;
-            my @httptmp;
-            foreach my $line (@{ $rawdata{$node}{"history"} }) {
-                if ($line =~ /Via HTTP/) {
-                    if ($httphit) {
-                        shift @httptmp if ($#httptmp > 0);
-                        push @httptmp, $line;
-                    } else {
-                        print "\t$line\n";
-                        $httphit = 1;
-                    }
-                } else {
-                    if ($#httptmp > -1) {
-                        print "\tVia HTTP ..........\n";
-                        print "\t$_\n" foreach (@httptmp);
-                    }
-                    @httptmp = ();
-                    $httphit = 0;
-                    print "\t$line\n";
-                }
-            }
-            my $statelist = "";
-            for (my $i = 0 ; $i < scalar(@{ $rawdata{$node}{statehistory} }) ; $i++) {
-                $statelist .= "$state_set_reverse{$rawdata{$node}{statehistory}[$i]} ";
-            }
-            probe_utils->send_msg("$output", "d", "[$node] state history: $statelist");
+        if (exists $macmap{$mac}) {
+            my $node   = $macmap{$mac}{"node"};
+            my $record = "Receive DHCPDISCOVER via $nic";
+            probe_utils->send_msg("stdout", "d", "[$node] $record") if ($monitor);
+            push(@{ $node_state_ref->{$node}{log} }, $log_ref->{msg}) if ($debug);
         }
+    } elsif ($log_ref->{msg} =~ /DHCPOFFER\s+on\s+(.+)\s+to\s+(.+)\s+via\s+(.+)/i) {
+        my $ip  = $1;
+        my $mac = $2;
+        my $nic = $3;
 
-        my @tmpnodestatehistory = @{ $rawdata{$node}{statehistory} };
-
-        #print "state history = @tmpnodestatehistory\n";
-
-        my %match_result;
-        my $procidx      = 0;
-        my $newloop      = 0;
-        my $notfirstloop = 0;
-        while (@tmpnodestatehistory) {
-            undef %match_result if ($notfirstloop);
-            $newloop = 0;
-            foreach my $type (keys %valid_process) {
-                if ($notfirstloop) {
-                    $procidx = 1;
-                } else {
-                    $procidx = 0;
-                }
-                my $proclen = scalar(@{ $valid_process{$type}{process} });
-                my $i;
-                for ($i = 0 ; $i < scalar(@tmpnodestatehistory) ; $i++) {
-                    if ($procidx < $proclen) {
-                        if ($tmpnodestatehistory[$i] == $valid_process{$type}{process}[$procidx]) {
-                            ++$procidx;
-                        } else {
-                            --$procidx if (notfirstloop);
-                            push @{ $match_result{$procidx} }, $type;
-                            last;
-                        }
-                    } else {
-                        splice(@tmpnodestatehistory, 0, $i);
-                        $newloop      = 1;
-                        $notfirstloop = 1;
-                        last;
-                    }
-                }
-                if ($i == scalar(@tmpnodestatehistory)) {
-                    push @{ $match_result{$procidx} }, $type;
-                    next;
-                }
-                last if ($newloop);
-            }
-            last if (!$newloop);
+        if (exists $macmap{$mac}) {
+            my $node   = $macmap{$mac}{"node"};
+            my $record = "Send DHCPOFFER on $ip back to $mac via $nic";
+            probe_utils->send_msg("stdout", "d", "[$node] $record") if ($monitor);
+            push(@{ $node_state_ref->{$node}{log} }, $log_ref->{msg}) if ($debug);
         }
+    } elsif ($log_ref->{msg} =~ /DHCPREQUEST\s+for\s+(.+)\s+[\(\)0-9\.]*\s*from\s+(.+)\s+via\s+(.+)/) {
+        my $ip  = $1;
+        my $mac = $2;
+        my $nic = $3;
 
-       #print "------------result---------------\n";
-       #print Dumper %match_result;
-
-        my $max_match = 0;
-        foreach my $key (keys %match_result) {
-            $max_match = $key if ($key > $max_match);
+        if (exists $macmap{$mac}) {
+            my $node   = $macmap{$mac}{"node"};
+            my $record = $log_ref->{msg};
+            probe_utils->send_msg("stdout", "d", "[$node] $record") if ($monitor);
+            push(@{ $node_state_ref->{$node}{log} }, $log_ref->{msg}) if ($debug);
         }
+    } elsif ($log_ref->{msg} =~ /DHCPACK\s+on\s+(.+)\s+to\s+(.+)\s+via\s+(.+)/) {
+        my $ip  = $1;
+        my $mac = $2;
+        my $nic = $3;
 
-        my $formatprefix;
-        if ($max_match == 0) {
-            my $statelist = "";
-            for (my $i = 0 ; $i < scalar(@{ $rawdata{$node}{statehistory} }) ; $i++) {
-                $statelist .= "$state_set_reverse{$rawdata{$node}{statehistory}[$i]} ";
+        if (exists $macmap{$mac}) {
+            my $node   = $macmap{$mac}{"node"};
+            my $record = "Send DHCPACK on $ip back to $mac via $nic";
+            probe_utils->send_msg("stdout", "d", "[$node] $record") if ($monitor);
+            push(@{ $node_state_ref->{$node}{log} }, $log_ref->{msg}) if ($debug);
+
+            if ($macmap{$mac}{"ip"} != "NOIP" and $macmap{$mac}{"ip"} != $ip) {
+                my $warn_msg = "The ip($ip) assigned to $mac via DHCP is different from the ip($macmap{$mac}{'ip'}) in node definition.";
+
+                #probe_utils->send_msg("stdout", "w", "$warn_msg") if ($monitor);
+                probe_utils->send_msg("stdout", "w", "$warn_msg");
             }
-            probe_utils->send_msg("$output", "f", "[$node] deployment failed");
-            probe_utils->send_msg("$output", "d", "\t$node did unknown process, state change history is $statelist");
-        } elsif (scalar(@{ $match_result{$max_match} }) > 1) {
-            probe_utils->send_msg("$output", "f", "[$node] deployment failed");
-            probe_utils->send_msg("$output", "d", "\tThere are more than one possible process satisfy $node situation");
-            foreach my $proc (@{ $match_result{$max_match} }) {
-                if ($valid_process{$proc}{type} eq "deploy") {
-                    probe_utils->send_msg("$output", "d", "\tpossible process \"deploy\", pass $state_set_reverse{$valid_process{$proc}{process}[$max_match-1]} stage, something wrong during $state_set_reverse{$valid_process{$proc}{process}[$max_match]} stage");
-                } elsif ($valid_process{$proc}{type} eq "reboot") {
-                    probe_utils->send_msg("$output", "d", "\tpossible process \"reboot\", pass $state_set_reverse{$valid_process{$proc}{process}[$max_match-1]} stage, something wrong during $state_set_reverse{$valid_process{$proc}{process}[$max_match]} stage");
-                }
+
+            $ipnodemap{$ip} = $node;
+            set_node_state($node_state_ref, $node, $::STATE_DHCP);
+        }
+    } elsif ($log_ref->{msg} =~ /BOOTREQUEST\s+from\s+(.+)\s+via\s+([^:]+)(.*)/) {
+        my $mac = $1;
+        my $nic = $2;
+        if (exists $macmap{$mac}) {
+            my $node   = $macmap{$mac}{"node"};
+            my $record = "Receive BOOTREQUEST from $mac via $nic";
+            probe_utils->send_msg("stdout", "d", "[$node] $record") if ($monitor);
+            push(@{ $node_state_ref->{$node}{log} }, $log_ref->{msg}) if ($debug);
+        }
+    } elsif ($log_ref->{msg} =~ /BOOTREPLY\s+for\s+(.+)\s+to\s+.+(\w\w:\w\w:\w\w:\w\w:\w\w:\w\w).+via\s+(.+)/) {
+        my $ip  = $1;
+        my $mac = $2;
+        my $nic = $3;
+
+        if (exists $macmap{$mac}) {
+            my $node   = $macmap{$mac}{"node"};
+            my $record = "Send BOOTREPLY on $ip back to $mac via $nic";
+            probe_utils->send_msg("stdout", "d", "[$node] $record") if ($monitor);
+            push(@{ $node_state_ref->{$node}{log} }, $log_ref->{msg}) if ($debug);
+
+            if ($macmap{$mac}{"ip"} != "NOIP" and $macmap{$mac}{"ip"} != $ip) {
+                my $warn_msg = "The ip($ip) assigned to $mac via DHCP is different from the ip($macmap{$mac}{'ip'}) in node definition.";
+
+                #probe_utils->send_msg("stdout", "w", "$warn_msg") if ($monitor);
+                probe_utils->send_msg("stdout", "w", "$warn_msg");
             }
-        } else {
-            if (($valid_process{ $match_result{$max_match}[0] }{type} eq "deploy") && ($valid_process{ $match_result{$max_match}[0] }{process}[ $max_match - 1 ] == $state_set{done})) {
-                probe_utils->send_msg("$output", "o", "[$node] deployment completed");
-            } elsif (($valid_process{ $match_result{$max_match}[0] }{type} eq "deploy") && ($valid_process{ $match_result{$max_match}[0] }{process}[ $max_match - 1 ] != $state_set{done})) {
-                probe_utils->send_msg("$output", "f", "[$node] deployment failed, pass $state_set_reverse{$valid_process{$match_result{$max_match}[0]}{process}[$max_match-1]} stage, something wrong during $state_set_reverse{$valid_process{$match_result{$max_match}[0]}{process}[$max_match]} stage")
-            } elsif (($valid_process{ $match_result{$max_match}[0] }{type} eq "reboot") && ($valid_process{ $match_result{$max_match}[0] }{process}[ $max_match - 1 ] == $state_set{done})) {
-                probe_utils->send_msg("$output", "f", "[$node] reboot completed, without deployment process");
-            } elsif (($valid_process{ $match_result{$max_match}[0] }{type} eq "reboot") && ($valid_process{ $match_result{$max_match}[0] }{process}[ $max_match - 1 ] != $state_set{done})) {
-                probe_utils->send_msg("$output", "f", "[$node] reboot failed, without deployment process, stop at $state_set_reverse{$valid_process{$match_result{$max_match}[0]}{process}[$max_match-1]} stage, something wrong during $state_set_reverse{$valid_process{$match_result{$max_match}[0]}{process}[$max_match]} stage");
+
+            $ipnodemap{$ip} = $node;
+            set_node_state($node_state_ref, $node, $::STATE_DHCP);
+        }
+    }
+}
+
+#------------------------------------------
+
+=head3
+    Description:
+        Handle one line TFTP log
+    Arguments:
+        log_ref: (input attribute) the reference of hash which save one line TFTP log.
+        node_state_ref: (output attribute), the reference of hash "%node_state". refer to function "init_node_state" for the structure of "%node_state"
+    Returns:
+        NULL
+=cut
+
+#------------------------------------------
+sub handle_tftp_msg {
+    my $log_ref        = shift;
+    my $node_state_ref = shift;
+
+    if ($log_ref->{msg} =~ /RRQ\s+from\s+(.+)\s+filename\s+(.+)/i) {
+        my $ip     = $1;
+        my $file   = $2;
+        my $record = "Via TFTP download $file";
+        if (exists($node_state_ref->{ $ipnodemap{$ip} })) {
+            probe_utils->send_msg("stdout", "d", "[$ipnodemap{$ip}] $record") if ($monitor);
+            push(@{ $node_state_ref->{ $ipnodemap{$ip} }{log} }, $log_ref->{msg}) if ($debug);
+
+            if ($file =~ /xcat\/xnba.*/i or $file =~ /\/boot\/grub2\/grub2\-/i or $file =~ /\yb\/node\/yaboot\-/i) {
+                set_node_state($node_state_ref, $ipnodemap{$ip}, $::STATE_BOOTLODER);
+            } elsif ($file =~ /vmlinuz|inst64|linux/) {
+                set_node_state($node_state_ref, $ipnodemap{$ip}, $::STATE_KERNEL);
+            } elsif ($file =~ /initrd/i) {
+                set_node_state($node_state_ref, $ipnodemap{$ip}, $::STATE_INITRD);
+            }
+        }
+    }
+}
+
+
+#------------------------------------------
+
+=head3
+    Description:
+        Handle one line HTTP log
+    Arguments:
+        log_ref: (input attribute) the reference of hash which save one line HTTP log.
+        node_state_ref: (output attribute), the reference of hash "%node_state". refer to function "init_node_state" for the structure of "%node_state"
+    Returns:
+        NULL
+=cut
+
+#------------------------------------------
+sub handle_http_msg {
+    my $log_ref        = shift;
+    my $node_state_ref = shift;
+    my $ip             = $log_ref->{sender};
+
+    if (exists($node_state_ref->{ $ipnodemap{$ip} })) {
+
+        if ($log_ref->{msg} =~ /GET\s+(.+)\s+HTTP.+/ or $log_ref->{msg} =~ /HEAD\s+(.+)\s+HTTP.+/) {
+            my $file   = $1;
+            my $record = "Via HTTP get $file";
+
+            probe_utils->send_msg("stdout", "d", "[$ipnodemap{$ip}] $record") if ($monitor);
+            push(@{ $node_state_ref->{ $ipnodemap{$ip} }{log} }, $log_ref->{msg}) if ($debug);
+
+            if ($file =~ /vmlinuz|inst64/i or ($file =~ /linux/i and $file =~ /osimage/i)) {
+                set_node_state($node_state_ref, $ipnodemap{$ip}, $::STATE_KERNEL);
+            } elsif ($file =~ /initrd/i and $file =~ /osimage/i) {
+                set_node_state($node_state_ref, $ipnodemap{$ip}, $::STATE_INITRD);
+            } elsif ($file =~ /\/install\/autoinst\/*/i) {
+                set_node_state($node_state_ref, $ipnodemap{$ip}, $::STATE_KICKSTART);
+            } elsif ($file =~ /\.deb/i or ($file =~ /\/install\// and $file =~ /\/repodata\//i)) {
+                set_node_state($node_state_ref, $ipnodemap{$ip}, $::STATE_INSTALLRPM);
             }
         }
     }
@@ -664,415 +856,151 @@ sub dump_history {
 
 =head3
     Description:
-       Monitor the process of os provision
+        Handle one line log comes from cluster.log
+    Arguments:
+        log_ref: (input attribute) the reference of hash which save one line log comes from cluster.log.
+        node_state_ref: (output attribute), the reference of hash "%node_state". refer to function "init_node_state" for the structure of "%node_state"
     Returns:
-        0: pass
-        1: failed
+        NULL
 =cut
 
 #------------------------------------------
-sub do_monitor {
-    $SIG{TERM} = $SIG{INT} = sub {
-        $terminal = 1;
-    };
+sub handle_cluster_msg {
+    my $log_ref        = shift;
+    my $node_state_ref = shift;
 
-    my $rst = 0;
-    my $startline =
-      "-------------------------------------------------------------
-Start capturing every message during OS provision process......
--------------------------------------------------------------
-";
+    my $log_msg = $log_ref->{msg};
+    my @split_log = split(/\s+/, $log_msg);
 
-    print("$startline\n");
+    if ($split_log[1] eq "Allowing") {
+        my $command     = $split_log[2];
+        my $nodes_str   = $split_log[4];
+        my $sub_command = $split_log[5];
 
-    my $varlogmsg  = "/var/log/messages";
-    my $clusterlog = "/var/log/xcat/cluster.log";
-    my $computelog = "/var/log/xcat/computes.log";
-
-    my $httplog;
-    if (-e "/var/log/httpd/access_log") {
-        $httplog = "/var/log/httpd/access_log";
-    } elsif (-e "/var/log/apache2/access_log") {
-        $httplog = "/var/log/apache2/access_log";
-    } elsif (-e "/var/log/apache2/access.log") {
-        $httplog = "/var/log/apache2/access.log";
-    }
-
-    my $varlogpid;
-    my $clusterpid;
-    my $httppid;
-    my $computerpid;
-
-    {
-        if (!-e $varlogmsg) {
-            probe_utils->send_msg("$output", "w", "$varlogmsg doesn't exist");
-        } else {
-            if (!($varlogpid = open(VARLOGMSGFILE, "tail -f -n 0 $varlogmsg 2>&1 |"))) {
-                probe_utils->send_msg("$output", "f", "Can't open $varlogmsg to get logs");
-                $rst = 1;
-                last;
-            }
-        }
-
-        if (!-e "$clusterlog") {
-            probe_utils->send_msg("$output", "w", "$clusterlog doesn't exist");
-            probe_utils->send_msg("$output", "i", "If using SLES11 or xCAT2.11.x, ignore above warning");
-        } else {
-            if (!($clusterpid = open(CLUSTERLOGFILE, "tail -f -n 0 $clusterlog 2>&1 |"))) {
-                probe_utils->send_msg("$output", "f", "Can't open $clusterlog to get logs");
-                $rst = 1;
-                last;
-            }
-        }
-        if (!-e "$httplog") {
-            probe_utils->send_msg("$output", "w", "$httplog doesn't exist");
-        } else {
-            if (!($httppid = open(HTTPLOGFILE, "tail -f -n 0 $httplog 2>&1 |"))) {
-                probe_utils->send_msg("$output", "f", "Can't open $httplog to get logs");
-                $rst = 1;
-                last;
-            }
-        }
-        if (!-e "$computelog") {
-            probe_utils->send_msg("$output", "w", "$computelog doesn't exist");
-            probe_utils->send_msg("$output", "i", "If sles11 or xCAT2.11.x, ignore above warning");
-        } else {
-            if (!($computerpid = open(COMPUTERFILE, "tail -f -n 0 $computelog 2>&1 |"))) {
-                probe_utils->send_msg("$output", "f", "Can't open $computelog to get logs");
-                $rst = 1;
-                last;
-            }
-        }
-
-        my $select = new IO::Select;
-        $select->add(\*VARLOGMSGFILE);
-        $select->add(\*CLUSTERLOGFILE);
-        $select->add(\*HTTPLOGFILE);
-        $select->add(\*COMPUTERFILE);
-        $| = 1;
-
-        my $line = "";
-        my @hdls;
-        my $hdl;
-
-        my $starttime = time();
-        $monitor = 1;
-        for (; ;) {
-            if (@hdls = $select->can_read(0)) {
-                foreach $hdl (@hdls) {
-                    if ($hdl == \*VARLOGMSGFILE) {
-                        chomp($line = <VARLOGMSGFILE>);
-                        my @tmp = split(/\s+/, $line);
-                        if ($tmp[4] =~ /dhcpd/i && $line =~ /$installnic/) {
-                            handle_dhcp_msg("$line");
-                        } elsif ($tmp[4] =~ /in.tftpd/i) {
-                            handle_tftp_msg("$line");
-                        } elsif (($tmp[4] =~ /^xcat/i) || ($tmp[5] =~ /^xcat/i)) {
-                            if (grep(/^$tmp[3]$/, @candidate_svr_hostname_inlog)) {
-                                handle_cluster_msg("$line");
-                            } else {
-                                handle_compute_msg("$line");
-                            }
-                        }
-                    } elsif ($hdl == \*CLUSTERLOGFILE) {
-                        chomp($line = <CLUSTERLOGFILE>);
-                        handle_cluster_msg("$line");
-                    } elsif ($hdl == \*HTTPLOGFILE) {
-                        chomp($line = <HTTPLOGFILE>);
-                        handle_http_msg("$line");
-                    } elsif ($hdl == \*COMPUTERFILE) {
-                        chomp($line = <COMPUTERFILE>);
-                        handle_compute_msg("$line");
-                    }
+        if ($command eq "rinstall" or $command eq "rnetboot" or ($command eq "rpower" and $sub_command =~ /on|boot|reset/)) {
+            my @nodes = probe_utils->parse_node_range($nodes_str);
+            foreach my $node (@nodes) {
+                if (exists $node_state_ref->{$node}) {
+                    probe_utils->send_msg("stdout", "d", "[$node] Use command $command to reboot node $node") if ($monitor);
+                    push(@{ $node_state_ref->{$node}{log} }, $log_ref->{msg}) if ($debug);
+                    set_node_state($node_state_ref, $node, $::STATE_POWER_ON);
                 }
             }
-
-            if ($terminal || (%monitor_nodes && all_monitor_node_done())) {
-                if ($terminal) {
-                    probe_utils->send_msg("$output", "d", "Get INT or TERM signal from STDIN");
-                } else {
-                    probe_utils->send_msg("$output", "o", "All nodes specified to monitor, have finished OS provision process");
-                }
-                last;
-            }
-
-            if (time() - $starttime > ($maxwaittime * 60)) {
-                probe_utils->send_msg("$output", "i", "$maxwaittime minutes have expired, stop monitoring");
-                last;
-            }
-            sleep 0.01;
         }
-        &dump_history;
-    }
-    kill 'INT', $varlogpid   if ($varlogpid);
-    kill 'INT', $clusterpid  if ($clusterpid);
-    kill 'INT', $httppid     if ($httppid);
-    kill 'INT', $computerpid if ($computerpid);
-    close(VARLOGMSGFILE)  if (VARLOGMSGFILE);
-    close(CLUSTERLOGFILE) if (CLUSTERLOGFILE);
-    close(HTTPLOGFILE)    if (HTTPLOGFILE);
-    close(COMPUTERFILE)   if (COMPUTERFILE);
+    } elsif ($log_ref->{msg} =~ /(.+)\s+status:\s+(.+)\s+statustime:/) {
+        my $node   = $1;
+        my $status = $2;
+        my $record = "Node status is changed to $status";
 
-    return $rst;
+        if (exists $node_state_ref->{$node}) {
+            probe_utils->send_msg("stdout", "d", "[$node] $record") if ($monitor);
+            push(@{ $node_state_ref->{$node}{log} }, $log_ref->{msg}) if ($debug);
+
+            if ($status eq "installing") {
+                set_node_state($node_state_ref, $node, $::STATE_INSTALLING);
+            } elsif ($status eq "powering-on") {
+                set_node_state($node_state_ref, $node, $::STATE_POWERINGON);
+            } elsif ($status eq "booting") {
+                set_node_state($node_state_ref, $node, $::STATE_BOOTING);
+            }
+        }
+    }
 }
 
-sub get_valid_logs {
-    my $ref_timestamp        = shift;
-    my $year                 = shift;
-    my $epoch_seconds_of_now = shift;
-    my $bthistory_ref        = shift;
-    my @orglogfilelist       = ("/var/log/xcat/cluster.log",
-        "/var/log/messages",
-        "/var/log/xcat/computes.log");
-    my $httplog;
-    if (-e "/var/log/httpd/access_log") {
-        $httplog = "/var/log/httpd/access_log";
-    } elsif (-e "/var/log/apache2/access_log") {
-        $httplog = "/var/log/apache2/access_log";
-    } elsif (-e "/var/log/apache2/access.log") {
-        $httplog = "/var/log/apache2/access.log";
-    }
-    push @orglogfilelist, $httplog;
+#------------------------------------------
 
-    foreach my $f (@orglogfilelist) {
-        my $filename = basename("$f");
-        $filename =~ s/(.+)\.(.+)/$1/g;
-        my $path_only = dirname("$f");
-        my @rotatefiles;
-        
-        #TODO  using opendir to refine below code
-        my @alltargetfiles = `ls -lt $path_only |awk -F" " '/ $filename/ {print \$9}'`;
-        foreach my $samenamefile (@alltargetfiles) {
-            chomp($samenamefile);
-            push @rotatefiles, "$path_only/$samenamefile";
-        }
+=head3
+    Description:
+        Handle one line log comes from computes.log
+    Arguments:
+        log_ref: (input attribute) the reference of hash which save one line log comes from computes.log.
+        node_state_ref: (output attribute), the reference of hash "%node_state". refer to function "init_node_state" for the structure of "%node_state"
+    Returns:
+        NULL
+=cut
 
-        my $ishttplog = 0;
-        $ishttplog = 1 if ($filename =~ /access/);
+#------------------------------------------
+sub handle_compute_msg {
+    my $log_ref        = shift;
+    my $node_state_ref = shift;
+    my $node           = $log_ref->{sender};
 
-        foreach my $file (@rotatefiles) {
-            my $fd;
-            my $filetype = `file $file 2>&1`;
-            chomp($filetype);
-            if ($filetype =~ /ASCII/) {
-                if (!open($fd, "$file")) {
-                    print "open $files failed\n";
-                    next;
-                }
-            } else {
-
-                #TODO  handle compression files
-            }
-
-            #print "--->load $file\n";
-            my $line;
-            my $historynum = 0;
-            last unless ($line = <$fd>);
-            chomp($line);
-            my $needrotate = 0;
-            my $logindex   = 0;
-            my @splitline  = split(/\s+/, $line);
-            my $timestamp;
-            my $timestampepoch;
-
-            if ($ishttplog) {
-                $splitline[3] =~ s/^\[(.+)/$1/g;
-                $timestampepoch = probe_utils->convert_to_epoch_seconds($splitline[3]);
-            } else {
-                $timestamp = join(" ", @splitline[ 0 .. 2 ]);
-                $timestampepoch = probe_utils->convert_to_epoch_seconds($timestamp, $year, $epoch_seconds_of_now);
-            }
-            if ($ref_timestamp <= $timestampepoch) {
-                $needrotate = 1;
-            } else {
-                seek($fd, 0, 2);
-                my $tail     = tell;
-                my $head     = 0;
-                my $lasttail = $tail;
-                my $i        = 0;
-                while ($head <= $tail) {
-                    my $middle = int(($tail - $head) / 2) + $head;
-                    seek($fd, $middle, 0);
-                    $line = <$fd>;
-                    $middle += length($line);
-                    last unless ($line = <$fd>);
-                    @splitline = split(/\s+/, $line);
-                    if ($ishttplog) {
-                        $splitline[3] =~ s/^\[(.+)/$1/g;
-                        $timestampepoch = probe_utils->convert_to_epoch_seconds($splitline[3]);
-                    } else {
-                        $timestamp = join(" ", @splitline[ 0 .. 2 ]);
-                        $timestampepoch = probe_utils->convert_to_epoch_seconds($timestamp, $year, $epoch_seconds_of_now);
-                    } if ($ref_timestamp == $timestampepoch) {
-                        $historynum = $middle;
-                        last;
-                    } elsif ($ref_timestamp < $timestampepoch) {
-                        $tail = $middle;
-                        last if ($tail == $lasttail);
-                        $lasttail = $tail;
-                    } else {
-                        $head = $middle;
-                    }
-                }
-                $historynum = $head unless ($historynum);
-            }
-
-            seek($fd, $historynum, 0);
-            while (<$fd>) {
-                chomp;
-                @splitline = split(/\s+/, $_);
-                if ($ishttplog) {
-                    $splitline[3] =~ s/^\[(.+)/$1/g;
-                    $timestampepoch = probe_utils->convert_to_epoch_seconds($splitline[3]);
-                } else {
-                    $timestamp = join(" ", @splitline[ 0 .. 2 ]);
-                    $timestampepoch = probe_utils->convert_to_epoch_seconds($timestamp, $year, $epoch_seconds_of_now);
-                }
-
-                if (($splitline[4] =~ /dhcpd/i && $_ =~ /$installnic/)
-                    || ($splitline[4] =~ /in.tftpd/i)
-                    || (($splitline[4] =~ /^xcat/i) || ($splitline[5] =~ /^xcat/i))
-                    || ($splitline[5] =~ /GET/ && $splitline[7] =~ /HTTP/)) {
-                    my $log = "$timestampepoch $filename$logindex $_";
-                    $logindex++;
-                    push @$bthistory_ref, $log;
-                }
-            }
-            close($fd);
-            last unless ($needrotate);
+    if (exists $node_state_ref->{$node}) {
+        probe_utils->send_msg("stdout", "d", "[$node] $log_ref->{msg}") if ($monitor);
+        push(@{ $node_state_ref->{$node}{log} }, $log_ref->{msg}) if ($debug);
+        if ($log_ref->{msg} =~ /Running postscript:/i) {
+            set_node_state($node_state_ref, $node, $::STATE_POSTSCRIPT);
+        } elsif ($log_ref->{msg} =~ /provision completed/) {
+            set_node_state($node_state_ref, $node, $::STATE_COMPLETED);
+            $node_state_ref->{$node}{done} = 1;
+            probe_utils->send_msg("stdout", "o", "[$node] provision completed") if ($monitor);
         }
     }
-
-    #sort logs depending on time
-    my @sort_ht = sort(@$bthistory_ref);
-    for ($i = 0 ; $i <= $#sort_ht ; $i++) {
-        $sort_ht[$i] =~ s/^(\d+) (\S+) (.+)/$1 $3/g;
-    }
-
-    #delete duplicate logs
-    my %count;
-    @$bthistory_ref = grep { ++$count{$_} < 2; } @sort_ht;
 }
 
-sub do_replay {
-    my $ref_timestamp = shift;
+#------------------------------------------
 
-    my $timestr = scalar(localtime($ref_timestamp));
-    print "Start logs search after '$timestr', waiting for a while.............\n";
+=head3
+    Description:
+        Dispatch log to related handler 
+    Arguments:
+        log_ref: (input attribute) the reference of hash which save one line log comes from computes.log.
+        candidate_mn_hostname_in_log_ref: (input attribute) The reference of array which save the candidate host name of MN 
+        node_state_ref: (output attribute), the reference of hash "%node_state". refer to function "init_node_state" for the structure of "%node_state"
+    Returns:
+        NULL
+=cut
 
-    my ($sec, $min, $hour, $day, $mon, $year, $wday, $yday, $isdst) = localtime(time());
-    my $epoch_seconds_of_now = time();
-    my @bthistory;
-    get_valid_logs($ref_timestamp, $year, $epoch_seconds_of_now, \@bthistory);
+#------------------------------------------
+sub dispatch_log_to_handler {
+    my $log_ref                          = shift;
+    my $candidate_mn_hostname_in_log_ref = shift;
+    my $node_state_ref                   = shift;
 
-    foreach my $line (@bthistory) {
-        $line =~ s/(\d+) (.+)/$2/g;
-        my @tmp = split(/\s+/, $line);
-        if ($tmp[4] =~ /dhcpd/i && $line =~ /$installnic/) {
-            handle_dhcp_msg("$line");
-        } elsif ($tmp[4] =~ /in.tftpd/i) {
-            handle_tftp_msg("$line");
-        } elsif (($tmp[4] =~ /^xcat/i) || ($tmp[5] =~ /^xcat/i)) {
-            if (grep(/^$tmp[3]$/, @candidate_svr_hostname_inlog)) {
-                handle_cluster_msg("$line");
-            } else {
-                handle_compute_msg("$line");
-            }
-        } elsif ($tmp[5] =~ /GET/ && $tmp[7] =~ /HTTP/) {
-            handle_http_msg("$line");
+    if ($log_ref->{label} == $::LOGLABEL_DHCPD) {
+        handle_dhcp_msg($log_ref, $node_state_ref);
+    } elsif ($log_ref->{label} == $::LOGLABEL_TFTP) {
+        handle_tftp_msg($log_ref, $node_state_ref);
+    } elsif ($log_ref->{label} == $::LOGLABEL_XCAT) {
+        if (grep(/$log_ref->{sender}/, @$candidate_mn_hostname_in_log_ref)) {
+            handle_cluster_msg($log_ref, $node_state_ref);
+        } else {
+            handle_compute_msg($log_ref, $node_state_ref);
         }
+    } elsif ($log_ref->{label} == $::LOGLABEL_HTTP) {
+        handle_http_msg($log_ref, $node_state_ref);
     }
-    &dump_history;
-    return 0;
 }
 
-#-------------------------------------
-# main process
-#-------------------------------------
-if (
-    !GetOptions("--help|h|?" => \$help,
-        "T"   => \$test,
-        "V"   => \$verbose,
-        "t"   => \$maxwaittime,
-        "r=s" => \$replaylog,
-        "n=s" => \$noderange))
-{
-    probe_utils->send_msg("$output", "f", "Invalid parameter for $program_name");
-    probe_utils->send_msg("$output", "d", "$::USAGE");
-    exit 1;
-}
+#------------------------------------------
 
-if ($help) {
-    if ($output ne "stdout") {
-        probe_utils->send_msg("$output", "d", "$::USAGE");
+=head3
+    Description:
+        Set node state in hash %node_state
+    Arguments:
+        node_state_ref: (input/output attribute), the reference of hash "%node_state". refer to function "init_node_state" for the structure of "%node_state"
+        node : (input attribute) The node name
+        newstate : (input attribute) The new state of node 
+    Returns:
+        NULL
+=cut
+
+#------------------------------------------
+sub set_node_state {
+    my $node_state_ref = shift;
+    my $node           = shift;
+    my $newstate       = shift;
+
+    if ($newstate == $::STATE_POWER_ON) {
+        push @{ $node_state_ref->{$node}{allstatehistory} }, @{ $node_state_ref->{$node}{statehistory} };
+        @{ $node_state_ref->{$node}{statehistory} } = ();
+        push @{ $node_state_ref->{$node}{statehistory} }, $newstate;
     } else {
-        print "$::USAGE";
-    }
-    exit 0;
-}
+        my $index = @{ $node_state_ref->{$node}{statehistory} } - 1;
 
-if ($test) {
-    probe_utils->send_msg("$output", "o", "Probe for OS provision process, realtime monitor of OS provision process.");
-    exit 0;
-}
-
-unless ($noderange) {
-    probe_utils->send_msg("$output", "f", "A noderange is required");
-    probe_utils->send_msg("$output", "d", "$::USAGE");
-    exit 1;
-}
-
-my $epoch_starttime = time();
-if ($replaylog) {
-    if ($replaylog =~ /(\d+)h(\d+)m/i) {
-        $epoch_starttime -= ($1 * 3600 + $2 * 60)
-    } elsif ($replaylog =~ /^(\d+)h*$/i) {
-        $epoch_starttime -= $1 * 3600;
-    } elsif ($replaylog =~ /^(\d+)m$/) {
-        $epoch_starttime -= $1 * 60;
-    } else {
-        probe_utils->send_msg("$output", "f", "Unsupported time format for history log replay");
-        print "$::USAGE";
-        exit 1;
+        if ($node_state_ref->{$node}{statehistory}->[$index] != $newstate) {
+            push @{ $node_state_ref->{$node}{statehistory} }, $newstate;
+        }
     }
 }
 
-my $msg = "All pre_defined nodes are valid";
-my $rc  = check_noderange($noderange);
-if ($rc) {
-    probe_utils->send_msg("$output", "f", $msg);
-    $rst = 1;
-} else {
-    probe_utils->send_msg("$output", "o", $msg);
-}
-
-unless ($installnic) {
-    my $masteripinsite = `tabdump site | awk -F',' '/^"master",/ { gsub(/"/, "", \$2) ; print \$2 }'`;
-    chomp($masteripinsite);
-    $installnic = `ip addr |grep -B2 $masteripinsite|awk -F" " '/mtu/{gsub(/:/,"",\$2); print \$2}'`;
-    chomp($installnic);
-    if (!$installnic) {
-        probe_utils->send_msg("$output", "f", "The value of 'master' in 'site' table is $masteripinsite, can't get corresponding network interface");
-        $rst = 1;
-    } else {
-        probe_utils->send_msg("$output", "i", "The installation network interface is $installnic");
-    }
-}
-
-exit $rst if ($rst);
-
-foreach my $node (keys %monitor_nodes) {
-    $rawdata{$node}{state} = $state_set{unknown};
-    push(@{ $rawdata{$node}{statehistory} }, $rawdata{$node}{state});
-}
-
-if ($replaylog) {
-    $rst = do_replay($epoch_starttime);
-    exit $rst;
-}
-
-$rst = do_monitor();
-
-exit $rst;
-
-

From e681e0fe99dd30c07fa3aea36ec85db9b7b183f9 Mon Sep 17 00:00:00 2001
From: hu-weihua <huweihua@cn.ibm.com>
Date: Wed, 21 Sep 2016 03:37:24 -0400
Subject: [PATCH 2/3] Modify depending on Mark's comments and fixed some bugs

---
 xCAT-probe/lib/perl/LogParse.pm    |  6 ++--
 xCAT-probe/lib/perl/probe_utils.pm | 10 +++---
 xCAT-probe/subcmds/osdeploy        | 57 +++++++++++++++++++-----------
 3 files changed, 45 insertions(+), 28 deletions(-)

diff --git a/xCAT-probe/lib/perl/LogParse.pm b/xCAT-probe/lib/perl/LogParse.pm
index 77141fc45..bd750267f 100644
--- a/xCAT-probe/lib/perl/LogParse.pm
+++ b/xCAT-probe/lib/perl/LogParse.pm
@@ -96,7 +96,7 @@ sub obtain_candidate_mn_hostname_in_log {
     if ($self->{debug}) {
         my $tmpstr = "";
         $tmpstr .= "$_ " foreach (@candidate_svr_hostname_inlog);
-        probe_utils->send_msg("stdout", "d", "The candidate MN hostname in log are $tmpstr");
+        probe_utils->send_msg("stdout", "d", "The candidate MN hostname(s) in log are $tmpstr");
     }
     return @candidate_svr_hostname_inlog;
 }
@@ -116,7 +116,7 @@ sub obtain_candidate_mn_hostname_in_log {
         $candidate_log{<label>}{type}
         <label> : The short name of log file 
         file:  The log file name, including full path
-        type: The valid type are $::LOGTYPE_HTTP and $::LOGTYPE_RSYSLOG, refer to 'probe_global_constant' for more information.
+        type: The valid types are $::LOGTYPE_HTTP and $::LOGTYPE_RSYSLOG, refer to 'probe_global_constant' for more information.
 =cut
 
 #------------------------------------------
@@ -285,7 +285,7 @@ sub obtain_one_second_logs {
                     $self->{log_open_info}->{$loglabel}{openning_file_index} = $i;
                     $self->{log_open_info}->{$loglabel}{next_read_point} = $next_read_point;
                     $self->{log_open_info}->{$loglabel}{next_start_time} = $next_start_time;
-                    $self->debuglogger("\tfound all vaild log");
+                    $self->debuglogger("\tfound all vaild logs");
                     last;
                 }
             }    #end ratate_files loop
diff --git a/xCAT-probe/lib/perl/probe_utils.pm b/xCAT-probe/lib/perl/probe_utils.pm
index d3c211de7..d62b9ef51 100644
--- a/xCAT-probe/lib/perl/probe_utils.pm
+++ b/xCAT-probe/lib/perl/probe_utils.pm
@@ -17,17 +17,17 @@ use Socket;
         The valid <flag> are debug, warning, failed, info and ok
     Arguments:
         output: where should the message be output 
-              The vaild value are:
+              The vaild values are:
               stdout : print message to STDOUT
-              a file name: print message to the file which "file name" specify 
-        tag:  the type of message, the valid value are:
+              a file name: print message to the specified "file name" 
+        tag:  the type of message, the valid values are:
               d: debug
               w: warning
               f: failed
               o: ok
               i: info
 
-              if tag is NULL, that means output message without tag
+              If tag is NULL, output message without a tag
              
         msg:  the information need to output
      Returns:
@@ -532,7 +532,7 @@ sub convert_to_epoch_seconds {
     Arguments:
         noderange : the range of node
     Returns:
-        A array which contain each node name
+        An array which contains each node name
 =cut
 
 #------------------------------------------
diff --git a/xCAT-probe/subcmds/osdeploy b/xCAT-probe/subcmds/osdeploy
index 2420b46fa..d948b3686 100755
--- a/xCAT-probe/subcmds/osdeploy
+++ b/xCAT-probe/subcmds/osdeploy
@@ -47,13 +47,13 @@ my $debug = 0;
 my $program_name = basename("$0");
 $::USAGE = "Usage:
     $program_name -h
-    $program_name -n <node_range> [-t <max_waiting_time]> [-V]
+    $program_name -n <node_range> [-t <max_waiting_time>] [-V]
     $program_name -n <node_range> -r <roll_back_duration> [-V]
 
 Description:
-    Probe operating system provision process. Support two modes, they are 'Realtime monitor' and 'Replay history'.
-    Realtime monitor: Used during doing provision, trigger 'Realtime monitor' before triggering target node reboot to do provision. This tool will monitor the provision state of node. If without specific instruction, do 'Realtime monitor' by default. 
-    Replay history: Used after finish provision, to probe the provision happened some while ago.
+    Probe operating system provision process. Supports two modes - 'Realtime monitor' and 'Replay history'. 
+    Realtime monitor: This is a default. This tool with monitor provision state of the node. Trigger 'Realtime monitor' before rebooting target node to do provisioning.
+    Replay history: Used after provisioning is finished to probe the previously completed provisioning.
 
     [NOTE] Currently, hierarchial structure is not supported.
 
@@ -61,9 +61,9 @@ Options:
     -h : Get usage information of $program_name
     -V : Output more information 
     -n : The range of nodes to be monitored or replayed.
-    -t : The maximum time to wait when doing monitor, unit is minute. default is 60.
-    -r : Trigger 'Replay history' mode. follow the duration of rolling back. Unit are 'h' (hour) or 'm' (minute)
-         Supported formats look like 3h30m (3 hours and 30 minutes ago), 2h (2 hours ago), 40m (40 minutes ago) and 3 (3 hours ago).
+    -t : The maximum time to wait when doing monitor, unit is minutes. default is 60.
+    -r : Trigger 'Replay history' mode. Follow the duration of rolling back. Units are 'h' (hour) or 'm' (minute)
+         Supported format examples: 3h30m (3 hours and 30 minutes ago), 2h (2 hours ago), 40m (40 minutes ago) and 3 (3 hours ago).
          If unit is not specified, hour will be used by default.
 ";
 
@@ -141,6 +141,8 @@ if ($rollforward_time_of_replay) {
         $start_time_of_replay -= $1 * 60;
     }
 
+    #    $start_time_of_replay = 1472437250;    ########################
+
     $rst = do_replay($noderange, $start_time_of_replay, $end_time_of_replay);
     exit $rst;
 }
@@ -171,7 +173,7 @@ sub do_pre_check {
     my @error = ();
     my $sub_func_rst = obtain_install_nic(\$installnic, \@error);
     if ($sub_func_rst) {
-        probe_utils->send_msg("stdout", "f", "Obtain install NIC in current server failed");
+        probe_utils->send_msg("stdout", "f", "Failed to obtain install NIC in current server");
         probe_utils->send_msg("stdout", "d", "$_") foreach (@error);
     } else {
         probe_utils->send_msg("stdout", "i", "The install NIC in current server is $installnic");
@@ -184,7 +186,7 @@ sub do_pre_check {
         probe_utils->send_msg("stdout", "f", "There is something wrong in node definition");
         probe_utils->send_msg("stdout", "d", "$_") foreach (@error);
     } else {
-        probe_utils->send_msg("stdout", "o", "All nodes which will be deployed are valid");
+        probe_utils->send_msg("stdout", "o", "All nodes to be deployed are valid");
     }
     $rst |= $sub_func_rst;
 
@@ -358,8 +360,14 @@ sub do_replay {
 
     my $rc = 0;
 
+    #handle INT/TERM  signal
+    my $terminal = 0;
+    $SIG{TERM} = $SIG{INT} = sub {
+        $terminal = 1;
+    };
+
     my $timestr = scalar(localtime($start_time_of_replay));
-    probe_utils->send_msg("stdout", "d", "Start to scan logs which are later than '$timestr', please waiting for a while.............");
+    probe_utils->send_msg("stdout", "d", "Starting to scan logs which are later than '$timestr', please waiting for a while.............");
 
     my %node_state;
     init_node_state($noderange, \%node_state);
@@ -375,7 +383,7 @@ sub do_replay {
         my @valid_one_second_log_set;
         my $rst = $log_parse->obtain_one_second_logs($start_time_of_replay, \@valid_one_second_log_set);
         if ($rst) {
-            probe_utils->send_msg("stdout", "d", "Obtain logs failed from log files");
+            probe_utils->send_msg("stdout", "d", "Failed to obtain logs from log files");
             $rc = 1; 
             last;
         }
@@ -385,6 +393,13 @@ sub do_replay {
         }
 
         $start_time_of_replay = $log_parse->obtain_next_second();
+    
+        # receive terminal signal from customer
+        if ($terminal) {
+            probe_utils->send_msg("stdout", "d", "Get INT or TERM signal!!!");
+            probe_utils->send_msg("stdout", "w", "Haven't scaned all valid logs, report based on the logs have been scaned");
+            last;
+        }
     }
     $log_parse->destory();
 
@@ -455,11 +470,15 @@ sub conclusion_report {
         #   that means reboot target node failed.
         # 2 if power on target node successfully and there is 'running postbootscript' in node state history, but without "installing" state,
         #   It is very possible to just do reboot process
+        # 3 There isn't reboot operation for target node during the rollback time window 
+        #   That means there isn't provision process happened 
 
         if ($start_rpower && !$power_on) {
             $failed_node{$node}{non_provision_prediction} = "Trigger target node reboot failed";
         } elsif ($start_rpower && $power_on && !$isntalling && $postbootscript) {
             $failed_node{$node}{non_provision_prediction} = "Target node just reboot from disk";
+        } elsif (! $start_rpower){
+            $failed_node{$node}{non_provision_prediction} = "Without provision process during rollback time window"; 
         } else {
             if ($stop_stage != $::STATE_COMPLETED) {
                 $failed_node{$node}{provision_stop_point} = $stop_stage;
@@ -471,9 +490,9 @@ sub conclusion_report {
     if (%failed_node) {
         my $failed_node_num = keys %failed_node;
         if ($failed_node_num > 1) {
-            probe_utils->send_msg("stdout", "d", "There are $failed_node_num node provision failed");
+            probe_utils->send_msg("stdout", "d", "There are $failed_node_num node provision failures");
         } elsif ($failed_node_num == 1) {
-            probe_utils->send_msg("stdout", "d", "There is $failed_node_num node provision failed");
+            probe_utils->send_msg("stdout", "d", "There is $failed_node_num node provision failures");
         }
 
         foreach my $node (keys %failed_node) {
@@ -484,7 +503,7 @@ sub conclusion_report {
             }
         }
     } else {
-        probe_utils->send_msg("stdout", "o", "All nodes provision successfully");
+        probe_utils->send_msg("stdout", "o", "All nodes provisioned successfully");
     }
     return 0;
 }
@@ -545,7 +564,7 @@ Start capturing every message during OS provision process....
                 }
             }
         } else {
-            probe_utils->send_msg("stdout", "f", "There isn't any vaild log file to be scaned");
+            probe_utils->send_msg("stdout", "f", "There are no valid log files to be scanned");
             $rst = 1;
         }
 
@@ -764,8 +783,6 @@ sub handle_dhcp_msg {
 
             if ($macmap{$mac}{"ip"} != "NOIP" and $macmap{$mac}{"ip"} != $ip) {
                 my $warn_msg = "The ip($ip) assigned to $mac via DHCP is different from the ip($macmap{$mac}{'ip'}) in node definition.";
-
-                #probe_utils->send_msg("stdout", "w", "$warn_msg") if ($monitor);
                 probe_utils->send_msg("stdout", "w", "$warn_msg");
             }
 
@@ -800,7 +817,7 @@ sub handle_tftp_msg {
             probe_utils->send_msg("stdout", "d", "[$ipnodemap{$ip}] $record") if ($monitor);
             push(@{ $node_state_ref->{ $ipnodemap{$ip} }{log} }, $log_ref->{msg}) if ($debug);
 
-            if ($file =~ /xcat\/xnba.*/i or $file =~ /\/boot\/grub2\/grub2\-/i or $file =~ /\yb\/node\/yaboot\-/i) {
+            if ($file =~ /xcat\/xnba.*/i or $file =~ /\/boot\/grub2\/powerpc-ieee1275\//i or $file =~ /\/yb\/node\/yaboot\-/i) {
                 set_node_state($node_state_ref, $ipnodemap{$ip}, $::STATE_BOOTLODER);
             } elsif ($file =~ /vmlinuz|inst64|linux/) {
                 set_node_state($node_state_ref, $ipnodemap{$ip}, $::STATE_KERNEL);
@@ -843,9 +860,9 @@ sub handle_http_msg {
                 set_node_state($node_state_ref, $ipnodemap{$ip}, $::STATE_KERNEL);
             } elsif ($file =~ /initrd/i and $file =~ /osimage/i) {
                 set_node_state($node_state_ref, $ipnodemap{$ip}, $::STATE_INITRD);
-            } elsif ($file =~ /\/install\/autoinst\/*/i) {
+            } elsif (($file =~ /^\/install\/autoinst\//i) and ($file !~ /getinstdisk$/i) and ($file !~ /\.pre$/i) and ($file !~ /\.post$/i)) {
                 set_node_state($node_state_ref, $ipnodemap{$ip}, $::STATE_KICKSTART);
-            } elsif ($file =~ /\.deb/i or ($file =~ /\/install\// and $file =~ /\/repodata\//i)) {
+            } elsif ($file =~ /\.deb$/i or $file =~ /\/Packages\/.+\.rpm$/ or $file =~ /\/suse\/noarch\/.+\.rpm$/i) {
                 set_node_state($node_state_ref, $ipnodemap{$ip}, $::STATE_INSTALLRPM);
             }
         }

From 9abbcd4fbcb3e6f081f9945645fe972e5bb44a02 Mon Sep 17 00:00:00 2001
From: hu-weihua <huweihua@cn.ibm.com>
Date: Fri, 23 Sep 2016 02:36:52 -0400
Subject: [PATCH 3/3] modify depending on comments

---
 xCAT-probe/lib/perl/LogParse.pm    | 16 ++++++----------
 xCAT-probe/lib/perl/probe_utils.pm |  2 +-
 xCAT-probe/subcmds/osdeploy        | 23 +++++++++++++++--------
 3 files changed, 22 insertions(+), 19 deletions(-)

diff --git a/xCAT-probe/lib/perl/LogParse.pm b/xCAT-probe/lib/perl/LogParse.pm
index bd750267f..d9e4a9e6d 100644
--- a/xCAT-probe/lib/perl/LogParse.pm
+++ b/xCAT-probe/lib/perl/LogParse.pm
@@ -62,7 +62,7 @@ sub new {
         my $logfiledir = "/tmp/xcatprobedebug/";
         mkpath("$logfiledir") unless (-d "$logfiledir");
         $self->{debuglogpath} = $logfiledir;
-        $self->{debuglogfd};
+        $self->{debuglogfd} = undef;
     }
 
     bless($self, ref($class) || $class);
@@ -94,8 +94,7 @@ sub obtain_candidate_mn_hostname_in_log {
     push(@candidate_svr_hostname_inlog, $svr_hostname_short);
     push(@candidate_svr_hostname_inlog, "$svr_hostname_short.$svr_hostname_domain");
     if ($self->{debug}) {
-        my $tmpstr = "";
-        $tmpstr .= "$_ " foreach (@candidate_svr_hostname_inlog);
+        my $tmpstr = join(" ", @candidate_svr_hostname_inlog);
         probe_utils->send_msg("stdout", "d", "The candidate MN hostname(s) in log are $tmpstr");
     }
     return @candidate_svr_hostname_inlog;
@@ -180,9 +179,7 @@ sub obtain_one_second_logs {
 
     if ($self->{debug}) {
         my $logfile = "$self->{debuglogpath}/obtain_one_second_logs.$the_time_to_load";
-        my $fd;
-        open($fd, "> $logfile");
-        $self->{debuglogfd} = $fd;
+        open($self->{debuglogfd}, "> $logfile");
     }
 
     #read log for the first time
@@ -554,13 +551,13 @@ sub convert_to_epoch_seconds {
         $yday = $self->{current_ref_year};
         $epoch_seconds = timelocal($s, $m, $h, $dday, $monthsmap{$mday}, $yday);
         if ($epoch_seconds > $self->{current_ref_time}) {
-            $yday -= 1;
+            --$yday;
             $epoch_seconds = timelocal($s, $m, $h, $dday, $monthsmap{$mday}, $yday);
         }
 
         # The time format looks like "15/Aug/2016:01:10:24"
     } elsif ($timestr =~ /(\d+)\/(\w+)\/(\d+):(\d+):(\d+):(\d+)/) {
-        $epoch_seconds = timelocal($6, $5, $4, $1, $monthsmap{$2}, ($3 - 1900));
+        $epoch_seconds = timelocal($6, $5, $4, $1, $monthsmap{$2}, $3);
     }
     return $epoch_seconds;
 }
@@ -676,8 +673,7 @@ sub debuglogger {
     my $self = shift;
     my $msg  = shift;
     if ($self->{debug}) {
-        my $fd = $self->{debuglogfd};
-        print $fd "$msg\n";
+        print {$self->{debuglogfd}} "$msg\n";
     }
 }
 1;
diff --git a/xCAT-probe/lib/perl/probe_utils.pm b/xCAT-probe/lib/perl/probe_utils.pm
index d62b9ef51..e99845562 100644
--- a/xCAT-probe/lib/perl/probe_utils.pm
+++ b/xCAT-probe/lib/perl/probe_utils.pm
@@ -540,7 +540,7 @@ sub parse_node_range {
     my $noderange = shift;
     $noderange= shift if (($noderange) && ($noderange =~ /probe_utils/));
     my @nodeslist = `nodels $noderange`;
-    chomp($_) foreach(@nodeslist);
+    chomp @nodeslist;
     return @nodeslist;
 }
 1;
diff --git a/xCAT-probe/subcmds/osdeploy b/xCAT-probe/subcmds/osdeploy
index d948b3686..e7c78a1d3 100755
--- a/xCAT-probe/subcmds/osdeploy
+++ b/xCAT-probe/subcmds/osdeploy
@@ -423,7 +423,7 @@ sub do_replay {
 sub conclusion_report {
     my $node_state_ref = shift;
 
-    probe_utils->send_msg("stdout", "", "==================conclusion_report=================");
+    probe_utils->send_msg("stdout", "", "==================osdeploy_probe_report=================");
 
     if ($debug) {
         print "---->the result of %node_state<------\n";
@@ -431,17 +431,22 @@ sub conclusion_report {
     }
 
     if ($verbose) {
-        probe_utils->send_msg("stdout", "d", "----------node history state----------");
+        probe_utils->send_msg("stdout", "d", "----------node state history----------");
         foreach my $node (keys %$node_state_ref) {
             my $allhistorystate;
             my $historystate;
+            probe_utils->send_msg("stdout", "d", "[$node]:");
             if (@{ $node_state_ref->{$node}{allstatehistory} }) {
-                $allhistorystate .= "$::STATE_DESC{$_} " foreach (@{ $node_state_ref->{$node}{allstatehistory} });
-                probe_utils->send_msg("stdout", "d", "[$node] the history before last loop provision: $allhistorystate");
+                $allhistorystate .= "$::STATE_DESC{$_}=>" foreach (@{ $node_state_ref->{$node}{allstatehistory} });
+                $allhistorystate =~ s/=>$//g;
+                probe_utils->send_msg("stdout", "d", "Setps executed prior to last provisioning attempt:");
+                probe_utils->send_msg("stdout", "d", "$allhistorystate");
             }
 
-            $historystate .= "$::STATE_DESC{$_} " foreach (@{ $node_state_ref->{$node}{statehistory} });
-            probe_utils->send_msg("stdout", "d", "[$node] last loop provision history: $historystate");
+            $historystate .= "$::STATE_DESC{$_}=>" foreach (@{ $node_state_ref->{$node}{statehistory} });
+            $historystate =~ s/=>$//g;
+            probe_utils->send_msg("stdout", "d", "Steps executed for last provisioning attempt:");
+            probe_utils->send_msg("stdout", "d", "$historystate");
         }
         probe_utils->send_msg("stdout", "d", "--------------------------------------");
     }
@@ -503,7 +508,7 @@ sub conclusion_report {
             }
         }
     } else {
-        probe_utils->send_msg("stdout", "o", "All nodes provisioned successfully");
+        probe_utils->send_msg("stdout", "o", "All nodes provision successfully");
     }
     return 0;
 }
@@ -919,7 +924,9 @@ sub handle_cluster_msg {
                 set_node_state($node_state_ref, $node, $::STATE_POWERINGON);
             } elsif ($status eq "booting") {
                 set_node_state($node_state_ref, $node, $::STATE_BOOTING);
-            }
+            } elsif ($status eq "failed") {
+                $node_state_ref->{$node}{done} = 1;
+            } 
         }
     }
 }