diff --git a/xCAT-probe/subcmds/osdeploy b/xCAT-probe/subcmds/osdeploy new file mode 100755 index 000000000..57b0f8470 --- /dev/null +++ b/xCAT-probe/subcmds/osdeploy @@ -0,0 +1,390 @@ +#! /usr/bin/perl +# IBM(c) 2016 EPL license http://www.eclipse.org/legal/epl-v10.html + +BEGIN { $::XCATROOT = $ENV{'XCATROOT'} ? $ENV{'XCATROOT'} : -d '/opt/xcat' ? '/opt/xcat' : '/usr'; } + +use lib "$::XCATROOT/probe/lib/perl"; + +use probe_utils; +use xCAT::NetworkUtils; +use File::Basename; +use IO::Select; +use Getopt::Long qw(:config no_ignore_case); + +my $program_name = basename("$0"); +my $help; +my $test; +my $output = "stdout"; +my $verbose = 0; +my $rst = 0; +my $noderange; +my %rawdata; +my %ipnodemap; +my %macmap; +my $terminal = 0; +my %monitor_nodes; + +$::USAGE = "Usage: + $program_name -h + $program_name -T + $program_name [-V] + $program_name -n + +Description: + Do probe for os provision process, realtime monitor of os provision process. + +Options: + -h : Get usage information of $program_name + -T : To verify if $program_name can work, reserve option for probe framework + -V : Output more information for debug + -n : The range of monitored node. +"; + +sub check_noderange{ + my $node_range = shift; + my @cmdoutput = `lsdef $node_range 2>&1`; + my $rst = 0; + my $currentnode = ""; + my %nodecheckrst; + my $ip; + + foreach (@cmdoutput) { + chomp($_); + $_ =~ s/^\s+|\s+$//g; + if ($_ =~ /^Error: Could not find an object named '(\w+)' .+/i) { + $currentnode = $1; + $nodecheckrst{$currentnode}{"error"} = "Could not find node definition"; + $rst = 1; + } elsif ($_ =~ /^\s*Object name: (\w+)/i) { + $monitor_nodes{$1} = 0; + $currentnode = $1; + } elsif ($_ =~ /^ip=(.+)/i) { + $ip = $1; + } elsif ($_ =~ /^mac=(.+)\|(.+)!\*NOIP\*/i) { + $macmap{$1}{"ip"} = $ip; + $macmap{$2}{"ip"} = $ip; + $macmap{$1}{"node"} = $currentnode; + $macmap{$2}{"node"} = $currentnode; + } elsif ($_ =~ /^mac=(.+)\|(.+)/i) { + $macmap{$1}{"ip"} = $ip; + $macmap{$2}{"ip"} = $ip; + $macmap{$1}{"node"} = $currentnode; + $macmap{$2}{"node"} = $currentnode; + } elsif ($_ =~ /^mac=(.+)/i) { + $macmap{$1}{"ip"} = $ip; + $macmap{$1}{"node"} = $currentnode; + } + } + + foreach my $node (keys %nodecheckrst) { + probe_utils->send_msg("$output", "d", "$node : $nodecheckrst{$node}{error}") if(exists($nodecheckrst{$node}{error})); + } + return $rst; +} + +sub handle_dhcp_msg { + my $msg = shift; + + if ($msg !~ /unknown lease/ && $msg !~ /ignored/ && $msg =~ /.+DHCPREQUEST\s+for\s+(.+)\s\((.+)\)\s+from\s+(.+)\s+via\s+(.+)/) { + my $ip = $1; + my $server = $2; + my $mac = $3; + my $nic = $4; + my $record = "Receive DHCPREQUEST from $mac for $ip via $nic"; + + if (exists $macmap{$mac}) { + my $node = $macmap{$mac}{"node"}; + probe_utils->send_msg("$output", "d", "$record"); + push(@{ $rawdata{$node}{"history"} }, $record); + + if ($macmap{$mac}{"ip"} != $ip) { + my $warn_msg = "The ip of $mac from DHCP $ip is different with definition $macmap{$mac}{'ip'}."; + probe_utils->send_msg("$output", "w", "$warn_msg"); + if (exists($rawdata{$node})) { + push(@{ $rawdata{$node}{"history"} }, $warn_msg); + } + } + } + } elsif ($msg =~ /.+DHCPACK\s+on\s+(.+)\s+to\s+(.+)\s+via\s+(.+)/) { + my $ip = $1; + my $mac = $2; + my $nic = $3; + my $record = "Send DHCPACK on $ip back to $mac via $nic"; + + if (exists $macmap{$mac}) { + my $node = $macmap{$mac}{"node"}; + if (exists($rawdata{$node})) { + probe_utils->send_msg("$output", "d", "$record"); + push(@{ $rawdata{$node}{"history"} }, $record); + $ipnodemap{$ip} = $node; + } + } + } + return 0; +} + +sub handle_http_msg { + my $msg = shift; + if ($msg =~ /(\d+\.\d+.\d+.\d+)\s.+GET\s+(.+)\s+HTTP.+/) { + my $ip = $1; + my $file = $2; + my $record = "Via HTTP $ip download $file"; + + if (exists($rawdata{"$ipnodemap{$ip}"})) { + probe_utils->send_msg("$output", "d", "$record"); + push(@{ $rawdata{ $ipnodemap{$ip} }{"history"} }, $record); + } + } + return 0; +} + +sub handle_cluster_msg { + my $line = shift; + my $sender = ""; + my $node = ""; + my $msg; + my $status; + + if ($line =~ /.+\d+:\d+:\d+\s+(.+)\s+(xcat.+)/i) { + $sender = $1; + $msg = $2; + + if (!xCAT::NetworkUtils->isIpaddr($sender)) { + $node = $sender; + } else { + $node = $ipnodemap{$sender}; + } + if ($node ne "" && exists($rawdata{$node})) { + my $record = "Recv from $node : $msg"; + probe_utils->send_msg("$output", "d", "$record"); + push(@{ $rawdata{ $node }{"history"} }, $record); + } + } + + if ($line =~ /.+\s+xcat:\s+(.+)\s+status:\s+(.+)\s+statustime:\s(.+)/) { + $node = $1; + $status = $2; + + if (exists($rawdata{$node})) { + my $record = "Recv from $node : xcat: status is $status"; + probe_utils->send_msg("$output", "d", "$record"); + push(@{ $rawdata{ $node }{"history"} }, $record); + } + + if (exists($rawdata{$node}) and ($status eq "booted")) { + $monitor_nodes{$node} = 1 if (defined($monitor_nodes{$node})); + probe_utils->send_msg("$output", "o", "Node $node has finished it's os provision process"); + } + } + return 0; +} + +sub all_monitor_node_done { + my $done = 1; + foreach my $node (keys %monitor_nodes) { + if ($monitor_nodes{$node} == 0) { + $done = 0; + last; + } + } + + return $done; +} + +sub dump_history { + + my $title = " +============================================================= += The summary of os provision: +============================================================= +"; + print "$title\n"; + + foreach $node (keys %rawdata) { + my $line_num = 0; + my $http_num = 0; + my $length_http; + for (my $i = @{ $rawdata{$node}{"history"} }; $i >=0; $i--) { + if (${ $rawdata{$node}{"history"} }[$i] =~ /Via HTTP/) { + $length_http = $i; + last; + } + } + foreach my $line (@{ $rawdata{$node}{"history"} }) { + if ($line =~ /Via HTTP/) + { + if (($http_num <= 4) or ($length_http - $line_num <= 4)){ + probe_utils->send_msg("$output", "d", "\t$line"); + } else { + probe_utils->send_msg("$output", "d", "\t......") if ($http_num ==5); + } + $http_num++; + } else { + probe_utils->send_msg("$output", "d", "\t$line"); + } + $line_num++; + } + } +} + +sub do_monitor{ + $SIG{TERM} = $SIG{INT} = sub { + $terminal = 1; + }; + + my $msg = "All pre_defined nodes are valid"; + my $rc = check_noderange($noderange); + if ($rc) { + probe_utils->send_msg("$output", "f", $msg); + $rst = 1; + unless (%monitor_nodes) { + return $rst; + } + } else { + probe_utils->send_msg("$output", "o", $msg); + } + + if (!$nics) { + my $masteripinsite = `tabdump site | awk -F',' '/master/ { gsub(/"/, "", \$2) ; print \$2 }'`; + chomp($masteripinsite); + $nics = `ip addr |grep -B2 $masteripinsite|awk -F" " '/mtu/{gsub(/:/,"",\$2); print \$2}'`; + chomp($nics); + if (!$nics) { + probe_utils->send_msg("$output", "f", "The value of master in site table is $masteripinsite, can't get corresponding network interface"); + return 1; + } + } + + my $rst = 0; + my $startline = "------------------------------------------------------------- +Start to capture every message during os provision process...... +------------------------------------------------------------- +"; + + probe_utils->send_msg("$output", "d", "$startline"); + + my $varlogmsg = "/var/log/messages"; + my $clusterlog = "/var/log/xcat/cluster.log"; + + my $httplog; + if (-e "/var/log/httpd/access_log") { + $httplog = "/var/log/httpd/access_log"; + } elsif (-e "/var/log/apache2/access_log") { + $httplog = "/var/log/apache2/access_log"; + } elsif (-e "/var/log/apache2/access.log") { + $httplog = "/var/log/apache2/access.log"; + } + + my $varlogpid; + my $clusterpid; + my $httppid; + + if (!($varlogpid = open(VARLOGMSGFILE, "tail -f $varlogmsg 2>&1 |"))) { + probe_utils->send_msg("$output", "f", "Can't open $varlogmsg to get logs"); + $rst = 1; + last; + } + if (!($clusterpid = open(CLUSTERLOGFILE, "tail -f $clusterlog 2>&1 |"))) { + probe_utils->send_msg("$output", "f", "Can't open $clusterlog to get logs"); + $rst = 1; + last; + } + if (!($httppid = open(HTTPLOGFILE, "tail -f $httplog 2>&1 |"))) { + probe_utils->send_msg("$output", "f", "Can't open $httplog to get logs"); + $rst = 1; + last; + } + + my $select = new IO::Select; + $select->add(\*VARLOGMSGFILE); + $select->add(\*CLUSTERLOGFILE); + $select->add(\*HTTPLOGFILE); + $| = 1; + + my $line = ""; + my @hdls; + my $hdl; + my $oldlines = 10; + my $varlogmsgcnt = 0; + my $clusterlogcnt = 0; + my $httplogcnt = 0; + + for (; ;){ + if (@hdls = $select->can_read(0)) { + foreach $hdl (@hdls) { + if ($hdl == \*VARLOGMSGFILE) { + chomp($line = ); + ++$varlogmsgcnt; + last if ($varlogmsgcnt <= $oldlines); + my @tmp = split(/\s+/, $line); + if ($tmp[4] =~ /dhcpd:/i && $line =~ /$nics/) { + handle_dhcp_msg("$line"); + } + } elsif ($hdl == \*CLUSTERLOGFILE) { + chomp($line = ); + ++$clusterlogcnt; + last if ($clusterlogcnt <= $oldlines); + handle_cluster_msg("$line"); + } elsif ($hdl == \*HTTPLOGFILE) { + chomp($line = ); + ++$httplogcnt; + last if ($httplogcnt <= $oldlines); + handle_http_msg("$line"); + } + } + } + + if ($terminal || (%monitor_nodes && all_monitor_node_done())) { + if ($terminal) { + probe_utils->send_msg("$output", "d", "Get INT or TERM signal from STDIN"); + } else { + probe_utils->send_msg("$output", "o", "All nodes need to monitor have finished os provision process"); + } + last; + } sleep 0.01; + } + &dump_history; + + kill 'INT', $varlogpid if ($varlogpid); + kill 'INT', $clusterpid if ($clusterpid); + kill 'INT', $httppid if ($httppid); + close(VARLOGMSGFILE) if (VARLOGMSGFILE); + close(CLUSTERLOGFILE) if(CLUSTERLOGFILE); + close(HTTPLOGFILE) if(HTTPLOGFILE); + + return $rst; +} + +#------------------------------------- +# main process +#------------------------------------- +if ( + !GetOptions("--help|h|?" => \$help, + "T" => \$test, + "V" => \$verbose, + "n=s" => \$noderange)) +{ + probe_utils->send_msg("$output", "f", "Invalid parameter for $program_name"); + probe_utils->send_msg("$output", "d", "$::USAGE"); + exit 1; +} + +if ($help) { + if ($output ne "stdout") { + probe_utils->send_msg("$output", "d", "$::USAGE"); + } else { + print "$::USAGE"; + } + exit 0; +} + +if ($test) { + probe_utils->send_msg("$output", "o", "Do probe for os provision process, realtime monitor of os provision process."); + exit 0; +} + +$rst = do_monitor(); + +exit $rst; + +