2
0
mirror of https://github.com/xcat2/xcat-core.git synced 2025-05-28 06:23:26 +00:00

xcatprobe osdeploy for monitoring os provision

This commit is contained in:
XuWei 2016-07-04 02:16:32 -04:00
parent 9988ee9ee1
commit 84a1793a59

390
xCAT-probe/subcmds/osdeploy Executable file
View File

@ -0,0 +1,390 @@
#! /usr/bin/perl
# IBM(c) 2016 EPL license http://www.eclipse.org/legal/epl-v10.html
BEGIN { $::XCATROOT = $ENV{'XCATROOT'} ? $ENV{'XCATROOT'} : -d '/opt/xcat' ? '/opt/xcat' : '/usr'; }
use lib "$::XCATROOT/probe/lib/perl";
use probe_utils;
use xCAT::NetworkUtils;
use File::Basename;
use IO::Select;
use Getopt::Long qw(:config no_ignore_case);
my $program_name = basename("$0");
my $help;
my $test;
my $output = "stdout";
my $verbose = 0;
my $rst = 0;
my $noderange;
my %rawdata;
my %ipnodemap;
my %macmap;
my $terminal = 0;
my %monitor_nodes;
$::USAGE = "Usage:
$program_name -h
$program_name -T
$program_name [-V]
$program_name -n <node_range>
Description:
Do probe for os provision process, realtime monitor of os provision process.
Options:
-h : Get usage information of $program_name
-T : To verify if $program_name can work, reserve option for probe framework
-V : Output more information for debug
-n : The range of monitored node.
";
sub check_noderange{
my $node_range = shift;
my @cmdoutput = `lsdef $node_range 2>&1`;
my $rst = 0;
my $currentnode = "";
my %nodecheckrst;
my $ip;
foreach (@cmdoutput) {
chomp($_);
$_ =~ s/^\s+|\s+$//g;
if ($_ =~ /^Error: Could not find an object named '(\w+)' .+/i) {
$currentnode = $1;
$nodecheckrst{$currentnode}{"error"} = "Could not find node definition";
$rst = 1;
} elsif ($_ =~ /^\s*Object name: (\w+)/i) {
$monitor_nodes{$1} = 0;
$currentnode = $1;
} elsif ($_ =~ /^ip=(.+)/i) {
$ip = $1;
} elsif ($_ =~ /^mac=(.+)\|(.+)!\*NOIP\*/i) {
$macmap{$1}{"ip"} = $ip;
$macmap{$2}{"ip"} = $ip;
$macmap{$1}{"node"} = $currentnode;
$macmap{$2}{"node"} = $currentnode;
} elsif ($_ =~ /^mac=(.+)\|(.+)/i) {
$macmap{$1}{"ip"} = $ip;
$macmap{$2}{"ip"} = $ip;
$macmap{$1}{"node"} = $currentnode;
$macmap{$2}{"node"} = $currentnode;
} elsif ($_ =~ /^mac=(.+)/i) {
$macmap{$1}{"ip"} = $ip;
$macmap{$1}{"node"} = $currentnode;
}
}
foreach my $node (keys %nodecheckrst) {
probe_utils->send_msg("$output", "d", "$node : $nodecheckrst{$node}{error}") if(exists($nodecheckrst{$node}{error}));
}
return $rst;
}
sub handle_dhcp_msg {
my $msg = shift;
if ($msg !~ /unknown lease/ && $msg !~ /ignored/ && $msg =~ /.+DHCPREQUEST\s+for\s+(.+)\s\((.+)\)\s+from\s+(.+)\s+via\s+(.+)/) {
my $ip = $1;
my $server = $2;
my $mac = $3;
my $nic = $4;
my $record = "Receive DHCPREQUEST from $mac for $ip via $nic";
if (exists $macmap{$mac}) {
my $node = $macmap{$mac}{"node"};
probe_utils->send_msg("$output", "d", "$record");
push(@{ $rawdata{$node}{"history"} }, $record);
if ($macmap{$mac}{"ip"} != $ip) {
my $warn_msg = "The ip of $mac from DHCP $ip is different with definition $macmap{$mac}{'ip'}.";
probe_utils->send_msg("$output", "w", "$warn_msg");
if (exists($rawdata{$node})) {
push(@{ $rawdata{$node}{"history"} }, $warn_msg);
}
}
}
} elsif ($msg =~ /.+DHCPACK\s+on\s+(.+)\s+to\s+(.+)\s+via\s+(.+)/) {
my $ip = $1;
my $mac = $2;
my $nic = $3;
my $record = "Send DHCPACK on $ip back to $mac via $nic";
if (exists $macmap{$mac}) {
my $node = $macmap{$mac}{"node"};
if (exists($rawdata{$node})) {
probe_utils->send_msg("$output", "d", "$record");
push(@{ $rawdata{$node}{"history"} }, $record);
$ipnodemap{$ip} = $node;
}
}
}
return 0;
}
sub handle_http_msg {
my $msg = shift;
if ($msg =~ /(\d+\.\d+.\d+.\d+)\s.+GET\s+(.+)\s+HTTP.+/) {
my $ip = $1;
my $file = $2;
my $record = "Via HTTP $ip download $file";
if (exists($rawdata{"$ipnodemap{$ip}"})) {
probe_utils->send_msg("$output", "d", "$record");
push(@{ $rawdata{ $ipnodemap{$ip} }{"history"} }, $record);
}
}
return 0;
}
sub handle_cluster_msg {
my $line = shift;
my $sender = "";
my $node = "";
my $msg;
my $status;
if ($line =~ /.+\d+:\d+:\d+\s+(.+)\s+(xcat.+)/i) {
$sender = $1;
$msg = $2;
if (!xCAT::NetworkUtils->isIpaddr($sender)) {
$node = $sender;
} else {
$node = $ipnodemap{$sender};
}
if ($node ne "" && exists($rawdata{$node})) {
my $record = "Recv from $node : $msg";
probe_utils->send_msg("$output", "d", "$record");
push(@{ $rawdata{ $node }{"history"} }, $record);
}
}
if ($line =~ /.+\s+xcat:\s+(.+)\s+status:\s+(.+)\s+statustime:\s(.+)/) {
$node = $1;
$status = $2;
if (exists($rawdata{$node})) {
my $record = "Recv from $node : xcat: status is $status";
probe_utils->send_msg("$output", "d", "$record");
push(@{ $rawdata{ $node }{"history"} }, $record);
}
if (exists($rawdata{$node}) and ($status eq "booted")) {
$monitor_nodes{$node} = 1 if (defined($monitor_nodes{$node}));
probe_utils->send_msg("$output", "o", "Node $node has finished it's os provision process");
}
}
return 0;
}
sub all_monitor_node_done {
my $done = 1;
foreach my $node (keys %monitor_nodes) {
if ($monitor_nodes{$node} == 0) {
$done = 0;
last;
}
}
return $done;
}
sub dump_history {
my $title = "
=============================================================
= The summary of os provision:
=============================================================
";
print "$title\n";
foreach $node (keys %rawdata) {
my $line_num = 0;
my $http_num = 0;
my $length_http;
for (my $i = @{ $rawdata{$node}{"history"} }; $i >=0; $i--) {
if (${ $rawdata{$node}{"history"} }[$i] =~ /Via HTTP/) {
$length_http = $i;
last;
}
}
foreach my $line (@{ $rawdata{$node}{"history"} }) {
if ($line =~ /Via HTTP/)
{
if (($http_num <= 4) or ($length_http - $line_num <= 4)){
probe_utils->send_msg("$output", "d", "\t$line");
} else {
probe_utils->send_msg("$output", "d", "\t......") if ($http_num ==5);
}
$http_num++;
} else {
probe_utils->send_msg("$output", "d", "\t$line");
}
$line_num++;
}
}
}
sub do_monitor{
$SIG{TERM} = $SIG{INT} = sub {
$terminal = 1;
};
my $msg = "All pre_defined nodes are valid";
my $rc = check_noderange($noderange);
if ($rc) {
probe_utils->send_msg("$output", "f", $msg);
$rst = 1;
unless (%monitor_nodes) {
return $rst;
}
} else {
probe_utils->send_msg("$output", "o", $msg);
}
if (!$nics) {
my $masteripinsite = `tabdump site | awk -F',' '/master/ { gsub(/"/, "", \$2) ; print \$2 }'`;
chomp($masteripinsite);
$nics = `ip addr |grep -B2 $masteripinsite|awk -F" " '/mtu/{gsub(/:/,"",\$2); print \$2}'`;
chomp($nics);
if (!$nics) {
probe_utils->send_msg("$output", "f", "The value of master in site table is $masteripinsite, can't get corresponding network interface");
return 1;
}
}
my $rst = 0;
my $startline = "-------------------------------------------------------------
Start to capture every message during os provision process......
-------------------------------------------------------------
";
probe_utils->send_msg("$output", "d", "$startline");
my $varlogmsg = "/var/log/messages";
my $clusterlog = "/var/log/xcat/cluster.log";
my $httplog;
if (-e "/var/log/httpd/access_log") {
$httplog = "/var/log/httpd/access_log";
} elsif (-e "/var/log/apache2/access_log") {
$httplog = "/var/log/apache2/access_log";
} elsif (-e "/var/log/apache2/access.log") {
$httplog = "/var/log/apache2/access.log";
}
my $varlogpid;
my $clusterpid;
my $httppid;
if (!($varlogpid = open(VARLOGMSGFILE, "tail -f $varlogmsg 2>&1 |"))) {
probe_utils->send_msg("$output", "f", "Can't open $varlogmsg to get logs");
$rst = 1;
last;
}
if (!($clusterpid = open(CLUSTERLOGFILE, "tail -f $clusterlog 2>&1 |"))) {
probe_utils->send_msg("$output", "f", "Can't open $clusterlog to get logs");
$rst = 1;
last;
}
if (!($httppid = open(HTTPLOGFILE, "tail -f $httplog 2>&1 |"))) {
probe_utils->send_msg("$output", "f", "Can't open $httplog to get logs");
$rst = 1;
last;
}
my $select = new IO::Select;
$select->add(\*VARLOGMSGFILE);
$select->add(\*CLUSTERLOGFILE);
$select->add(\*HTTPLOGFILE);
$| = 1;
my $line = "";
my @hdls;
my $hdl;
my $oldlines = 10;
my $varlogmsgcnt = 0;
my $clusterlogcnt = 0;
my $httplogcnt = 0;
for (; ;){
if (@hdls = $select->can_read(0)) {
foreach $hdl (@hdls) {
if ($hdl == \*VARLOGMSGFILE) {
chomp($line = <VARLOGMSGFILE>);
++$varlogmsgcnt;
last if ($varlogmsgcnt <= $oldlines);
my @tmp = split(/\s+/, $line);
if ($tmp[4] =~ /dhcpd:/i && $line =~ /$nics/) {
handle_dhcp_msg("$line");
}
} elsif ($hdl == \*CLUSTERLOGFILE) {
chomp($line = <CLUSTERLOGFILE>);
++$clusterlogcnt;
last if ($clusterlogcnt <= $oldlines);
handle_cluster_msg("$line");
} elsif ($hdl == \*HTTPLOGFILE) {
chomp($line = <HTTPLOGFILE>);
++$httplogcnt;
last if ($httplogcnt <= $oldlines);
handle_http_msg("$line");
}
}
}
if ($terminal || (%monitor_nodes && all_monitor_node_done())) {
if ($terminal) {
probe_utils->send_msg("$output", "d", "Get INT or TERM signal from STDIN");
} else {
probe_utils->send_msg("$output", "o", "All nodes need to monitor have finished os provision process");
}
last;
} sleep 0.01;
}
&dump_history;
kill 'INT', $varlogpid if ($varlogpid);
kill 'INT', $clusterpid if ($clusterpid);
kill 'INT', $httppid if ($httppid);
close(VARLOGMSGFILE) if (VARLOGMSGFILE);
close(CLUSTERLOGFILE) if(CLUSTERLOGFILE);
close(HTTPLOGFILE) if(HTTPLOGFILE);
return $rst;
}
#-------------------------------------
# main process
#-------------------------------------
if (
!GetOptions("--help|h|?" => \$help,
"T" => \$test,
"V" => \$verbose,
"n=s" => \$noderange))
{
probe_utils->send_msg("$output", "f", "Invalid parameter for $program_name");
probe_utils->send_msg("$output", "d", "$::USAGE");
exit 1;
}
if ($help) {
if ($output ne "stdout") {
probe_utils->send_msg("$output", "d", "$::USAGE");
} else {
print "$::USAGE";
}
exit 0;
}
if ($test) {
probe_utils->send_msg("$output", "o", "Do probe for os provision process, realtime monitor of os provision process.");
exit 0;
}
$rst = do_monitor();
exit $rst;