From 90bb2a2639138ae1f19bc2b4b836328ed573dccc Mon Sep 17 00:00:00 2001 From: XuWei Date: Thu, 20 Oct 2016 03:23:22 -0400 Subject: [PATCH] enhance xcatprobe discovery sub command --- xCAT-probe/lib/perl/LogParse.pm | 22 +- xCAT-probe/lib/perl/probe_global_constant.pm | 36 +- xCAT-probe/subcmds/discovery | 2130 ++++++++++-------- 3 files changed, 1296 insertions(+), 892 deletions(-) diff --git a/xCAT-probe/lib/perl/LogParse.pm b/xCAT-probe/lib/perl/LogParse.pm index 85b0c6ca5..a7c6e5379 100644 --- a/xCAT-probe/lib/perl/LogParse.pm +++ b/xCAT-probe/lib/perl/LogParse.pm @@ -489,12 +489,20 @@ sub obtain_log_content { $log_content{label} = $::LOGLABEL_DHCPD; } elsif ($split_line[2] =~ /in.tftpd/i) { $log_content{label} = $::LOGLABEL_TFTP; + } elsif ($split_line[2] =~ /^xcat.genesis.doxcat/i or $split_line[3] =~ /^xcat.genesis.doxcat/i) { + $log_content{label} = $::LOGLABEL_DOXCAT; + } elsif ($split_line[2] =~ /^xcat.genesis.dodiscovery/i or $split_line[3] =~ /^xcat.genesis.dodiscovery/i) { + $log_content{label} = $::LOGLABEL_DISCOVERY; } elsif ($split_line[2] =~ /^xcat/i) { $log_content{label} = $::LOGLABEL_XCAT; } else { $log_content{label} = $::LOGLABEL_UNDEF; } - $log_content{msg} = join(" ", @split_line[ 3 .. @split_line - 1 ]); + if ($split_line[3] =~ /^xcat.genesis.doxcat/i or $split_line[3] =~ /^xcat.genesis.dodiscovery/i) { + $log_content{msg} = join(" ", @split_line[ 4 .. @split_line - 1 ]); + } else { + $log_content{msg} = join(" ", @split_line[ 3 .. @split_line - 1 ]); + } } else { my $timestamp = join(" ", @split_line[ 0 .. 2 ]); $log_content{time} = $self->convert_to_epoch_seconds($timestamp); @@ -508,12 +516,20 @@ sub obtain_log_content { $log_content{label} = $::LOGLABEL_DHCPD; } elsif ($split_line[4] =~ /in.tftpd/i) { $log_content{label} = $::LOGLABEL_TFTP; - } elsif ($split_line[4] =~ /^xcat/i) { + } elsif ($split_line[4] =~ /^xcat.genesis.doxcat/i or $split_line[5] =~ /^xcat.genesis.doxcat/i) { + $log_content{label} = $::LOGLABEL_DOXCAT; + } elsif ($split_line[4] =~ /^xcat.genesis.dodiscovery/i or $split_line[5] =~ /^xcat.genesis.dodiscovery/i) { + $log_content{label} = $::LOGLABEL_DISCOVERY; + } elsif ($split_line[4] =~ /^xcat/i or $split_line[5] =~ /^xcat/i) { $log_content{label} = $::LOGLABEL_XCAT; } else { $log_content{label} = $::LOGLABEL_UNDEF; } - $log_content{msg} = join(" ", @split_line[ 5 .. @split_line - 1 ]); + if ($split_line[5] =~ /^xcat.genesis.doxcat/i or $split_line[5] =~ /^xcat.genesis.dodiscovery/i) { + $log_content{msg} = join(" ", @split_line[ 6 .. @split_line - 1 ]); + } else { + $log_content{msg} = join(" ", @split_line[ 5 .. @split_line - 1 ]); + } } } elsif ($log_type == $::LOGTYPE_HTTP) { $split_line[3] =~ s/^\[(.+)/$1/g; diff --git a/xCAT-probe/lib/perl/probe_global_constant.pm b/xCAT-probe/lib/perl/probe_global_constant.pm index a40f3c1e3..74d692636 100644 --- a/xCAT-probe/lib/perl/probe_global_constant.pm +++ b/xCAT-probe/lib/perl/probe_global_constant.pm @@ -7,11 +7,13 @@ $::LOGTYPE_RSYSLOG = 0; #rsyslog $::LOGTYPE_HTTP = 1; #apache log #The lable of specific line log -$::LOGLABEL_DHCPD = 0; -$::LOGLABEL_TFTP = 1; -$::LOGLABEL_HTTP = 2; -$::LOGLABEL_XCAT = 3; -$::LOGLABEL_UNDEF = 4; +$::LOGLABEL_DHCPD = 0; +$::LOGLABEL_TFTP = 1; +$::LOGLABEL_HTTP = 2; +$::LOGLABEL_XCAT = 3; +$::LOGLABEL_UNDEF = 4; +$::LOGLABEL_DOXCAT = 5; +$::LOGLABEL_DISCOVERY = 6; #The important stage of provision process $::STATE_POWER_ON = 1; @@ -44,4 +46,28 @@ $::STATE_COMPLETED = 13; $::STATE_POSTBOOTSCRIPT => "running_postbootscripts", $::STATE_COMPLETED => "complete", ); + +# The important stage of discovery process +$::STATE_DISCOVER_DHCP = 1; +$::STATE_DISCOVER_BOOTLODER = 2; +$::STATE_DISCOVER_KERNEL = 3; +$::STATE_DISCOVER_INITRD = 4; +$::STATE_DISCOVER_DOXCAT = 5; +$::STATE_DISCOVER_DISCOVERY = 6; +$::STATE_DISCOVER_REPORT = 7; +$::STATE_DISCOVER_COMPLETED = 8; + +#The description of every important stage of discovery process +%::STATE_DISCOVER_DESC = ( + $::STATE_DISCOVER_DHCP => "got_ip_from_dhcp", + $::STATE_DISCOVER_BOOTLODER => "download_genesis_bootloder", + $::STATE_DISCOVER_KERNEL => "download_genesis_kernel", + $::STATE_DISCOVER_INITRD => "download_genesis_initrd", + $::STATE_DISCOVER_DOXCAT => "enter_genesis", + $::STATE_DISCOVER_DISCOVERY => "start_to_discovery", + $::STATE_DISCOVER_REPORT => "send_discovery_request", + $::STATE_DISCOVER_COMPLETED => "discovery_complete", +); + + 1; diff --git a/xCAT-probe/subcmds/discovery b/xCAT-probe/subcmds/discovery index 4d4859c27..729dd249f 100755 --- a/xCAT-probe/subcmds/discovery +++ b/xCAT-probe/subcmds/discovery @@ -5,34 +5,40 @@ BEGIN { $::XCATROOT = $ENV{'XCATROOT'} ? $ENV{'XCATROOT'} : -d '/opt/xcat' ? '/o use lib "$::XCATROOT/probe/lib/perl"; use probe_utils; +use LogParse; +use probe_global_constant; use xCAT::NetworkUtils; use File::Basename; use IO::Select; +use Time::Local; use Getopt::Long qw(:config no_ignore_case); use Data::Dumper; -my $program_name = basename("$0"); -my $help; -my $test; -my $output = "stdout"; -my $verbose = 0; -my $rst = 0; -my $no_pre_check = 0; -my $discovery_type; -my @valid_discovery_type = ("mtms", "switch"); -my $valid_discovery_type_str = join(",", @valid_discovery_type); -my $noderange; -my $nics; #reservation attribute, format : xxx|xxx|xxx +#--------------------------------------------- +# Global attributes +#--------------------------------------------- -#used for discovery monitor -my %rawdata; +#------IP to mac map list------- +# $ipmacmap{ip_arrd} = "x.x.x.x" +#-------------------------------- my %ipmacmap; -my $terminal = 0; -my %monitor_nodes; +#Used by customer or developer, to obtain more output information +my $verbose = 0; + +#if no specific instruction, do monitor by default +my $monitor = 1; + +#used by developer, to debug the detail information about function running +my $debug = 0; + +#--------------------------------------------- +# Command Usage +#--------------------------------------------- +my $program_name = basename("$0"); $::USAGE = "Usage: $program_name -h - $program_name [-V] [-m -n ] [--noprecheck] + $program_name -n -m [-t ] [-V] [--noprecheck] Description: Probe the discovery process, including pre-check for required configuration and realtime monitor of discovery process. @@ -46,15 +52,413 @@ Options: -m : The method of discovery, the valid values are $valid_discovery_type_str. -n : The range of predefined nodes, must be used with option -m. --noprecheck : skip pre-checking discovery to validate correct configuration. + -t : The maximum time to wait when doing monitor, unit is minutes. default is 60. "; +#---------------------------------------------- +# Main process +#---------------------------------------------- + +# parse command line arguments +my $help = 0; +my $test = 0; +my $maxwaittime = 60; #unit is minute, the max wait time of monitor +my $rollforward_time_of_replay; #used by feature replay discovery log +my $noderange; +my $discovery_type; +my @valid_discovery_type = ("mtms", "switch"); +my $valid_discovery_type_str = join(",", @valid_discovery_type); +my $no_pre_check = 0; +my $nics; +if ( + !GetOptions("--help|h|?" => \$help, + "T" => \$test, + "V" => \$verbose, + "--noprecheck" => \$no_pre_check, + "m=s" => \$discovery_type, + "n=s" => \$noderange, + "t=s" => \$maxwaittime, +# "r=s" => \$rollforward_time_of_replay, + "i=s" => \$nics)) #option i is a reservation option, dosen't show up in usage now +{ + probe_utils->send_msg("stdout", "f", "Invalid parameter for $program_name"); + probe_utils->send_msg("stdout", "", "$::USAGE"); + exit 1; +} + +if ($help) { + probe_utils->send_msg("stdout", "", "$::USAGE"); + exit 0; +} + +if ($test) { + probe_utils->send_msg("stdout", "o", "Probe for discovery process, including pre-check for required configuration and realtime monitor of discovery process."); + exit 0; +} + +unless ($noderange) { + probe_utils->send_msg("stdout", "f", "A noderange is required"); + probe_utils->send_msg("stdout", "", "$::USAGE"); + exit 1; +} + +unless ($discovery_type) { + probe_utils->send_msg("stdout", "f", "Option '-n' must used with '-m'"); + probe_utils->send_msg("stdout", "", "$::USAGE"); + exit 1; +} + +if (defined($discovery_type)) { + unless (grep(/^$discovery_type$/, @valid_discovery_type)) { + probe_utils->send_msg("stdout", "f", "Invalid discovery type. the vaild types are $valid_discovery_type_str"); + probe_utils->send_msg("stdout", "", "$::USAGE"); + exit 1; + } +} + +my $rst = check_pre_defined_node(); +probe_utils->send_msg("stdout", "o", "All pre_defined nodes are valid") unless ($rst); +exit $rst if ($rst); + +if ($rollforward_time_of_replay) { + if (($rollforward_time_of_replay !~ /(\d+)h(\d+)m/i) && ($rollforward_time_of_replay !~ /^(\d+)h*$/i) && ($rollforward_time_of_replay !~ /^(\d+)m$/i)) { + probe_utils->send_msg("stdout", "f", "Unsupported time format for option '-r'"); + probe_utils->send_msg("stdout", "", "$::USAGE"); + exit 1; + } +} + +if (!$no_pre_check) { + $rst = do_pre_check(); + exit 1 if ($rst); +} + +if ($rollforward_time_of_replay) { + $monitor = 0; + + my $start_time_of_replay = time(); + my $end_time_of_replay = $start_time_of_replay; + if ($rollforward_time_of_replay =~ /(\d+)h(\d+)m/i) { + $start_time_of_replay -= ($1 * 3600 + $2 * 60) + } elsif ($rollforward_time_of_replay =~ /^(\d+)h*$/i) { + $start_time_of_replay -= $1 * 3600; + } elsif ($rollforward_time_of_replay =~ /^(\d+)m$/) { + $start_time_of_replay -= $1 * 60; + } + + $rst = do_replay($start_time_of_replay, $end_time_of_replay); + exit $rst; +} + +$rst = do_monitor(); + +exit $rst; + #------------------------------------------ =head3 Description: - Check if all genesis files are available. + Check if all predefined node are valid + Returns: + 0: pass + 1: failed +=cut + +#------------------------------------------ +sub check_pre_defined_node { + my $rst = 0; + my @cmdoutput; + my %nodecheckrst; + my @errornodes; + my $currentnode = ""; + + @cmdoutput = `lsdef $noderange 2>&1`; + foreach (@cmdoutput) { + if ($_ =~ /^Error: Could not find an object named '(\w+)' .+/i) { + $currentnode = $1; + push @errornodes, $currentnode; + $rst = 1; + } elsif ($_ =~ /^\s*Object name: (\w+)/i) { + $currentnode = $1; + $monitor_nodes{$1} = 0; + } elsif ($_ =~ /^\s+(\w+)\s*=\s*(\w+)/) { + $nodecheckrst{$currentnode}{$1} = $2; + } + } + + if ($rst) { + my $errornode = join(",", @errornodes); + probe_utils->send_msg("stdout", "f", "[$errornode] : Could not find node definition"); + } + + # check whether there is bmc node in noderange + my @bmcnodes = (); + foreach my $node (keys %nodecheckrst) { + if (($nodecheckrst{$node}{"nodetype"} eq "mp") and ($nodecheckrst{$node}{"hwtype"} eq "bmc")) { + push @bmcnodes, $node; + } + } + if (@bmcnodes) { + my $bmcnode = join(",", @bmcnodes); + probe_utils->send_msg("stdout", "f", "[$bmcnode] : bmc node(s)"); + $rst = 1; + } + + # if discover type is mtms, check whether mtms is only for per-define node and bmc node + @errornodes = (); + if ($discovery_type eq "mtms") { + + # get all nodes and their mtms in vpd table, %mtms_node save mtms and node map + my %mtms_node = (); + my @vpd = `tabdump vpd`; + foreach my $vpd_line (@vpd) { + next if ($vpd_line =~ /#node,serial,mtm,side,asset,uuid,comments,disable/); + chomp ($vpd_line); + $vpd_line =~ s/"//g; + my @split_vpd = split(",", $vpd_line); + if (($split_vpd[1] ne "") and ($split_vpd[2] ne "")) { + my $mtmsvpd = "$split_vpd[2]*$split_vpd[1]"; + push @{ $mtms_node{$mtmsvpd} }, $split_vpd[0]; + } + } + + my @error_mtms; + foreach my $node (keys %nodecheckrst) { + + # check pre-define node whether has mtm and serial + # if nodetype is mp and hwtye is bmc, the node is bmc node, do not check whether exists mtms + if (!(exists($nodecheckrst{$node}{"mtm"}) && exists($nodecheckrst{$node}{"serial"}))) { + next if (($nodecheckrst{$node}{"nodetype"} eq "mp") and ($nodecheckrst{$node}{"hwtype"} eq "bmc")); + push @errornodes, $node; + } else { + + # check if there is one or more node has the same mtms with current node + my $mtms = "$nodecheckrst{$node}{\"mtm\"}*$nodecheckrst{$node}{\"serial\"}"; + my $mtms_num = @{$mtms_node{$mtms}}; + if ($mtms_num > 2) { + push @error_mtms, $mtms if (!grep {$_ eq $mtms} @error_mtms); + } elsif ($mtms_num == 2) { + foreach my $mtmsnode (@{$mtms_node{$mtms}}) { + next if ($mtmsnode eq $node); + if (exists($nodecheckrst{$mtmsnode})) { + if (($nodecheckrst{$mtmsnode}{"nodetype"} eq $nodecheckrst{$node}{"nodetype"}) and ($nodecheckrst{$mtmsnode}{"mp"} eq $nodecheckrst{$node}{"hwtype"})) { + push @error_mtms, $mtms if (!grep {$_ eq $mtms} @error_mtms); + } + } else { + my $nodetype = `lsdef $mtmsnode -i nodetype -c |awk -F"=" '{print \$2}'`; + my $hwtype = `lsdef $mtmsnode -i hwtype -c |awk -F"=" '{print \$2}'`; + chomp($nodetype); + chomp($hwtype); + if (($nodetype eq $nodecheckrst{$node}{"nodetype"}) and ($hwtype eq $nodecheckrst{$node}{"hwtype"})) { + push @error_mtms, $mtms if (!grep {$_ eq $mtms} @error_mtms); + } + } + } + } + } + } + + if (@errornodes) { + my $errornode = join(",", @errornodes); + probe_utils->send_msg("stdout", "f", "[$errornode] : No mtm or serival for '$discovery_type' type discovery"); + $rst = 1; + } + + if (@error_mtms) { + foreach (@error_mtms) { + my $errornode = join(",", @{$mtms_node{$_}}); + probe_utils->send_msg("stdout", "f", "[$errornode] : Duplicate node definition found for the same mtms $_."); + $rst = 1; + } + } + } + + if ($discovery_type eq "switch") { + my @switchnodes = (); + foreach my $node (keys %nodecheckrst) { + if ($nodecheckrst{$node}{"nodetype"} eq "switch") { + push @switchnodes, $node; + } + } + + if (@switchnodes){ + my $switchnode = join(",", @switchnodes); + probe_utils->send_msg("stdout", "f", "[$switchnode] : switch node(s)"); + $rst = 1; + } + + my %switch_node = (); + my @switchoutput = `tabdump switch`; + foreach my $switch_line (@switchoutput) { + next if ($switch_line =~ /#node,switch,port,vlan,interface,comments,disable/); + chomp ($switch_line); + $switch_line =~ s/"//g; + my @split_switch = split(",", $switch_line); + if (($split_switch[1] ne "") and ($split_switch[2] ne "")) { + my $switchport = "$split_switch[1]*$split_switch[2]"; + push @{ $switch_node{$switchport} }, $split_switch[0]; + } + } + + my @error_switchport; + my %errorhash = (); + my $keystring; + foreach my $node (keys %nodecheckrst) { + { + $keystring = ""; + if (!(exists($nodecheckrst{$node}{"switch"}) && exists($nodecheckrst{$node}{"switchport"}))) { + last if ($nodecheckrst{$node}{"nodetype"} eq "switch"); + $keystring = "Atrribute 'switch' or 'switchport' isn't defined for '$discovery_type' type discovery"; + last; + } else { + my $switchport = "$nodecheckrst{$node}{\"switch\"}*$nodecheckrst{$node}{\"switchport\"}"; + my $switch_num = @{ $switch_node{$switchport} }; + if ($switch_num > 1) { + if (!grep {$_ eq $switchport} @error_switchport) { + push @error_switchport, $switchport; + } + } + } + + my $tmpoutput = `lsdef $nodecheckrst{$node}{"switch"} 2>&1`; + if ($?) { + $keystring = "Missing definition for related switch $nodecheckrst{$node}{\"switch\"}"; + last; + } + if ($tmpoutput !~ /snmpversion=/) { + $keystring = "Missing attribute 'snmpversion' definition for related switch $nodecheckrst{$node}{\"switch\"}"; + last; + } + if ($tmpoutput !~ /username=/) { + $keystring = "Missing attribute 'username' definition for related switch $nodecheckrst{$node}{\"switch\"}"; + last; + } + if ($tmpoutput !~ /password=/) { + $keystring = "Missing attribute 'password' definition for related switch $nodecheckrst{$node}{\"switch\"}"; + last; + } + } + + if ($keystring) { + if (exists($errorhash{$keystring})) { + $errorhash{$keystring} .= ",$node"; + } else { + $errorhash{$keystring} = $node; + } + } + } + + foreach my $key (keys %errorhash) { + probe_utils->send_msg("stdout", "f", "[$errorhash{$key}] : $key"); + $rst = 1; + } + foreach my $switch_port (@error_switchport) { + my $switchnode = join(",", @{ $switch_node{$switch_port} }); + probe_utils->send_msg("stdout", "f", "[$switchnode] : Duplicate node definition found for the same switch and switchport $switch_port."); + $rst = 1; + } + } + + return $rst; +} + +#----------------------------------------- + +=head3 + Description: + Do pre_checking Arguments: - arch: valid value are ppc64 and x86_64 + nics: target network interface + if not specified, using the network which master belongs to + Returns: + 0: pass + 1: failed +=cut + +#----------------------------------------- +sub do_pre_check { + my @nets = (); + my $rst = 0; + my $msg; + + # $nics is undef now, this part is for reservation + if ($nics) { + if ($nics =~/[^\w|]/) { + probe_utils->send_msg("stdout", "f", "Invalid NIC list"); + probe_utils->send_msg("stdout", "", "$::USAGE"); + exit 1; + } + + $msg = "The input network interfaces $nics exist and are configured correctly on current server"; + my @errors = (); + my @nic_array = split(",", $nics); + + foreach my $nic (@nic_array) { + my $tmp_nic = `ip addr show $nic >/dev/null 2>&1`; + if ($?) { + push @errors, "Network interface $nic doesn't exist on current server"; + } else { + my $tmp = `echo $tmp_nic |awk -F" " '/inet / {print \$2}'`; + chomp($tmp); + if (!length($tmp)) { + push @errors, "Network interface $nic isn't set IP address"; + } else { + my ($ip, $mask) = split("/", $tmp); + my $strmask = xCAT::NetworkUtils::formatNetmask($mask, 1, 0); + push(@nets, probe_utils->get_network($ip, $strmask)); + } + } + } + + if (@errors) { + probe_utils->send_msg("stdout", "f", $msg); + probe_utils->send_msg("stdout", "d", "$_") foreach (@errors); + } + } else { + $msg = "Attribute 'master' in 'site' table is configured well"; + my $masteripinsite = `tabdump site | awk -F',' '/^"master",/ { gsub(/"/, "", \$2) ; print \$2 }'`; + chomp($masteripinsite); + if ($masteripinsite eq "") { + probe_utils->send_msg("stdout", "f", $msg); + probe_utils->send_msg("stdout", "d", "There isn't 'master' definition in 'site' table"); + exit 1; + } + + if (!xCAT::NetworkUtils->isIpaddr("$masteripinsite")) { + probe_utils->send_msg("stdout", "f", $msg); + probe_utils->send_msg("stdout", "d", "The value of 'master' in 'site' table isn't an IP address"); + exit 1; + } + my $tmpoutput = `ip addr 2>&1 |grep $masteripinsite`; + if ($?) { + probe_utils->send_msg("stdout", "f", $msg); + probe_utils->send_msg("stdout", "d", "The IP $masteripinsite of 'master' in 'site' table dosen't belong to any network on current server"); + exit 1; + } + probe_utils->send_msg("stdout", "o", $msg); + + chomp($tmpoutput); + my $tmp = `echo $tmpoutput | awk -F" " '{print \$2}'`; + chomp($tmp); + my ($ip, $mask) = split("/", $tmp); + my $strmask = xCAT::NetworkUtils::formatNetmask($mask, 1, 0); + push(@nets, probe_utils->get_network($ip, $strmask)); + } + + $sub_func_rst = dhcp_dynamic_range_check(\@nets); + $rst |= $sub_func_rst; + + $sub_func_rst = check_genesis_file(); + $rst |= $sub_func_rst; + + return $rst; +} + +#------------------------------------------ + +=head3 + Description: + Check if all genesis files are available. Returns: 0 : pass 1 : failed @@ -62,41 +466,35 @@ Options: #------------------------------------------ sub check_genesis_file { - my $arch = shift; - if (($arch ne "ppc64") and ($arch ne "x86_64")) { - probe_utils->send_msg("$output", "d", "Specify correct arch type") if ($verbose); - return 1; - } - - my $rst_f = 0; - probe_utils->send_msg("$output", "d", "Starting to check genesis files for $arch...") if ($verbose); + my $msg = "Genesis files are avaliable"; + my $rst = 0; my $os = probe_utils->get_os(); - my $genesis_base; - my $genesis_scripts; - - if ($arch eq "x86_64") { - $arch_tmp = "amd64"; - } else { - $arch_tmp = $arch; - } - if ($os =~ "unknown") { - probe_utils->send_msg("$output", "d", "The OS is not supported.") if ($verbose); + probe_utils->send_msg("stdout", "f", $msg); + probe_utils->send_msg("stdout", "d", "The OS is not supported."); return 1; } elsif ($os =~ "ubuntu") { - $genesis_base = `dpkg -l | grep -iE "ii\\s+xcat-genesis-base" | grep -i "$arch_tmp"`; - $genesis_scripts = `dpkg -l | grep -iE "ii\\s+xcat-genesis-scripts" | grep -i "$arch_tmp"`; + my $genesis_output = `dpkg -l | grep -iE "ii\\s+xcat-genesis"`; + unless (($genesis_output =~ /base/ and $genesis_output =~ /ppc64/) and + ($genesis_output =~ /scripts/ and $genesis_output =~ /ppc64/) and + ($genesis_output =~ /base/ and $genesis_output =~ /amd64/) and + ($genesis_output =~ /scripts/ and $genesis_output =~ /amd64/)) { + probe_utils->send_msg("stdout", "f", $msg); + probe_utils->send_msg("stdout", "d", "xCAT-genesis is not installed."); + return 1; + } } else { - $genesis_base = `rpm -qa | grep -i "xcat-genesis-base" | grep -i "$arch"`; - $genesis_scripts = `rpm -qa | grep -i "xcat-genesis-scripts" | grep -i "$arch"`; + my $genesis_output = `rpm -qa | grep -i "xcat-genesis"`; + unless (($genesis_output =~ /base/ and $genesis_output =~ /ppc64/) and + ($genesis_output =~ /scripts/ and $genesis_output =~ /ppc64/) and + ($genesis_output =~ /base/ and $genesis_output =~ /x86_64/) and + ($genesis_output =~ /scripts/ and $genesis_output =~ /x86_64/)) { + probe_utils->send_msg("stdout", "f", $msg); + probe_utils->send_msg("stdout", "d", "xCAT-genesis is not installed."); + return 1; + } } - unless ($genesis_base and $genesis_scripts) { - probe_utils->send_msg("$output", "d", "xCAT-genesis for $arch is not installed.") if ($verbose); - return 1; - } - - probe_utils->send_msg("$output", "d", "xCAT-genesis for $arch is installed, starting to check files...") if ($verbose); my $tftpdir = `tabdump site | awk -F',' '/^"tftpdir",/ { gsub(/"/, "", \$2) ; print \$2 }'`; chomp($tftpdir); @@ -105,20 +503,22 @@ sub check_genesis_file { my @genesis_files; my $genesis_line; my $wget_rst; + my @errors; - if ($arch eq "ppc64") { + { # check genesis files for ppc64 arch $genesis_folder = "$tftpdir/pxelinux.cfg/p"; unless (-d "$genesis_folder") { - probe_utils->send_msg("$output", "d", "There is no genesis file for $arch. Run 'mknb ppc64' if using ppc64/ppc64le machine.") if ($verbose); - return 1; + push @errors, "There is no genesis file for ppc64. Run 'mknb ppc64' if using ppc64/ppc64le machine."; + $rst = 1; + last; } @genesis_files = glob("$genesis_folder/*"); foreach (@genesis_files) { unless (open(FILE, $_)) { - probe_utils->send_msg("$output", "d", "Cannot open file $_.") if ($verbose); - $rst_f = 1; + push @errors, "Cannot open file $_."; + $rst = 1; next; } @@ -131,10 +531,8 @@ sub check_genesis_file { $initrd_path = $initrd_info[1]; $wget_rst = system("wget -q --spider $initrd_path -T 0.5 -t 3"); if ($wget_rst) { - probe_utils->send_msg("$output", "d", "'initrd' cannot be downloaded from $initrd_path.") if ($verbose); - $rst_f = 1; - } else { - probe_utils->send_msg("$output", "d", "Check initrd file: $initrd_path PASS.") if ($verbose); + push @errors, "'initrd' cannot be downloaded from $initrd_path."; + $rst = 1; } } @@ -143,19 +541,20 @@ sub check_genesis_file { $kernel_path = $kernel_info[1]; $wget_rst = system("wget -q --spider $kernel_path -T 0.5 -t 3"); if ($wget_rst) { - probe_utils->send_msg("$output", "d", "kernel cannot be downloaded from $kernel_path.") if ($verbose); - $rst_f = 1; - } else { - probe_utils->send_msg("$output", "d", "Check kernel file: $kernel_path PASS.") if ($verbose); + push @errors, "kernel cannot be downloaded from $kernel_path."; + $rst = 1; } - } + } } - } - } else { + } + } + + { # check genesis files for x86_64 arch $genesis_folder = "$tftpdir/xcat/xnba/nets"; unless (-d "$genesis_folder") { - probe_utils->send_msg("$output", "d", "There is no genesis file for $arch. Run 'mknb x86_64' if using x86_64 machine.") if ($verbose); - return 1; + push @errors, "There is no genesis file for x86_64. Run 'mknb x86_64' if using x86_64 machine."; + $rst = 1; + last; } my @host_ip_arr; @@ -169,6 +568,7 @@ sub check_genesis_file { } @genesis_files = glob("$genesis_folder/*"); + foreach (@genesis_files) { if ($_ =~ /uefi$/) { my $file_name = basename($_); @@ -185,14 +585,14 @@ sub check_genesis_file { } unless ($host_ip) { - probe_utils->send_msg("$output", "d", "There is no IP for range $ip_range") if ($verbose); - $rst_f = 1; + push @errors, "There is no IP for range $ip_range"; + $rst = 1; next; } unless (open(FILE, $_)) { - probe_utils->send_msg("$output", "d", "Cannot open file $_."); - $rst_f = 1; + push @errors, "Cannot open file $_."; + $rst = 1; next; } @@ -208,22 +608,19 @@ sub check_genesis_file { $wget_rst = system("wget -q --spider $elilo_efi -T 0.5 -t 3"); if ($wget_rst) { - probe_utils->send_msg("$output", "d", "elilo-x64.efi cannot be downloaded from $elilo_efi.") if ($verbose); - $rst_f = 1; - } else { - probe_utils->send_msg("$output", "d", "Check elilo-x64.efi file: $elilo_efi PASS.") if ($verbose); + push @errors, "elilo-x64.efi cannot be downloaded from $elilo_efi."; + $rst = 1; } my $elilo_http = "http://$host_ip/$elilo_path"; $wget_rst = system("wget -q --spider $elilo_http -T 0.5 -t 3"); if ($wget_rst) { - probe_utils->send_msg("$output", "d", "elilo file cannot be downloaded from $elilo_http.") if ($verbose); - $rst_f = 1; + push @errors, "elilo file cannot be downloaded from $elilo_http."; + $rst = 1; } else { - probe_utils->send_msg("$output", "d", "Check elilo file: $elilo_http PASS.") if ($verbose); unless (open(FILE_ELILO, $elilo_path)) { - probe_utils->send_msg("$output", "d", "Cannot open file $_.") if ($verbose); - $rst_f = 1; + push @errors, "Cannot open file $_."; + $rst = 1; next; } @@ -237,10 +634,8 @@ sub check_genesis_file { $wget_rst = system("wget -q --spider $image_http -T 0.5 -t 3"); if ($wget_rst) { - probe_utils->send_msg("$output", "d", "image cannot be downloaded from $image_http.") if ($verbose); - $rst_f = 1; - } else { - probe_utils->send_msg("$output", "d", "Check image file: $image_http PASS.") if ($verbose); + push @errors, "image cannot be downloaded from $image_http."; + $rst = 1; } } if ($line_elilo =~ /^initrd/) { @@ -250,20 +645,164 @@ sub check_genesis_file { $wget_rst = system("wget -q --spider $initrd_http -T 0.5 -t 3"); if ($wget_rst) { - probe_utils->send_msg("$output", "d", "'initrd' cannot be downloaded from $initrd_http.") if ($verbose); - $rst_f = 1; - } else { - probe_utils->send_msg("$output", "d", "Check initrd file: $initrd_http PASS.") if ($verbose); + push @errors, "'initrd' cannot be downloaded from $initrd_http."; + $rst = 1; } } } } } } + } + } + } + + if ($rst) { + probe_utils->send_msg("stdout", "f", $msg); + probe_utils->send_msg("stdout", "d", $_) foreach (@errors); + } else { + probe_utils->send_msg("stdout", "o", $msg); + } + + return $rst; +} + +#------------------------------------------ + +=head3 + Description: + 1. check if there are dynamic range for specific networks defineded in dhcp conf file + 2. check if these specific networks have corresponding genesis configuration + Arguments: + networks: Array of networks. Every network is combined by network address and mask(i.e. network/mask). + For example: (10.0.0.0/255.0.0.0, 50.1.0.0/255.255.0.0) + Returns: + 0 : pass + 1 : failed +=cut + +#------------------------------------------ +sub dhcp_dynamic_range_check { + my $nets = shift; + my $rst = 0; + my $msg = "DHCP dynamic range is configured well"; + + my $dhcpconfig; + if (-e "/etc/dhcp/dhcpd.conf") { + $dhcpconfig = "/etc/dhcp/dhcpd.conf"; + } elsif (-e "/etc/dhcp3/dhcpd.conf") { + $dhcpconfig = "/etc/dhcp3/dhcpd.conf"; + } elsif (-e "/etc/dhcpd.conf") { + $dhcpconfig = "/etc/dhcpd.conf"; + } + + unless ($dhcpconfig) { + probe_utils->send_msg("stdout", "f", $msg); + probe_utils->send_msg("stdout", "d", "Cannot find the dhcpd.conf file."); + return 1; + } + + my $config_line; + my $subnet; + my @dynamic_range; + my %subnet_hash; + + unless (open(FILE, $dhcpconfig)) { + probe_utils->send_msg("stdout", "f", $msg); + probe_utils->send_msg("stdout", "d", "Cannot open file $dhcpconfig."); + return 1; + } + + while ($config_line = ) { + chomp($config_line); + $config_line =~ s/^\s+|\s+$//g; + + if ($config_line =~ /^subnet\s+(\d+\.\d+\.\d+\.\d+)\s+netmask\s+(\d+\.\d+\.\d+\.\d+)\s+/) { + $subnet = "$1/$2"; + $subnet_hash{$subnet} = "unknown"; + } + if ($config_line =~ /subnet_end/) { + $subnet_hash{$subnet} = [@dynamic_range] if (@dynamic_range); + $subnet = ""; + @dynamic_range = (); + } + if ($config_line =~ /^range dynamic-bootp (\d+.\d+.\d+.\d+) (\d+.\d+.\d+.\d+)/) { + if (compare_ip_value($1, $2)) { + push @dynamic_range, "$1-$2"; + } else { + push @dynamic_range, "$2-$1"; } } } - return $rst_f; + + my $net_ip; + my $netmask; + my $net_file_p; + my $net_file_x; + my $net_cdir; + my $tftpdir = `lsdef -t site -i tftpdir -c | awk -F "=" '{print \$2}'`; + chomp($tftpdir); + + unless ($tftpdir) { + $tftpdir = "/tftpboot"; + } + + my %node_ip; + if ($noderange) { + %node_ip = get_node_ip(); + } + + my @errors = (); + foreach my $net (@$nets) { + if (!exists($subnet_hash{$net})) { + push @errors, "The net $net is not matched."; + $rst = 1; + next; + } + + if ($subnet_hash{$net} ne "unknown") { + if (%node_ip) { + foreach my $node (keys %node_ip) { + foreach my $dr (@{ $subnet_hash{$net} }) { + my @dr_ip = split(/-/, $dr); + + if (compare_ip_value($dr_ip[0], $node_ip{$node}) and compare_ip_value($node_ip{$node}, $dr_ip[1])) { + push @errors, "$node ip $node_ip{$node} is conflicting with dynamic range."; + $rst = 1; + next; + } + } + } + } + } else { + push @errors, "Dynamic range for net $net is not configured."; + $rst = 1; + next; + } + + ($net_ip, $net_mask) = split('/', $net); + $net_cdir = xCAT::NetworkUtils::formatNetmask($net_mask, 0, 1); + $net_file_p = "$tftpdir/pxelinux.cfg/p/$net_ip" . "_$net_cdir"; + $net_file_x = "$tftpdir/xcat/xnba/nets/$net_ip" . "_$net_cdir.uefi"; + + if (! -e "$net_file_p") { + push @errors, "The default petitboot configuration file $net_file_p for net $net dose not exist."; + $rst = 1; + } + + if (! -e "$net_file_x") { + push @errors, "The default netboot configuration file $net_file_x for net $net dose not exist."; + $rst = 1; + } + } + + if ($rst) { + probe_utils->send_msg("stdout", "f", $msg); + probe_utils->send_msg("stdout", "d", "$_") foreach (@errors); + } else { + probe_utils->send_msg("stdout", "o", $msg); + } + return $rst; } sub get_node_ip { @@ -286,13 +825,13 @@ sub get_node_ip { foreach my $node (keys %nodeip) { $ip_net = xCAT::NetworkUtils->getipaddr($node); if ($nodeip{$node} and ($nodeip{$node} ne $ip_net)) { - probe_utils->send_msg("$output", "d", "IP $nodeip{$node} definition for $node is not correct") if ($verbose); + probe_utils->send_msg("stdout", "d", "IP $nodeip{$node} definition for $node is not correct") if ($verbose); } $nodeip{$node} = $ip_net; } foreach my $node (keys %nodecheckrst) { - probe_utils->send_msg("$output", "d", "$node : $nodecheckrst{$node}{error}") if (exists($nodecheckrst{$node}{error})); + probe_utils->send_msg("stdout", "d", "$node : $nodecheckrst{$node}{error}") if (exists($nodecheckrst{$node}{error})); } return %nodeip; @@ -319,647 +858,7 @@ sub compare_ip_value { =head3 Description: - 1. check if there are dynamic range for specific networks defineded in dhcp conf file - 2. check if these specific networks have corresponding genesis configuration - Arguments: - networks: Array of networks. Every network is combined by network address and mask(i.e. network/mask). - For example: (10.0.0.0/255.0.0.0, 50.1.0.0/255.255.0.0) - Returns: - 0 : pass - 1 : failed -=cut - -#------------------------------------------ -sub dhcp_dynamic_range_check { - my $nets = shift; - my $rst = 0; - - my $dhcpconfig; - if (-e "/etc/dhcp/dhcpd.conf") { - $dhcpconfig = "/etc/dhcp/dhcpd.conf"; - } elsif (-e "/etc/dhcp3/dhcpd.conf") { - $dhcpconfig = "/etc/dhcp3/dhcpd.conf"; - } elsif (-e "/etc/dhcpd.conf") { - $dhcpconfig = "/etc/dhcpd.conf"; - } - - unless ($dhcpconfig) { - probe_utils->send_msg("$output", "d", "Cannot find the dhcpd.conf file.") if ($verbose); - return 1; - } - - my $config_line; - my $subnet; - my @dynamic_range; - my %subnet_hash; - - unless (open(FILE, $dhcpconfig)) { - probe_utils->send_msg("$output", "d", "Cannot open file $dhcpconfig.") if ($verbose); - ($net_ip, $net_mask) = split('/', $net); - return 1; - } - - while ($config_line = ) { - chomp($config_line); - $config_line =~ s/^\s+|\s+$//g; - - if ($config_line =~ /^subnet\s+(\d+\.\d+\.\d+\.\d+)\s+netmask\s+(\d+\.\d+\.\d+\.\d+)\s+/) { - $subnet = "$1/$2"; - $subnet_hash{$subnet} = "unknown"; - } - if ($config_line =~ /subnet_end/) { - $subnet_hash{$subnet} = [@dynamic_range] if (@dynamic_range); - $subnet = ""; - @dynamic_range = ""; - } - if ($config_line =~ /^range dynamic-bootp (\d+.\d+.\d+.\d+) (\d+.\d+.\d+.\d+)/) { - if (compare_ip_value($1, $2)) { - push @dynamic_range, "$1-$2"; - } else { - push @dynamic_range, "$2-$1"; - } - - } - } - - my $net_ip; - my $netmask; - my $netfile; - my $net_cdir; - my $arch = `uname -i`; - chomp($arch); - my $tftpdir = `lsdef -t site -i tftpdir -c | awk -F "=" '{print \$2}'`; - chomp($tftpdir); - - unless ($tftpdir) { - $tftpdir = "/tftpboot"; - } - - my %node_ip; - if ($noderange) { - %node_ip = get_node_ip(); - } - - foreach my $net (@$nets) { - - if (!exists($subnet_hash{$net})) { - probe_utils->send_msg("$output", "d", "The net $net is not matched.") if ($verbose); - $rst = 1; - next; - } - - if ($subnet_hash{$net} ne "unknown") { - probe_utils->send_msg("$output", "d", "Dynamic range for net $net is @{$subnet_hash{$net}}.") if ($verbose); - - if (%node_ip) { - foreach my $node (keys %node_ip) { - foreach my $dr (@{ $subnet_hash{$net} }) { - my @dr_ip = split(/-/, $dr); - - if (compare_ip_value($dr_ip[0], $node_ip{$node}) and compare_ip_value($node_ip{$node}, $dr_ip[1])) { - probe_utils->send_msg("$output", "d", "$node ip $node_ip{$node} is conflicting with dynamic range.") if ($verbose); - $rst = 1; - } - } - } - } - } else { - probe_utils->send_msg("$output", "d", "Dynamic range for net $net is not configured.") if ($verbose); - $rst = 1; - next; - } - - ($net_ip, $net_mask) = split('/', $net); - $net_cdir = xCAT::NetworkUtils::formatNetmask($net_mask, 0, 1); - if ($arch =~ /ppc64/i) { - $net_file = "$tftpdir/pxelinux.cfg/p/$net_ip" . "_$net_cdir"; - } else { - $net_file = "$tftpdir/xcat/xnba/nets/$net_ip" . "_$net_cdir.uefi"; - } - - if (-e "$net_file") { - probe_utils->send_msg("$output", "d", "The genesis file $net_file for net $net exists.") if ($verbose); - } else { - probe_utils->send_msg("$output", "d", "The genesis file $net_file for net $net dose not exist.") if ($verbose); - $rst = 1; - } - } - - return $rst; -} - -#------------------------------------------ - -=head3 - Description: - Handle one line log come from tftp log file - Arguments: - msg: one line tftp log - Returns: - 0 : pass - 1 : failed -=cut - -#------------------------------------------ -sub handle_tftp_msg { - my $msg = shift; - if ($msg =~ /RRQ\s+from\s+(.+)\s+filename\s+(.+)/i) { - my $ip = $1; - my $file = $2; - my $record = "Via TFTP $ip download $file"; - probe_utils->send_msg("$output", "d", "$record"); - if (exists($rawdata{"$ipmacmap{$ip}"})) { - push(@{ $rawdata{ $ipmacmap{$ip} }{"history"} }, $record); - } - } - return 0; -} - -#------------------------------------------ - -=head3 - Description: - Handle one line log come from http log file - Arguments: - msg: one line http log - Returns: - 0 : pass - 1 : failed -=cut - -#------------------------------------------ -sub handle_http_msg { - my $msg = shift; - if ($msg =~ /(\d+\.\d+.\d+.\d+)\s.+GET\s+(.+)\s+HTTP.+/) { - my $ip = $1; - my $file = $2; - my $record = "Via HTTP $ip download $file"; - probe_utils->send_msg("$output", "d", "$record"); - if (exists($rawdata{"$ipmacmap{$ip}"})) { - push(@{ $rawdata{ $ipmacmap{$ip} }{"history"} }, $record); - } - } - return 0; -} - -#------------------------------------------ - -=head3 - Description: - Handle one line log come from dhcp log file - Arguments: - msg: one line http log - nics: target network interfaces - Returns: - 0 : pass - 1 : failed -=cut - -#------------------------------------------ -sub handle_dhcp_msg { - my $msg = shift; - - if ($msg =~ /.+DHCPDISCOVER\s+from\s+(.+)\s+via\s+([^:]+)(.*)/i) { - my $mac = $1; - my $nic = $2; - if ($3 =~ /no free leases/) { - probe_utils->send_msg("$output", "d", "Receive DHCPDISCOVER from $mac via $nic, no free leases"); - return 0; - } - my $record = "Receive DHCPDISCOVER from $mac via $nic"; - probe_utils->send_msg("$output", "d", "$record"); - push(@{ $rawdata{$mac}{"history"} }, $record); - } elsif ($msg =~ /.+DHCPOFFER\s+on\s+(.+)\s+to\s+(.+)\s+via\s+(.+)/i) { - my $ip = $1; - my $mac = $2; - my $nic = $3; - my $record = "Send DHCPOFFER on $ip back to $mac via $nic"; - probe_utils->send_msg("$output", "d", "$record"); - if (exists($rawdata{$mac})) { - push(@{ $rawdata{$mac}{"history"} }, $record); - } - } elsif ($msg !~ /unknown lease/ && $msg !~ /ignored/ && $msg =~ /.+DHCPREQUEST\s+for\s+(.+)\s\((.+)\)\s+from\s+(.+)\s+via\s+(.+)/) { - my $ip = $1; - my $server = $2; - my $mac = $3; - my $nic = $4; - my $record = "Receive DHCPREQUEST from $mac for $ip via $nic"; - probe_utils->send_msg("$output", "d", "$record"); - if (exists($rawdata{$mac})) { - push(@{ $rawdata{$mac}{"history"} }, $record); - } - } elsif ($msg =~ /.+DHCPACK\s+on\s+(.+)\s+to\s+(.+)\s+via\s+(.+)/) { - my $ip = $1; - my $mac = $2; - my $nic = $3; - my $record = "Send DHCPACK on $ip back to $mac via $nic"; - probe_utils->send_msg("$output", "d", "$record"); - if (exists($rawdata{$mac})) { - $rawdata{$mac}{"ip"} = $ip; - push(@{ $rawdata{$mac}{"history"} }, $record); - $ipmacmap{$ip} = $mac; - } - } elsif ($msg =~ /.+BOOTREQUEST\s+from\s+(.+)\s+via\s+([^:]+)(.*)/) { - my $mac = $1; - my $nic = $2; - if ($3 =~ /no dynamic leases/) { - probe_utils->send_msg("$output", "d", "Receive DHCPDISCOVER from $mac via $nic, no dynamic leases"); - return 0; - } - my $record = "Receive BOOTREQUEST from $mac via $nic"; - probe_utils->send_msg("$output", "d", "$record"); - push(@{ $rawdata{$mac}{"history"} }, $record); - } elsif ($msg =~ /.+BOOTREPLY\s+for\s+(.+)\s+to\s+.+(\w\w:\w\w:\w\w:\w\w:\w\w:\w\w).+via\s+(.+)/) { - my $ip = $1; - my $mac = $2; - my $nic = $3; - my $record = "Send BOOTREPLY on $ip back to $mac via $nic"; - probe_utils->send_msg("$output", "d", "$record"); - if (exists($rawdata{$mac})) { - $rawdata{$mac}{"ip"} = $ip; - push(@{ $rawdata{$mac}{"history"} }, $record); - $ipmacmap{$ip} = $mac; - } - } - return 0; -} - -#------------------------------------------ - -=head3 - Description: - Handle one line log come from computes.log - Arguments: - msg: one line compute log - Returns: - 0 : pass - 1 : failed -=cut - -#------------------------------------------ -sub handle_compute_msg { - - my $line = shift; - my $sender = ""; - my $ip = ""; - my $msg; - - my @splitline = split(/\s+/, $line); - if (($splitline[4] =~ /^xcat/i) || ($splitline[5] =~ /^xcat/i)) { - $sender = $splitline[3]; - if (($splitline[4] =~ /^xcat/i) && ($splitline[5] !~ /^xcat/i)) { - splice(@splitline, 0, 4); - } else { - splice(@splitline, 0, 5); - } - $msg = join(" ", @splitline); - - if (!xCAT::NetworkUtils->isIpaddr($sender)) { - $ip = xCAT::NetworkUtils->getipaddr($sender); - } else { - $ip = $sender; - } - - if ($ip ne "" && defined($ipmacmap{$ip})) { - my $record = "Recv from $ip : $msg"; - probe_utils->send_msg("$output", "d", "$record"); - push(@{ $rawdata{ $ipmacmap{$ip} }{"history"} }, $record); - } - - # There is a node finish discovry process - if ($msg =~ /xcat.genesis.dodiscovery: Restart/) { - my $node = `lsdef -i mac -c 2>&1 | awk -F: '/$ipmacmap{$ip}/ {print \$1}'`; - chomp($node); - $monitor_nodes{$node} = 1 if (defined($monitor_nodes{$node})); - probe_utils->send_msg("$output", "o", "Node $node has finished its discovery process"); - my $output = `lsdef $node 2>&1`; - print "-------------------\n$output-------------------\n"; - } - } - return 0; -} - -#------------------------------------------ - -=head3 - Description: - Handle one line log come from cluster.log - Arguments: - msg: one line log - Returns: - 0 : pass - 1 : failed -=cut - -#------------------------------------------ -sub handle_cluster_msg { - my $line = shift; - my $sender = ""; - my $ip = ""; - my $msg; - - my @splitline = split(/\s+/, $line); - if (($splitline[4] =~ /^xcat/i) || ($splitline[5] =~ /^xcat/i)) { - if (($splitline[5] =~ /^xcat.discovery/i) && ($splitline[6] =~ /^\((.+)\)$/)) { - my $mac = $1; - if (xCAT::NetworkUtils->isValidMAC($mac) && defined($rawdata{$mac})) { - splice(@splitline, 0, 5); - splice(@splitline, 1, 1); - $msg = join(" ", @splitline); - if (defined($rawdata{$mac}{"ip"})) { - $record = "Recv from $rawdata{$mac}{ip} : $msg"; - } else { - $record = "Recv from $mac : $msg"; - } - probe_utils->send_msg("$output", "d", "$record"); - push(@{ $rawdata{$mac}{"history"} }, $record); - } - } - } - return 0; -} - -#------------------------------------------ - -=head3 - Description: - Dump monitor history, categorised by mac address. - Arguments: - NULL - Returns: -=cut - -#------------------------------------------ -sub dump_history { - - my $title = " -============================================================= -= The summary of discovery: -============================================================= -"; - print "$title\n"; - - foreach $mac (keys %rawdata) { - my $nodehostname = `lsdef -i mac -c 2>&1 | awk -F: '/$mac/ {print \$1}'`; - chomp($nodehostname); - if ($nodehostname ne "") { - probe_utils->send_msg("$output", "d", "[$mac ($nodehostname)]"); - } else { - probe_utils->send_msg("$output", "d", "[$mac]:"); - } - foreach my $line (@{ $rawdata{$mac}{"history"} }) { - probe_utils->send_msg("$output", "d", "\t$line"); - } - print "\n"; - } -} - -#------------------------------------------ - -=head3 - Description: - Check if all predefined node are valid - Arguments: - discovery_type: valid value are mtms and switch - noderange: node range - Returns: - 0: pass - 1: failed -=cut - -#------------------------------------------ -sub check_pre_defined_node { - my $discovery_type = shift; - my $noderange = shift; - - my $rst = 0; - my @cmdoutput; - my %nodecheckrst; - my $currentnode = ""; - - @cmdoutput = `lsdef $noderange 2>&1`; - foreach (@cmdoutput) { - if ($_ =~ /^Error: Could not find an object named '(\w+)' .+/i) { - $currentnode = $1; - $nodecheckrst{$currentnode}{"error"} = "Could not find node definition"; - $rst = 1; - } elsif ($_ =~ /^\s*Object name: (\w+)/i) { - $currentnode = $1; - $monitor_nodes{$1} = 0; - } elsif ($_ =~ /^\s+(\w+)\s*=\s*(\w+)/) { - $nodecheckrst{$currentnode}{$1} = $2; - } - } - - #print Dumper(%nodecheckrst); - - foreach my $node (keys %nodecheckrst) { - if (!exists($nodecheckrst{$node}{error})) { - if ($discovery_type eq "mtms") { - if (!(exists($nodecheckrst{$node}{"mtm"}) && exists($nodecheckrst{$node}{"serial"}))) { - $nodecheckrst{$node}{"error"} = "Node definition is wrong for '$discovery_type' type discovery"; - $rst = 1; - } - } elsif ($discovery_type eq "switch") { - { #important to hold a block - if (!(exists($nodecheckrst{$node}{"switch"}) && exists($nodecheckrst{$node}{"switchport"}))) { - $nodecheckrst{$node}{"error"} = "Atrribute 'switch' or 'switchport' isn't defined for '$discovery_type' type discovery"; - $rst = 1; - last; - } - - my $tmpoutput = `lsdef $nodecheckrst{$node}{"switch"} 2>&1`; - if ($?) { - $nodecheckrst{$node}{"error"} = "Missing definition for related switch $nodeswitch"; - $rst = 1; - last; - } - - if ($tmpoutput !~ /snmpversion=/) { - $nodecheckrst{$node}{"error"} = "Missing attribute 'snmpversion' definition for related switch $nodeswitch"; - $rst = 1; - last; - } - if ($tmpoutput !~ /username=/) { - $nodecheckrst{$node}{"error"} = "Missing attribute 'username' definition for related switch $nodeswitch"; - $rst = 1; - last; - } - if ($tmpoutput !~ /password=/) { - $nodecheckrst{$node}{"error"} = "Missing attribute 'password' definition for related switch $nodeswitch"; - $rst = 1; - last; - } - } - } - } - } - - foreach my $node (keys %nodecheckrst) { - probe_utils->send_msg("$output", "d", "$node : $nodecheckrst{$node}{error}") if (exists($nodecheckrst{$node}{error})); - } - - return $rst; -} - -#------------------------------------------ - -=head3 - Description: - Test if all nodes have finished job - Arguments: - One golble attribute %monitor_nodes; - Returns: -=cut - -#------------------------------------------ -sub all_monitor_node_done { - my $done = 1; - foreach my $node (keys %monitor_nodes) { - if ($monitor_nodes{$node} == 0) { - $done = 0; - last; - } - } - return $done; -} - -#------------------------------------------ - -=head3 - Description: - Do pre_checking - Arguments: - nics: target network interface - if not specified, using the network which master belongs to - Returns: - 0: pass - 1: failed -=cut - -#------------------------------------------ -sub do_pre_check { - my $nics = shift; - my @nets = (); - my $rst = 0; - my $msg; - - #The block of $nics is ture is a reservation part, this part don't show up in usage - if ($nics) { - if ($nics =~ /[^\w|]/) { - probe_utils->send_msg("$output", "f", "Invalid NIC list"); - probe_utils->send_msg("$output", "d", "$::USAGE"); - exit 1; - } - - $msg = "The input network interfaces $nics exist and are configured correctly on current server"; - my $miss = 0; - my @nic_array = split(",", $nics); - foreach my $nic (@nic_array) { - my $tmp_nic = `ip addr show $nic >/dev/null 2>&1`; - if ($?) { - probe_utils->send_msg("$output", "d", "Network interface $nic doesn't exist on current server") if ($verbose); - $miss = 1; - } else { - my $tmp = `echo $tmp_nic |awk -F" " '/inet / {print \$2}'`; - chomp($tmp); - if (!length($tmp)) { - probe_utils->send_msg("$output", "d", "Network interface $nic isn't set IP address") if ($verbose); - $miss = 1; - } else { - my ($ip, $mask) = split("/", $tmp); - my $strmask = xCAT::NetworkUtils::formatNetmask($mask, 1, 0); - push(@nets, probe_utils->get_network($ip, $strmask)); - } - } - } - if ($miss) { - probe_utils->send_msg("$output", "f", $msg); - $rst = 1; - } else { - probe_utils->send_msg("$output", "o", $msg); - } - } else { - $msg = "Attribute 'master' in 'site' table is configured well"; - my $masteripinsite = `tabdump site | awk -F',' '/^"master",/ { gsub(/"/, "", \$2) ; print \$2 }'`; - chomp($masteripinsite); - if ($masteripinsite eq "") { - probe_utils->send_msg("$output", "d", "There isn't 'master' definition in 'site' table") if ($verbose); - probe_utils->send_msg("$output", "f", $msg); - exit 1; - } - - if (!xCAT::NetworkUtils->isIpaddr("$masteripinsite")) { - probe_utils->send_msg("$output", "d", "The value of 'master' in 'site' table isn't an IP address") if ($verbose); - probe_utils->send_msg("$output", "f", $msg); - exit 1; - } - my $tmpoutput = `ip addr 2>&1 |grep $masteripinsite`; - if ($?) { - probe_utils->send_msg("$output", "d", "The IP $masteripinsite of 'master' in 'site' table dosen't belong to any network on current server") if ($verbose); - probe_utils->send_msg("$output", "f", $msg); - exit 1; - } - probe_utils->send_msg("$output", "o", $msg); - - chomp($tmpoutput); - my $tmp = `echo $tmpoutput | awk -F" " '{print \$2}'`; - chomp($tmp); - my ($ip, $mask) = split("/", $tmp); - my $strmask = xCAT::NetworkUtils::formatNetmask($mask, 1, 0); - push(@nets, probe_utils->get_network($ip, $strmask)); - } - - my $arch = `uname -i`; - chomp($arch); - $msg = "Genesis files for $arch are available"; - if ($arch =~ /ppc64/i) { - if (check_genesis_file("ppc64")) { - probe_utils->send_msg("$output", "f", $msg); - $rst = 1; - } else { - probe_utils->send_msg("$output", "o", $msg); - if (check_genesis_file("x86_64")) { - probe_utils->send_msg("$output", "w", "Genesis files for x86 are not available"); - probe_utils->send_msg("$output", "i", "Ignore above warning if not planning on managing x86 server"); - } else { - probe_utils->send_msg("$output", "o", "Genesis files for x86 are available"); - } - } - } elsif ($arch =~ /x86/i) { - if (check_genesis_file("x86_64")) { - probe_utils->send_msg("$output", "f", $msg); - $rst = 1; - } else { - probe_utils->send_msg("$output", "o", $msg); - if (check_genesis_file("ppc64")) { - probe_utils->send_msg("$output", "w", "Genesis files for ppc64/ppc64le are not available"); - probe_utils->send_msg("$output", "i", "ignore above warning if not planning on managing ppc64/ppc64le server"); - } else { - probe_utils->send_msg("$output", "o", "Genesis files for ppc64/ppc64le are available"); - } - } - } - - $msg = "DHCP dynamic range is configured well"; - if (dhcp_dynamic_range_check(\@nets)) { - probe_utils->send_msg("$output", "f", $msg); - $rst = 1; - } else { - probe_utils->send_msg("$output", "o", $msg); - } - - return $rst; -} - -#------------------------------------------ - -=head3 - Description: - Monitor the process of discovery - Arguments: - nics: target network interface - if not specified, using the network which master belongs to + Monitor the process of discovery Returns: 0: pass 1: failed @@ -967,22 +866,13 @@ sub do_pre_check { #------------------------------------------ sub do_monitor { + my $rst = 0; + my $terminal = 0; $SIG{TERM} = $SIG{INT} = sub { $terminal = 1; }; - if (!$nics) { - my $masteripinsite = `tabdump site | awk -F',' '/^"master",/ { gsub(/"/, "", \$2) ; print \$2 }'`; - chomp($masteripinsite); - $nics = `ip addr |grep -B2 $masteripinsite|awk -F" " '/mtu/{gsub(/:/,"",\$2); print \$2}'`; - chomp($nics); - if (!$nics) { - probe_utils->send_msg("$output", "f", "The value of 'master' in 'site' table is $masteripinsite, can't get corresponding network interface"); - return 1; - } - } - my $startline = " ------------------------------------------------------------- ___ @@ -993,183 +883,655 @@ sub do_monitor { /_/\\_\\\\____/_/ \\_\\_| .+|______|__.-||__)`-'(((/ (((/ ------------------------------------------------------------- "; - print "$startline\nStart capturing every message during discovery process......\n"; - my $varlogmsg = "/var/log/messages"; - my $clusterlog = "/var/log/xcat/cluster.log"; - my $computelog = "/var/log/xcat/computes.log"; + probe_utils->send_msg("stdout", "", "$startline\nStart capturing every message during discovery process......\n"); - #http logs are saved in different file in different distro - my $httplog; - if (-e "/var/log/httpd/access_log") { - $httplog = "/var/log/httpd/access_log"; - } elsif (-e "/var/log/apache2/access_log") { - $httplog = "/var/log/apache2/access_log"; - } elsif (-e "/var/log/apache2/access.log") { - $httplog = "/var/log/apache2/access.log"; - } + my @openfilepids; + my @openfilefds; + my %fd_filetype_map; - my $clusterpid; - my $httppid; - my $computerpid; - my $varlogpid; - my $rst = 0; - { #important to hold a block - if (!-e "$varlogmsg") { - probe_utils->send_msg("$output", "w", "$varlogmsg doesn't exist"); - } - if (!-e "$clusterlog") { - probe_utils->send_msg("$output", "w", "$clusterlog doesn't exist"); - } - if (!-e "$computelog") { - probe_utils->send_msg("$output", "w", "$computelog doesn't exist"); - } - if (!-e "$httplog") { - probe_utils->send_msg("$output", "w", "$httplog doesn't exist"); + { #a very important brace to hold a code block + my $log_parse = LogParse->new($verbose); + my $candidate_log_ref = $log_parse->obtain_log_file_list(); + + #open candidate log file to obtain realtime log + if (%$candidate_log_ref) { + foreach my $logfile (keys %$candidate_log_ref) { + my $pid; + my $fd; + if (!($pid = open($fd, "tail -f -n 0 $candidate_log_ref->{$logfile}{file} 2>&1 |"))) { + probe_utils->send_msg("stdout", "f", "Can't open $candidate_log_ref->{$logfile}{file} to get logs"); + $rst = 1; + last; + } else { + push @openfilepids, $pid; + push @openfilefds, $fd; + $fd_filetype_map{$fd} = $candidate_log_ref->{$logfile}{type}; + } + } + } else { + probe_utils->send_msg("stdout", "f", "There are no valid log files to be scanned"); + $rst = 1; } - # start ot obtain logs from every log file - if (!($varlogpid = open(VARLOGMSGFILE, "tail -f -n 0 $varlogmsg 2>&1 |"))) { - probe_utils->send_msg("$output", "f", "Can't open $varlogmsg to get logs"); - $rst = 1; - last; - } - if (!($clusterpid = open(CLUSTERLOGFILE, "tail -f -n 0 $clusterlog 2>&1 |"))) { - probe_utils->send_msg("$output", "f", "Can't open $clusterlog to get logs"); - $rst = 1; - last; - } - if (!($httppid = open(HTTPLOGFILE, "tail -f -n 0 $httplog 2>&1 |"))) { - probe_utils->send_msg("$output", "f", "Can't open $httplog to get logs"); - $rst = 1; - last; - } - if (!($computerpid = open(COMPUTERFILE, "tail -f -n 0 $computelog 2>&1 |"))) { - probe_utils->send_msg("$output", "f", "Can't open $computelog to get logs"); - $rst = 1; - last; - } + last if ($rst); + + my %node_state; + init_node_state($noderange, \%node_state); my $select = new IO::Select; - $select->add(\*VARLOGMSGFILE); - $select->add(\*CLUSTERLOGFILE); - $select->add(\*HTTPLOGFILE); - $select->add(\*COMPUTERFILE); + $select->add(\*$_) foreach (@openfilefds); $| = 1; - my $line = ""; my @hdls; - my $hdl; + my $starttime = time(); + my @candidate_mn_hostname_in_log = $log_parse->obtain_candidate_mn_hostname_in_log(); + + #read log realtimely, then handle each log for (; ;) { if (@hdls = $select->can_read(0)) { - foreach $hdl (@hdls) { - if ($hdl == \*VARLOGMSGFILE) { - chomp($line = ); - my @tmp = split(/\s+/, $line); - if ($tmp[4] =~ /dhcpd:/i && $line =~ /$nics/) { - handle_dhcp_msg("$line"); - } elsif ($tmp[4] =~ /in.tftpd/i) { - handle_tftp_msg("$line"); - } - } elsif ($hdl == \*CLUSTERLOGFILE) { - chomp($line = ); - handle_cluster_msg("$line"); - } elsif ($hdl == \*HTTPLOGFILE) { - chomp($line = ); - handle_http_msg("$line"); - } elsif ($hdl == \*COMPUTERFILE) { - chomp($line = ); - handle_compute_msg("$line"); - } + foreach my $hdl (@hdls) { + my $line = ""; + chomp($line = <$hdl>); + my $log_content_ref = $log_parse->obtain_log_content($fd_filetype_map{$hdl}, $line); + dispatch_log_to_handler($log_content_ref, \@candidate_mn_hostname_in_log, \%node_state); } } - if ($terminal || (%monitor_nodes && all_monitor_node_done())) { - if ($terminal) { - probe_utils->send_msg("$output", "d", "Got from STDIN"); - } else { - probe_utils->send_msg("$output", "o", "All nodes specified to monitor have finished discovery process"); - } + # stop reading log at below 3 scenarios + # 1 receive terminal signal from customer + if ($terminal) { + probe_utils->send_msg("stdout", "d", "Get INT or TERM signal from STDIN"); last; + + # 2 all node have finished the discovery + } elsif (all_monitor_node_done(\%node_state)) { + probe_utils->send_msg("stdout", "o", "All nodes specified to monitor, have finished discovery process"); + last; + + # 3 exceed the max waiting time + } elsif (time() - $starttime > ($maxwaittime * 60)) { + probe_utils->send_msg("stdout", "i", "$maxwaittime minutes have expired, stop monitoring"); + last; + } else { + sleep 0.01; } - sleep 0.01; } - &dump_history; + + conclusion_report(\%node_state); + $log_parse->destory(); } - kill 'INT', $clusterpid if ($clusterpid); - kill 'INT', $httppid if ($httppid); - kill 'INT', $computerpid if ($computerpid); - kill 'INT', $varlogpid if ($varlogpid); - close(VARLOGMSGFILE) if (VARLOGMSGFILE); - close(CLUSTERLOGFILE) if (CLUSTERLOGFILE); - close(COMPUTERFILE) if (COMPUTERFILE); - close(HTTPLOGFILE) if (HTTPLOGFILE); + # close all running sub process + my $existrunningpid = 0; + $existrunningpid = 1 if (@openfilepids); + my $trytime = 0; + while ($existrunningpid) { + + #send terminal signal to all running process at same time + if ($try < 5) { #try INT 5 up to 5 times + foreach my $pid (@openfilepids) { + kill 'INT', $pid if ($pid); + } + } elsif ($try < 10) { #try TERM 5 up to 5 times + foreach my $pid (@openfilepids) { + kill 'TERM', $pid if ($pid); + } + } else { #try KILL 1 time + foreach my $pid (@openfilepids) { + kill 'KILL', $pid if ($pid); + } + } + ++$try; + sleep 1; + + #To check how many process exit, set the flag of exited process to 0 + for (my $i = 0 ; $i <= $#openfilepids ; $i++) { + $openfilepids[$i] = 0 if (waitpid($openfilepids[$i], WNOHANG)); + } + + #To check if there are processes still running, if there are, try kill again in next loop + $existrunningpid = 0; + $existrunningpid |= $_ foreach (@openfilepids); + + #just try 10 times, if still can't kill some process, give up + if ($try > 10) { + my $leftpid; + foreach my $pid (@openfilepids) { + $leftpid .= "$pid " if ($pid); + } + probe_utils->send_msg("stdout", "d", "Can't stop process $leftpid, please handle manually."); + last; + } + } + + # close all openning file descriptors + close($_) foreach (@openfilefds); return $rst; } -#------------------------------------- -## main process -#------------------------------------- -if ( - !GetOptions("--help|h|?" => \$help, - "T" => \$test, - "V" => \$verbose, - "--noprecheck" => \$no_pre_check, - "m=s" => \$discovery_type, - "n=s" => \$noderange, - "N=s" => \$nics)) #option N is a reservation option, dosen't show up in usage now -{ - probe_utils->send_msg("$output", "f", "Invalid parameter for $program_name"); - probe_utils->send_msg("$output", "d", "$::USAGE"); - exit 1; +#------------------------------------------ + +=head3 + Description: + Implement the replay feature. + Arguments: + start_time_of_replay: the start time point of scaning log + end_time_of_replay: the end time point of scaning log + + Returns: + 0: success + 1: failed +=cut + +#------------------------------------------ +sub do_replay { + my $start_time_of_replay = shift; + my $end_time_of_replay = shift; + + my $rc = 0; + + #handle INT/TERM signal + my $terminal = 0; + $SIG{TERM} = $SIG{INT} = sub { + $terminal = 1; + }; + + my $timestr = scalar(localtime($start_time_of_replay)); + probe_utils->send_msg("stdout", "d", "Starting to scan logs which are later than '$timestr', please waiting for a while............."); + + my %node_state; + init_node_state($noderange, \%node_state); + if ($debug) { + print "Dumper node_state-------\n"; + print Dumper \%node_state; + } + + my $log_parse = LogParse->new($verbose); + my @candidate_mn_hostname_in_log = $log_parse->obtain_candidate_mn_hostname_in_log(); + + while ($start_time_of_replay < $end_time_of_replay) { + my @valid_one_second_log_set; + my $rst = $log_parse->obtain_one_second_logs($start_time_of_replay, \@valid_one_second_log_set); + if ($rst) { + probe_utils->send_msg("stdout", "d", "Failed to obtain logs from log files"); + $rc = 1; + last; + } + + foreach my $log_ref (@valid_one_second_log_set) { + dispatch_log_to_handler($log_ref, \@candidate_mn_hostname_in_log, \%node_state); + } + + $start_time_of_replay = $log_parse->obtain_next_second(); + + # receive terminal signal from customer + if ($terminal) { + probe_utils->send_msg("stdout", "d", "Get INT or TERM signal!!!"); + probe_utils->send_msg("stdout", "w", "Haven't scaned all valid logs, report based on the logs have been scaned"); + last; + } + } + $log_parse->destory(); + + conclusion_report(\%node_state); + return $rc; } -if ($help) { - if ($output ne "stdout") { - probe_utils->send_msg("$output", "d", "$::USAGE"); +#------------------------------------------ + +=head3 + Description: + Initailize a very important hash "%node_state" which will save the state information of every node + Arguments: + noderange: (input attribute) The range of node + node_state_ref: (output attribute) the reference of hash "%node_state" + The strucuture of hash "%node_state" are : + $node_state{}{statehistory} Array. save the latest loop discovery states + $node_state{}{allstatehistory} Array. save the history states before the latest loop discovery. Used in debug mode. + $node_state{}{log} Array. save all related logs of mac. Used in debug mode. + $node_state{}{id} Scalar. the node related withe the mac. + $node_state{}{type} Scalar. the flag of if the node have finished the discovery + Returns: + NULL +=cut + +#------------------------------------------ +sub init_node_state { + my $noderange = shift; + my $node_state_ref = shift; + + my @nodes = probe_utils->parse_node_range($noderange); + foreach my $node (@nodes) { + $node_state_ref->{$node}{type} = "node"; + $node_state_ref->{$node}{done} = 0; + } +} + +#------------------------------------------ + +=head3 + Description: + Dispatch log to related handler + Arguments: + log_ref: (input attribute) the reference of hash which save one line log comes from computes.log. + candidate_mn_hostname_in_log_ref: (input attribute) The reference of array which save the candidate host name of MN + node_state_ref: (output attribute), the reference of hash "%node_state". refer to function "init_node_state" for the structure of "%node_state" + Returns: + NULL +=cut + +#------------------------------------------ +sub dispatch_log_to_handler { + my $log_ref = shift; + my $candidate_mn_hostname_in_log_ref = shift; + my $node_state_ref = shift; + + if ($log_ref->{label} == $::LOGLABEL_DHCPD) { + handle_dhcp_msg($log_ref, $node_state_ref); + } elsif ($log_ref->{label} == $::LOGLABEL_TFTP) { + handle_tftp_msg($log_ref, $node_state_ref); + } elsif ($log_ref->{label} == $::LOGLABEL_XCAT or $log_ref->{label} == $::LOGLABEL_DOXCAT or $log_ref->{label} == $::LOGLABEL_DISCOVERY) { + if (grep(/$log_ref->{sender}/, @$candidate_mn_hostname_in_log_ref)) { + handle_cluster_msg($log_ref, $node_state_ref); + } else { + handle_compute_msg($log_ref, $node_state_ref); + } + } elsif ($log_ref->{label} == $::LOGLABEL_HTTP) { + handle_http_msg($log_ref, $node_state_ref); + } +} + +#------------------------------------------ + +=head3 + Description: + Handle one line DHCP log + Arguments: + log_ref: (input attribute) the reference of hash which save one line dhcp log. + node_state_ref: (output attribute), the reference of hash "%node_state". refer to function "init_node_state" for the structure of "%node_state" + Returns: + NULL +=cut + +#------------------------------------------ +sub handle_dhcp_msg { + my $log_ref = shift; + my $node_state_ref = shift; + + if ($log_ref->{msg} =~ /DHCPDISCOVER\s+from\s+(\w\w:\w\w:\w\w:\w\w:\w\w:\w\w).+via\s+([^:]+)(.*)/i) { + my $mac = $1; + my $nic = $2; + + if ($3 =~ /no free leases/) { + probe_utils->send_msg("stdout", "d", "[$mac] Receive DHCPDISCOVER from $mac via $nic, no free leases") if ($monitor); + return 0; + } + my $record = "Receive DHCPDISCOVER from $mac via $nic"; + probe_utils->send_msg("stdout", "d", "[$mac] $record") if ($monitor); + push(@{ $node_state_ref->{$mac}{log} }, $log_ref->{msg}) if ($debug); + } elsif ($log_ref->{msg} =~ /DHCPOFFER\s+on\s+(.+)\s+to\s+(\w\w:\w\w:\w\w:\w\w:\w\w:\w\w).+via\s+(.+)/i) { + my $ip = $1; + my $mac = $2; + my $nic = $3; + my $record = "Send DHCPOFFER on $ip back to $mac via $nic"; + + probe_utils->send_msg("stdout", "d", "[$mac] $record") if ($monitor); + push(@{ $node_state_ref->{$mac}{log} }, $log_ref->{msg}) if ($debug); + } elsif ($log_ref->{msg} =~ /DHCPREQUEST\s+for\s+(.+)\s+[\(\)0-9\.]*\s*from\s+(\w\w:\w\w:\w\w:\w\w:\w\w:\w\w).+via\s+(.+)/) { + my $ip = $1; + my $mac = $2; + my $nic = $3; + my $record = "Receive DHCPREQUEST from $mac for $ip via $nic"; + + probe_utils->send_msg("stdout", "d", "[$mac] $record") if ($monitor); + push(@{ $node_state_ref->{$mac}{log} }, $log_ref->{msg}) if ($debug); + } elsif ($log_ref->{msg} =~ /DHCPACK\s+on\s+(.+)\s+to\s+(\w\w:\w\w:\w\w:\w\w:\w\w:\w\w).+via\s+(.+)/) { + my $ip = $1; + my $mac = $2; + my $nic = $3; + my $record = "Send DHCPACK on $ip back to $mac via $nic"; + + probe_utils->send_msg("stdout", "d", "[$mac] $record") if ($monitor); + push(@{ $node_state_ref->{$mac}{log} }, $log_ref->{msg}) if ($debug); + $ipmacmap{$ip} = $mac; + $node_state_ref->{$mac}{type} = "mac"; + set_node_state($node_state_ref, $mac, $::STATE_DISCOVER_DHCP); + } elsif ($log_ref->{msg} =~ /BOOTREQUEST\s+from\s+(\w\w:\w\w:\w\w:\w\w:\w\w:\w\w).+via\s+([^:]+)(.*)/) { + my $mac = $1; + my $nic = $2; + if ($3 =~ /no dynamic leases/) { + probe_utils->send_msg("stdout", "d", "Receive DHCPDISCOVER from $mac via $nic, no free leases") if ($monitor); + return 0; + } + my $record = "Receive BOOTREQUEST from $mac via $nic"; + probe_utils->send_msg("stdout", "d", "[$mac] $record") if ($monitor); + push(@{ $node_state_ref->{$mac}{log} }, $log_ref->{msg}) if ($debug); + } elsif ($log_ref->{msg} =~ /BOOTREPLY\s+for\s+(.+)\s+to\s+.+(\w\w:\w\w:\w\w:\w\w:\w\w:\w\w).+via\s+(.+)/) { + my $ip = $1; + my $mac = $2; + my $nic = $3; + my $record = "Send BOOTREPLY on $ip back to $mac via $nic"; + + probe_utils->send_msg("stdout", "d", "[$mac] $record") if ($monitor); + push(@{ $node_state_ref->{$mac}{log} }, $log_ref->{msg}) if ($debug); + $ipmacmap{$ip} = $mac; + $node_state_ref->{$mac}{type} = "mac"; + set_node_state($node_state_ref, $mac, $::STATE_DISCOVER_DHCP); + } +} + +#------------------------------------------ + +=head3 + Description: + Handle one line TFTP log + Arguments: + log_ref: (input attribute) the reference of hash which save one line TFTP log. + node_state_ref: (output attribute), the reference of hash "%node_state". refer to function "init_node_state" for the structure of "%node_state" + Returns: + NULL +=cut + +#------------------------------------------ +sub handle_tftp_msg { + my $log_ref = shift; + my $node_state_ref = shift; + + if ($log_ref->{msg} =~ /RRQ\s+from\s+(.+)\s+filename\s+(.+)/i) { + my $ip = $1; + my $file = $2; + my $mac = $ipmacmap{$ip}; + + if (exists($node_state_ref->{$mac})) { + my $record = "Via TFTP $ip download $file"; + probe_utils->send_msg("stdout", "d", "[$mac] $record") if ($monitor); + push(@{ $node_state_ref->{$mac}{log} }, $log_ref->{msg}) if ($debug); + + if ($file =~ /\/pxelinux.cfg\//i or $file =~ /\/xcat\/xnba\/nets\//i) { + set_node_state($node_state_ref, $mac, $::STATE_DISCOVER_BOOTLODER); + } elsif ($file =~ /genesis\.kernel/i) { + set_node_state($node_state_ref, $mac, $::STATE_DISCOVER_KERNEL); + } elsif ($file =~ /genesis\.fs/i) { + set_node_state($node_state_ref, $mac, $::STATE_DISCOVER_INITRD); + } + } + } +} + +#------------------------------------------ + +=head3 + Description: + Handle one line HTTP log + Arguments: + log_ref: (input attribute) the reference of hash which save one line HTTP log. + node_state_ref: (output attribute), the reference of hash "%node_state". refer to function "init_node_state" for the structure of "%node_state" + Returns: + NULL +=cut + +#------------------------------------------ +sub handle_http_msg { + my $log_ref = shift; + my $node_state_ref = shift; + my $ip = $log_ref->{sender}; + my $mac = $ipmacmap{$ip}; + + if (exists($node_state_ref->{$mac})) { + if ($log_ref->{msg} =~ /GET\s+(.+)\s+HTTP.+/ or $log_ref->{msg} =~ /HEAD\s+(.+)\s+HTTP.+/) { + my $file = $1; + my $record = "Via HTTP $ip get $file"; + probe_utils->send_msg("stdout", "d", "[$mac] $record") if ($monitor); + push(@{ $node_state_ref->{$mac}{log} }, $log_ref->{msg}) if ($debug); + + if ($file =~ /\/pxelinux.cfg\//i or $file =~ /\/xcat\/xnba\/nets\//i) { + set_node_state($node_state_ref, $mac, $::STATE_DISCOVER_BOOTLODER); + } elsif ($file =~ /genesis\.kernel/i) { + set_node_state($node_state_ref, $mac, $::STATE_DISCOVER_KERNEL); + } elsif ($file =~ /genesis\.fs/i) { + set_node_state($node_state_ref, $mac, $::STATE_DISCOVER_INITRD); + } + } + } +} + +#------------------------------------------ + +=head3 + Description: + Handle one line log comes from cluster.log + Arguments: + log_ref: (input attribute) the reference of hash which save one line log comes from cluster.log. + node_state_ref: (output attribute), the reference of hash "%node_state". refer to function "init_node_state" for the structure of "%node_state" + Returns: + NULL +=cut + +#------------------------------------------ +sub handle_cluster_msg { + my $log_ref = shift; + my $node_state_ref = shift; + my $log_msg = $log_ref->{msg}; + + if ($log_ref->{msg} =~ /xcat.discovery.(.+): \((.+)\) Found node: (.+)/) { + my $mac = $2; + my $node = $3; + $node_state_ref->{$mac}{id} = $node; + $node_state_ref->{$node}{id} = $mac; + my $record = "Found node $node"; + probe_utils->send_msg("stdout", "d", "[$mac] $record") if ($monitor); + push(@{ $node_state_ref->{$mac}{log} }, $log_ref->{msg}) if ($debug); + } +} + +#------------------------------------------ + +=head3 + Description: + Handle one line log comes from computes.log + Arguments: + log_ref: (input attribute) the reference of hash which save one line log comes from computes.log. + node_state_ref: (output attribute), the reference of hash "%node_state". refer to function "init_node_state" for the structure of "%node_state" + Returns: + NULL +=cut + +#------------------------------------------ +sub handle_compute_msg { + my $log_ref = shift; + my $node_state_ref = shift; + my $ip = $log_ref->{sender}; + my $mac = $ipmacmap{$ip}; + + if (exists $node_state_ref->{$mac}) { + probe_utils->send_msg("stdout", "d", "[$mac] ($ip) $log_ref->{msg}") if ($monitor); + push(@{ $node_state_ref->{$mac}{log} }, $log_ref->{msg}) if ($debug); + + if ($log_ref->{label} == $::LOGLABEL_DOXCAT) { + set_node_state($node_state_ref, $mac, $::STATE_DISCOVER_DOXCAT); + } elsif ($log_ref->{label} == $::LOGLABEL_DISCOVERY) { + if ($log_ref->{msg} =~ /Beginning node discovery process.../i) { + set_node_state($node_state_ref, $mac, $::STATE_DISCOVER_DISCOVERY); + } elsif ($log_ref->{msg} =~ /Sending the discovery packet to xCAT/i) { + set_node_state($node_state_ref, $mac, $::STATE_DISCOVER_REPORT); + } elsif ($log_ref->{msg} =~ /Restart network interfaces.../i) { + my $node = ""; + if (exists ($node_state_ref->{$mac}{id})) { + $node = $node_state_ref->{$mac}{id}; + } else { + $node = `lsdef -i mac -c 2>&1 | awk -F: '/$mac/ {print \$1}'`; + chomp($node); + $node_state_ref->{$mac}{id} = $node; + $node_state_ref->{$node}{id} = $mac; + } + if ($node ne "") { + $node_state_ref->{$node}{done} = 1; + probe_utils->send_msg("stdout", "o", "[$mac] node $node discovery completed") if ($monitor); + set_node_state($node_state_ref, $mac, $::STATE_DISCOVER_COMPLETED); + } + } + } + } +} + +#------------------------------------------ + +=head3 + Description: + Set node state in hash %node_state + Arguments: + node_state_ref: (input/output attribute), the reference of hash "%node_state". refer to function "init_node_state" for the structure of "%node_state" + node : (input attribute) The node name + newstate : (input attribute) The new state of node + Returns: + NULL +=cut + +#------------------------------------------ +sub set_node_state { + my $node_state_ref = shift; + my $node = shift; + my $newstate = shift; + + if ($newstate == $::STATE_DISCOVER_BOOTLODER) { + pop(@{ $node_state_ref->{$node}{statehistory} }); + push @{ $node_state_ref->{$node}{allstatehistory} }, @{ $node_state_ref->{$node}{statehistory} }; + @{ $node_state_ref->{$node}{statehistory} } = (); + push @{ $node_state_ref->{$node}{statehistory} }, $::STATE_DISCOVER_DHCP; + push @{ $node_state_ref->{$node}{statehistory} }, $newstate; } else { - print "$::USAGE"; - } - exit 0; -} + my $index = @{ $node_state_ref->{$node}{statehistory} } - 1; -if ($test) { - probe_utils->send_msg("$output", "o", "Probe the discovery process, including pre-check for required configuration and realtime monitor of discovery process. Before using this command, please install 'nslookup' command. Currently, this command does not support hierarchy."); - exit 0; -} - -if (defined($noderange) && !defined($discovery_type)) { - probe_utils->send_msg("$output", "f", "Option '-n' must used with '-m'"); - probe_utils->send_msg("$output", "d", "$::USAGE"); - exit 1; -} - -if (defined($discovery_type)) { - unless (grep(/^$discovery_type$/, @valid_discovery_type)) { - probe_utils->send_msg("$output", "f", "Invalid discovery type. the vaild types are $valid_discovery_type_str"); - probe_utils->send_msg("$output", "d", "$::USAGE"); - exit 1; + if ($node_state_ref->{$node}{statehistory}->[$index] != $newstate) { + push @{ $node_state_ref->{$node}{statehistory} }, $newstate; + } } } -if (defined($noderange) && defined($discovery_type)) { - $msg = "All pre_defined nodes are valid"; - my $rc = check_pre_defined_node($discovery_type, $noderange); - if ($rc) { - probe_utils->send_msg("$output", "f", $msg); - exit 1; +#------------------------------------------ + +=head3 + Description: + Check if all node have been finished the discovery process + Arguments: + node_state_ref: The reference of hash "%node_state". refer to function "init_node_state" for the structure of "%node_state" + Returns: + 0: success + 1: failed +=cut + +#------------------------------------------ +sub all_monitor_node_done { + my $node_state_ref = shift; + my $done = 1; + + foreach my $node (keys %$node_state_ref) { + if (($node_state_ref->{$node}{type} eq "node") and ($node_state_ref->{$node}{done} == 0)) { + $done = 0; + last; + } + } + + return $done; +} + +#------------------------------------------ + +=head3 + Description: + Calculate the discovery of every node. offer a report to customer + Arguments: + node_state_ref: The reference of hash "%node_state". refer to function "init_node_state" for the structure of "%node_state" + Returns: + 0: success + 1: failed +=cut + +#------------------------------------------ +sub conclusion_report { + my $node_state_ref = shift; + + probe_utils->send_msg("stdout", "", "==================discovery_probe_report================="); + + if ($debug) { + print "---->the result of %node_state<------\n"; + print Dumper $node_state_ref; + } + + if ($verbose) { + probe_utils->send_msg("stdout", "d", "----------node state history----------"); + foreach my $identify (keys %$node_state_ref) { + if ($node_state_ref->{$identify}{type} eq "mac") { + my $allhistorystate; + my $historystate; + probe_utils->send_msg("stdout", "d", "[$identify]:"); + if (@{ $node_state_ref->{$identify}{allstatehistory} }) { + $allhistorystate .= "$::STATE_DISCOVER_DESC{$_}=>" foreach (@{ $node_state_ref->{$identify}{allstatehistory} }); + $allhistorystate =~ s/=>$//g; + probe_utils->send_msg("stdout", "d", "Setps executed prior to last discoverying attempt:"); + probe_utils->send_msg("stdout", "d", "$allhistorystate"); + } + + $historystate .= "$::STATE_DISCOVER_DESC{$_}=>" foreach (@{ $node_state_ref->{$identify}{statehistory} }); + $historystate =~ s/=>$//g; + probe_utils->send_msg("stdout", "d", "Steps executed for last discoverying attempt:"); + probe_utils->send_msg("stdout", "d", "$historystate"); + + if (exists($node_state_ref->{$identify}{id})) { + probe_utils->send_msg("stdout", "d", "Node $node_state_ref->{$identify}{id} matched"); + } + } + } + + probe_utils->send_msg("stdout", "d", "--------------------------------------"); + } + + my %failed_mac; + my @success_node; + my @failed_node; + foreach my $identify (keys %$node_state_ref) { + if ($node_state_ref->{$identify}{type} eq "node") { + my $mac = $node_state_ref->{$identify}{id}; + if ($mac) { + push @success_node, $identify; + } else { + push @failed_node, $identify; + } + } elsif ($node_state_ref->{$identify}{type} eq "mac") { + foreach (@{ $node_state_ref->{$identify}{statehistory} }) { + $stop_stage = $_ if ($stop_stage < $_); + } + if ($stop_stage != $::STATE_DISCOVER_COMPLETED) { + $failed_mac{$identify}{stop_point} = $stop_stage; + } + } + } + + if (@failed_node) { + if (@success_node) { + my $success_node_num = @success_node; + probe_utils->send_msg("stdout", "o", "Found $success_node_num node(s) matched successfully"); + } + + my $failed_node_num = @failed_node; + my $failed_nodes = join(",", @failed_node); + if ($failed_node_num > 1) { + probe_utils->send_msg("stdout", "d", "There are $failed_node_num nodes matched failures: $failed_nodes"); + } elsif ($failed_node_num == 1) { + probe_utils->send_msg("stdout", "d", "There is $failed_node_num node matched failures: $failed_nodes"); + } + + foreach my $mac (keys %failed_mac) { + probe_utils->send_msg("stdout", "f", "[$mac] : stop at stage '$::STATE_DISCOVER_DESC{$failed_mac{$mac}{stop_point}}'"); + } } else { - probe_utils->send_msg("$output", "o", $msg); + probe_utils->send_msg("stdout", "o", "All nodes matched successfully"); } + + return 0; } -if (!$no_pre_check) { - $rst = do_pre_check(); - exit 1 if ($rst); -} -$rst = do_monitor(); -exit $rst;