From 6e891d066d03b8420c243b035e1fbc6f90e25bac Mon Sep 17 00:00:00 2001 From: hu-weihua Date: Wed, 2 Nov 2016 05:19:50 -0400 Subject: [PATCH 1/3] Refine probe tool 'xcatmn' --- xCAT-probe/subcmds/xcatmn | 1619 ++++++++++++++++++------------------- 1 file changed, 777 insertions(+), 842 deletions(-) diff --git a/xCAT-probe/subcmds/xcatmn b/xCAT-probe/subcmds/xcatmn index 6890bfc8f..f09e111b9 100755 --- a/xCAT-probe/subcmds/xcatmn +++ b/xCAT-probe/subcmds/xcatmn @@ -5,45 +5,43 @@ BEGIN { $::XCATROOT = $ENV{'XCATROOT'} ? $ENV{'XCATROOT'} : -d '/opt/xcat' ? '/o use lib "$::XCATROOT/probe/lib/perl"; use probe_utils; +use hierarchy; use xCAT::ServiceNodeUtils; use xCAT::NetworkUtils; use File::Basename; -use Getopt::Long qw(:config no_ignore_case); -use IO::Select; use Data::Dumper; +use Getopt::Long qw(:config no_ignore_case); -my $program_name = basename("$0"); #current sub_command name my $help = 0; #command line attribute '-h', get usage information my $test = 0; #command line attribute '-T' +my $hierarchy = 0; my $verbose = 0; #command line attribute '-V' my $noderange; #command line attribute '-n' -my $installnic; my $output = "stdout"; #used by probe_utils->send_msg("$output", "o", "xxxxxxxxxx"); print output to STDOUT -my $is_sn = 0; #flag current server is SN my $rst = 0; #the exit code of current command my $terminal = 0; #means get INT signal from STDIN -my %summaryoutput; #save all output from commands running on SNs and MN +my $installnic; -#a map of SNs and command which will be dispatched to current SN -# $dispatchcmd{snname} = "command" -my %dispatchcmd; +#save all output from commands running on SNs and MN +# one example: +# $summaryoutput{mn} = @mn_output_history +# $summaryoutput{SN1} = @SN1_output_history +my %summaryoutput; -#save command line attributes from STDIN -my @tmpargv; +my $is_sn; +$is_sn = 1 if (-e "/etc/xCATSN"); -#-------------------------------- -# below are some options rules used by default -# -h : Get usage information of current sub command -# -V : Output more information for debug -# -T : To verify if $program_name can work, reserved option for probe framework, not for use by customer -# -n : In xCAT probe, -n is used to specify node range uniformly -#-------------------------------- + +#------------------------------------- +# Usage +#------------------------------------- +my $program_name = basename("$0"); #current sub_command name $::USAGE = "Usage: $program_name -h - $program_name [-i ] [-V] + $program_name -i [-V] Description: - After xcat installation, use this command to check if xcat has been installed correctly and is ready for use. + After xCAT installation, use this command to check if xCAT has been installed correctly and is ready for use. For hierarchical cluster, only the provision network on the same network as the management node is supported. If in the different network, ignore the results. Options: @@ -52,626 +50,142 @@ Options: -i : Required. Specify the network interface name of provision network on management node "; - -sub returncmdoutput { - my $rst = shift; - chomp($rst); - my $outputtarget = shift; - my @lines = split("[\n\r]", $rst); - foreach my $line (@lines) { - probe_utils->send_msg("$outputtarget", "d", "$line"); - } -} - -#------------------------------------- -# TWO FUNCTIONS MUST BE IMPLEMENTED BY EACH SUB COMMAND -# They are do_main_job and summary_all_jobs_output -#------------------------------------- - -#------------------------------------ -# Please implement the main checking job of current command in do_main_job function -# If $outputtarget has input value, that means do_main_job is running on MN, so every message needed to print on STDOUT should be written into pipe $outputtarget. -# If $outputtarget has no value, that means do_main_job is running on SN, all message just need to print on STDOUT -# Recommand to use probe_utils->send_msg() to handle message you plan to print out -#------------------------------------ sub do_main_job { - my $outputtarget = shift; - $outputtarget = "stdout" if (!$outputtarget); my $rst = 0; + my @error = (); + my $checkpoint; + my $rc = 0; + my $installnicip; - my $msg; - my $serverip; + #check if all xcat deamons are running + $checkpoint = "All xCAT deamons are running"; + $rst = check_all_xcat_deamons(\@error); + print_check_result($checkpoint, "f", $rst, \@error); + return $rst if($rst); - $msg = "Sub process 'xcatd: SSL listener' is running"; - my $xcatdproc = `ps aux|grep -v grep|grep xcatd`; - chomp($xcatdproc); - if ($xcatdproc =~ /xcatd: SSL listener/) { - probe_utils->send_msg($outputtarget, "o", "$msg"); - } else { - probe_utils->send_msg($outputtarget, "f", "$msg"); - $rst = 1; + #check if xcatd can receive request + $checkpoint = "xcatd can receive command request"; + $rst = check_xcatd_receive_request(\@error); + print_check_result($checkpoint, "f", $rst, \@error); + return $rst if($rst); + + #check 'site' table configuratiions + my %sitetable; + $checkpoint = "'site' table is configured correctly"; + $rst = check_site_table(\%sitetable,\@error); + print_check_result($checkpoint, "f", $rst, \@error); + return $rst if($rst); + + #check network configuratiions + $checkpoint = "Provision network is configured correctly"; + $rst = check_network(\%sitetable,\$installnicip,\@error); + print_check_result($checkpoint, "f", $rst, \@error); + return $rst if($rst); + + #check 'passwd' table configuratiions + $checkpoint = "'passwd' table is configured correctly"; + $rst = check_passwd_table(\@error); + print_check_result($checkpoint, "f", $rst, \@error); + $rc |= $rst; + + #check important directory + $checkpoint = "Important directory is configured correctly"; + $rst = check_directory(\%sitetable,\@error); + print_check_result($checkpoint, "f", $rst, \@error); + $rc |= $rst; + + #check if SElinux is disabled + $checkpoint = "SELinux is disabled on current server"; + $rst = check_selinux(\@error); + print_check_result($checkpoint, "f", $rst, \@error); + $rc |= $rst; + + #check http service + $checkpoint = "HTTP service works well"; + $rst = check_http_service($installnicip, \@error); + print_check_result($checkpoint, "f", $rst, \@error); + $rc |= $rst; + + #check tftp service + $checkpoint = "TFTP service works well"; + $rst = check_tftp_service($installnicip,\@error); + print_check_result($checkpoint, "f", $rst, \@error); + $rc |= $rst; + + #check DNS service + $checkpoint = "DNS service works well"; + $rst = check_dns_service(\%sitetable, $installnicip,\@error); + print_check_result($checkpoint, "f", $rst, \@error); + $rc |= $rst; + + #check DHCP service + $checkpoint = "DHCP service works well"; + $rst = check_dhcp_service($installnicip, \@error); + print_check_result($checkpoint, "f", $rst, \@error); + $rc |= $rst; + + #Below are the 'warning` level check points + + #check if firewall is close + $checkpoint = "Firewall is closed on current server"; + $rst = check_firewall(\@error); + print_check_result($checkpoint, "w", $rst, \@error); + $rc |= $rst; + + #check disk space + $checkpoint = "The disk space is enough for xCAT to work"; + $rst = check_disk(\@error); + print_check_result($checkpoint, "w", $rst, \@error); + $rc |= $rst; + + + #some sepecific check points in MN + if(! $is_sn){ + #check if server ip is a static ip in MN + $checkpoint = "The IP of master is a static IP address"; + $rst = check_server_ip_static($installnicip,\@error); + print_check_result($checkpoint, "w", $rst, \@error); + $rc |= $rst; + + #check if dhcpd.leases is less than 100M + $checkpoint = "The dhcpd.leases file is less than 100M"; + $rst = check_dhcp_leases(\@error); + print_check_result($checkpoint, "w", $rst, \@error); + $rc |= $rst; } - $msg = "Sub process 'xcatd: DB Access' is running"; - if ($xcatdproc =~ /xcatd: DB Access/) { - probe_utils->send_msg($outputtarget, "o", "$msg"); - } else { - probe_utils->send_msg($outputtarget, "f", "$msg"); - $rst = 1; - } - - $msg = "Sub process 'xcatd: UDP listener' is running"; - if ($xcatdproc =~ /xcatd: UDP listener/) { - probe_utils->send_msg($outputtarget, "o", "$msg"); - } else { - probe_utils->send_msg($outputtarget, "f", "$msg"); - $rst = 1; - } - - $msg = "Sub process 'xcatd: install monitor' is running"; - if ($xcatdproc =~ /xcatd: install monitor/) { - probe_utils->send_msg($outputtarget, "o", "$msg"); - } else { - probe_utils->send_msg($outputtarget, "f", "$msg"); - $rst = 1; - } - - $msg = "Sub process 'xcatd: Discovery worker' is running"; - if ($xcatdproc =~ /xcatd: Discovery worker/) { - probe_utils->send_msg($outputtarget, "o", "$msg"); - } else { - probe_utils->send_msg($outputtarget, "f", "$msg"); - $rst = 1; - } - - $msg = "Sub process 'xcatd: Command log writer' is running"; - if ($xcatdproc =~ /xcatd: Command log writer/) { - probe_utils->send_msg($outputtarget, "o", "$msg"); - } else { - probe_utils->send_msg($outputtarget, "w", "Sub process 'xcatd: Command log writer' isn't running"); - } - return 1 if ($rst); - - my $xcatdport = `lsdef -t site -i xcatdport -c | awk -F'=' '{print \$2}'`; - chomp($xcatdport); - probe_utils->send_msg($outputtarget, "d", "The port used by the xcatd daemon for client/server communication is $xcatdport") if ($verbose); - $msg = "xcatd is listening on port $xcatdport"; - my $cmdoutput = `netstat -ant|grep LISTEN|grep $xcatdport`; - if ($?) { - probe_utils->send_msg($outputtarget, "f", "$msg"); - $rst = 1; - } else { - probe_utils->send_msg($outputtarget, "o", "$msg"); - } - - my $xcatiport = `lsdef -t site -i xcatiport -c | awk -F'=' '{print \$2}'`; - chomp($xcatiport); - probe_utils->send_msg($outputtarget, "d", "The port used by xcatd to receive install status updates from nodes is $xcatiport") if ($verbose); - $msg = "xcatd is listening on port $xcatiport"; - $cmdoutput = `netstat -antp | grep -i xcatd|grep LISTEN|grep $xcatiport`; - if ($?) { - probe_utils->send_msg($outputtarget, "f", "$msg"); - $rst = 1; - } else { - probe_utils->send_msg($outputtarget, "o", "$msg"); - } - return 1 if ($rst); - - $msg = "'lsxcatd -a' works"; - $cmdoutput = `lsxcatd -a 2>&1`; - $rst = $?; - returncmdoutput($cmdoutput, $outputtarget) if ($verbose); - if ($rst) { - probe_utils->send_msg($outputtarget, "f", "$msg"); - return $rst; - } else { - probe_utils->send_msg($outputtarget, "o", "$msg"); - } - - my $masteripinsite = `lsdef -t site -i master -c | awk -F'=' '{print \$2}'`; - chomp($masteripinsite); - probe_utils->send_msg($outputtarget, "d", "The value of 'master' in 'site' table is $masteripinsite") if ($verbose); - probe_utils->send_msg($outputtarget, "f", "There isn't 'master' definition in 'site' table") if ($masteripinsite eq ""); - - $msg = "The value of 'master' in 'site' table is an IP address"; - if (probe_utils->is_ip_addr("$masteripinsite")) { - probe_utils->send_msg($outputtarget, "o", "$msg"); - } else { - probe_utils->send_msg($outputtarget, "f", "$msg"); - return 1; - } - - if (!$is_sn) { - - # on MN, check the validity of installnic and get ip address of the NIC - $msg = "NIC $installnic exists on current server"; - my $nics = `ip addr show $installnic >/dev/null 2>&1`; - if ($?) { - probe_utils->send_msg($outputtarget, "f", "$msg"); - probe_utils->send_msg($outputtarget, "d", "Use 'ip addr show' to check if there is NIC named $installnic on current server"); - return 1; - } else { - probe_utils->send_msg($outputtarget, "o", "$msg"); - } - - $msg = "Get IP address of NIC $installnic"; - $serverip = `ip addr show $installnic | awk -F" " '/inet / {print \$2}'|awk -F"/" '{print \$1}'`; - chomp($serverip); - if (!defined($serverip) || ($serverip eq "")) { - probe_utils->send_msg($outputtarget, "f", "$msg"); - probe_utils->send_msg($outputtarget, "d", "Use 'ip addr show' to check if there is IP assigned to $installnic"); - return 1; - } else { - probe_utils->send_msg($outputtarget, "d", "The IP of NIC $installnic is $serverip") if ($verbose); - probe_utils->send_msg($outputtarget, "o", "$msg"); - } - - if ($serverip) { - $msg = "The IP $serverip of $installnic equals the value of 'master' in 'site' table"; - if ($serverip eq $masteripinsite) { - probe_utils->send_msg($outputtarget, "o", "$msg"); - } else { - probe_utils->send_msg($outputtarget, "f", "$msg"); - $rst = 1; - } - } - - $msg = "IP $serverip of NIC $installnic is a static IP on current server"; - if (probe_utils->is_static_ip("$serverip", "$installnic")) { - probe_utils->send_msg($outputtarget, "o", "$msg"); - } else { - probe_utils->send_msg($outputtarget, "w", "IP $serverip of $installnic is not a static IP on current server"); - } - } else { - - # on SN, get ip address by compare 'master' attribute in 'site' table - # choose the one in the same network with 'master' - my @ipoutput = `ip addr show | grep inet | grep -v inet6 2>&1`; - - foreach (@ipoutput) { - if ($_ =~ /inet\s+(.+)\/(.+)\s+brd\s+(.+)\s+scope global/i) { - if (xCAT::NetworkUtils::isInSameSubnet($masteripinsite, $1, $2, 1)) { - $serverip = $1; - } - } - } - - $msg = "Get IP address that on the same network as master $masteripinsite"; - if (!defined($serverip) || ($serverip eq "")) { - probe_utils->send_msg($outputtarget, "f", "$msg"); - return 1; - } else { - probe_utils->send_msg($outputtarget, "d", "The IP is $serverip") if ($verbose); - probe_utils->send_msg($outputtarget, "o", "$msg"); - } - } - - $msg = "$serverip belongs to one of networks defined in 'networks' table"; - my $networks = `tabdump networks|grep -v "^#"`; - $networks =~ s/\"//g; - my $netcnt = `echo "$networks"|wc -l`; - my $hit = 0; - for (my $i = 1 ; $i < $netcnt + 1 ; $i++) { - my $line = `echo "$networks" |sed -n ${i}p |awk -F"," '{print \$2,\$3,\$4}'`; - chomp($line); - if ($line =~ /(.+) (.+) (.+)/) { - if (!$is_sn) { - $hit = 1 if (probe_utils->is_ip_belong_to_net("$1", "$2", $serverip) && ("$3" eq "$installnic")); - } else { - $hit = 1 if (probe_utils->is_ip_belong_to_net("$1", "$2", $serverip)); - } - } - } - if ($hit) { - probe_utils->send_msg($outputtarget, "o", "$msg"); - } else { - probe_utils->send_msg($outputtarget, "f", "$msg"); - $rst = 1; - } - - $msg = "There is domain definition in 'site' table"; - my $domain = `lsdef -t site -i domain -c | awk -F'=' '{print \$2}'`; - chomp($domain); - if ($domain) { - probe_utils->send_msg($outputtarget, "d", "The value of 'domain' in 'site' table is $domain") if ($verbose); - probe_utils->send_msg($outputtarget, "o", "$msg"); - } else { - probe_utils->send_msg($outputtarget, "f", "$msg"); - $rst = 1; - } - - $msg = "There is a configuration in 'passwd' table for 'system' for node provisioning"; - my $passwd = `tabdump passwd |awk -F',' '/system/ { gsub(/"/, "", \$2); gsub(/"/, "", \$3); print \$2,\$3 }'`; - chomp($passwd); - my ($username, $pw) = split(" ", $passwd); - if ($username eq "" || $pw eq "") { - probe_utils->send_msg($outputtarget, "f", "$msg"); - probe_utils->send_msg($outputtarget, "d", "Define username and password for 'system' in 'passwd' table"); - $rst = 1; - } else { - probe_utils->send_msg($outputtarget, "o", "$msg"); - } - - my $installdir = `lsdef -t site -i installdir -c | awk -F'=' '{print \$2}'`; - chomp($installdir); - probe_utils->send_msg($outputtarget, "d", "The 'install' directory is set to $installdir in 'site' table on current server") if ($verbose); - my $tftpdir = `lsdef -t site -i tftpdir -c | awk -F'=' '{print \$2}'`; - chomp($tftpdir); - probe_utils->send_msg($outputtarget, "d", "The 'tftp' directory is set to $tftpdir in 'site' table on current server") if ($verbose); - - $msg = "There is $installdir directory on current server"; - if (-e "$installdir/postscripts/") { - probe_utils->send_msg($outputtarget, "o", "$msg"); - } else { - probe_utils->send_msg($outputtarget, "f", "$msg"); - $rst = 1; - } - - if ($is_sn) { - my $mountoutput = `mount | grep '$installdir'`; - chomp($mountoutput); - - my $mountip; - if ($mountoutput =~ /(.+):$installdir on $installdir /) { - my $mountsource = $1; - if (xCAT::NetworkUtils->isIpaddr($mountsource)) { - $mountip = $mountsource; - } else { - $mountip = xCAT::NetworkUtils->getipaddr($mountsource); - } - } - - $msg = "installdir $installdir is mounted from the management node"; - if ($mountip eq $masteripinsite) { - probe_utils->send_msg($outputtarget, "o", "$msg"); - } - else { - probe_utils->send_msg($outputtarget, "f", "$msg"); - $rst = 1; - } - } - - $msg = "There is $tftpdir directory on current server"; - if (-e "$tftpdir") { - probe_utils->send_msg($outputtarget, "o", "$msg"); - } else { - probe_utils->send_msg($outputtarget, "f", "$msg"); - $rst = 1; - } - - if ($is_sn) { - my $mountoutput = `mount | grep '$tftpdir'`; - chomp($mountoutput); - - my $mountip; - if ($mountoutput =~ /(.+):$tftpdir on $tftpdir /) { - my $mountsource = $1; - if (xCAT::NetworkUtils->isIpaddr($mountsource)) { - $mountip = $mountsource; - } else { - $mountip = xCAT::NetworkUtils->getipaddr($mountsource); - } - } - - $msg = "tftpdir $tftpdir is mounted from the management node"; - if ($mountip eq $masteripinsite) { - probe_utils->send_msg($outputtarget, "o", "$msg"); - } - else { - probe_utils->send_msg($outputtarget, "f", "$msg"); - $rst = 1; - } - } - - #check the free space of specific directory - #if "/var" is mounted on standalone disk, more than 1G free space is expected - #if "/tmp" is mounted on standalone disk, more than 1G free space is expected - #if installdir is mounted on standalone disk, more than 10G free space is expected. - #if any one of above three directories hasn't standalone disk, "/" directory should cover its space requirement. - my @dir_expectedspace_list = (["/var", "1"], ["/tmp", "1"], ["$installdir", "10"], ["/" , "0"]); - foreach my $dir (@dir_expectedspace_list){ - next if($dir->[0] eq "/" && $dir->[1] == 0); - my $checkrst = probe_utils->is_dir_has_enough_space($dir->[0], $dir->[1]); - if($checkrst == 2){ - $dir_expectedspace_list[$#dir_expectedspace_list][1] += $dir->[1]; - }elsif($checkrst == 1){ - probe_utils->send_msg($outputtarget, "o", "The free space of '$dir->[0]' directory is more than $dir->[1] G"); - }elsif($checkrst == 0){ - probe_utils->send_msg($outputtarget, "w", "The free space of '$dir->[0]' is less than $dir->[1] G"); - } - - } - - $msg = "SELinux is disabled on current server"; - if (probe_utils->is_selinux_enable()) { - probe_utils->send_msg($outputtarget, "f", "$msg"); - $rst = 1; - } else { - probe_utils->send_msg($outputtarget, "o", "$msg"); - } - - $msg = "Firewall is closed on current server"; - if (probe_utils->is_firewall_open()) { - probe_utils->send_msg($outputtarget, "w", "Firewall is configured on current server"); - } else { - probe_utils->send_msg($outputtarget, "o", "$msg"); - } - - `which wget > /dev/null 2>&1`; - if ($?) { - probe_utils->send_msg($outputtarget, "w", "'wget' tool isn't installed, skip checking HTTP service. Install 'wget' then try again"); - } else { - $msg = "HTTP service is ready on $serverip"; - if (probe_utils->is_http_ready("$serverip")) { - probe_utils->send_msg($outputtarget, "o", "$msg"); - } else { - probe_utils->send_msg($outputtarget, "f", "$msg"); - $rst = 1; - } - } - - my $nodename = `hostname -s`; - chomp($nodename); - - # For sn, 'setuptftp' attribute could be set to '0' or '1'. - # if '0', sn does not need to provie TFTP service, will not check it - my $checktftp = 1; - if ($is_sn) { - $checktftp = `lsdef $nodename -i setuptftp -c | awk -F'=' '{print \$2}'`; - chomp($checktftp); - unless ($checktftp) { - probe_utils->send_msg($outputtarget, "d", "SN $nodename is not set to provide TFTP service"); - } - } - if ($checktftp) { - `which tftp > /dev/null 2>&1`; - if ($?) { - probe_utils->send_msg($outputtarget, "w", "'tftp' tool isn't installed, skip checking tftp service. Install 'tftp' then try again"); - } else { - $msg = "TFTP service is ready on $serverip"; - if (probe_utils->is_tftp_ready("$serverip")) { - probe_utils->send_msg($outputtarget, "o", "$msg"); - } else { - probe_utils->send_msg($outputtarget, "f", "$msg"); - $rst = 1; - } - } - } - - # For sn, 'setupdns' attribute could be set to '0' or '1'. - # if '0', sn does not need to provie DNS service, will not check it - my $checkdns = 1; - if ($is_sn) { - $checkdns = `lsdef $nodename -i setupnameserver -c | awk -F'=' '{print \$2}'`; - chomp($checkdns); - unless ($checkdns) { - probe_utils->send_msg($outputtarget, "d", "SN $nodename is not set to provide DNS service"); - } - } - - if ($checkdns) { - `which nslookup > /dev/null 2>&1`; - if ($?) { - probe_utils->send_msg($outputtarget, "w", "'nslookup' tool isn't installed, skip checking DNS service. Install 'nslookup' then try again"); - } else { - $msg = "DNS server is ready on $serverip"; - probe_utils->send_msg($outputtarget, "d", "Domain used to check DNS is $domain") if ($verbose); - - my $rc = 0; - if (!$is_sn) { - - # if this is a hierarchical cluster, nslookup one of sn to check DNS service - my @snlist = xCAT::ServiceNodeUtils->getAllSN(); - my $sntmp = shift(@snlist); - if ($sntmp) { - my $sninfo = `cat /etc/hosts | grep $sntmp`; - if ($sninfo =~ /(\d+).(\d+).(\d+).(\d+)/) { - my $snip = "$1.$2.$3.$4"; - if (!probe_utils->is_dns_ready("$snip", "$serverip", "$sntmp", "$domain")) { - probe_utils->send_msg("$outputtarget", "d", "nslookup $sntmp $snip failed") if($verbose); - $rc = 1; - } - } - } else { - - # if there is no sn, nslookup mnip - my $nslkp = `nslookup $serverip $serverip 2>&1`; - chomp($nslkp); - my $tmp = grep {$_ =~ "Server:[\t\s]*$serverip"} split (/\n/, $nslkp); - if (!$tmp) { - probe_utils->send_msg($outputtarget, "d", "nslookup $serverip $serverip failed") if($verbose); - $rc = 1; - } - } - if ($rc) { - probe_utils->send_msg($outputtarget, "f", "$msg"); - $rst = 1; - } else { - probe_utils->send_msg($outputtarget, "o", "$msg"); - } - } else { - - # on sn, nslookup it's ip to check DNS service - if (!probe_utils->is_dns_ready("$serverip", "$masteripinsite", "$nodename", "$domain")) { - probe_utils->send_msg($outputtarget, "d", "nslookup $serverip failed"); - probe_utils->send_msg($outputtarget, "f", "$msg"); - } - else { - probe_utils->send_msg($outputtarget, "o", "$msg"); - } - } - } - } - - # For sn, 'setupdhcp' attribute could be set to '0' or '1'. - # if '0', sn does not need to provie DHCP service, will not check it - my $checkdhcp = 1; - my $rc = 0; - if ($is_sn) { - $checkdhcp = `lsdef $nodename -i setupdhcp -c | awk -F'=' '{print \$2}'`; - chomp($checkdhcp); - if ($checkdhcp) { - - # on sn, just check dhcpd service whether running - $msg = "DHCP service is ready on $serverip"; - my $dhcpoutput = `ps aux | grep dhcpd |grep -v grep`; - if ($dhcpoutput) { - probe_utils->send_msg($outputtarget, "o", "$msg"); - } else { - probe_utils->send_msg($outputtarget, "f", "$msg"); - } - } else { - probe_utils->send_msg($outputtarget, "d", "SN $nodename is not set to provide DHCP service"); - } - } else { - my $leasefile = ""; - if (-e "/var/lib/dhcpd/dhcpd.leases") { - $leasefile = "/var/lib/dhcpd/dhcpd.leases"; - } elsif (-e "/var/lib/dhcp/db/dhcpd.leases") { - $leasefile = "/var/lib/dhcp/db/dhcpd.leases"; - } elsif (-e "/var/lib/dhcp/dhcpd.leases") { - $leasefile = "/var/lib/dhcp/dhcpd.leases"; - } - - $msg = "The size of $leasefile is less than 100M"; - my $filesizetmp = `du -sb $leasefile`; - if ($?) { - returncmdoutput($filesizetmp) if ($verbose); - probe_utils->send_msg($outputtarget, "f", "$msg"); - $rst = 1; - } else { - chomp($filesizetmp); - my ($size, $file) = split(" ", $filesizetmp); - probe_utils->send_msg($outputtarget, "d", "The size of $leasefile is $size byte") if ($verbose); - if ($size > 104857600) { - probe_utils->send_msg($outputtarget, "w", "The size of $leasefile is more than 100M"); - } else { - probe_utils->send_msg($outputtarget, "o", "$msg"); - } - } - - my $rc = 0; - my $msg = "DHCP service is ready on $serverip"; - - { #very important brace to create a block - my @snlist = xCAT::ServiceNodeUtils->getAllSN(); - my $sntmp = shift(@snlist); - if ($sntmp) { - my $tmp = `makedhcp -q $sntmp`; - if ($?) { - returncmdoutput($tmp, $outputtarget) if ($verbose); - probe_utils->send_msg($outputtarget, "d", "makedhcp -q $sntmp failed") if ($verbose); - $rc = 1; - last; - } - chomp($tmp); - my $snip = xCAT::NetworkUtils->getipaddr($sntmp); - my $snmac = `lsdef $sntmp -i mac -c | awk -F'=' '{print \$2}'`; - chomp ($snmac); - my $tmpmac; - if ($tmp =~ /$sntmp: ip-address = $snip, hardware-address = (.+)/) { - $tmpmac = $1; - if ($tmpmac !~ $snmac) { - returncmdoutput($tmp, $outputtarget) if ($verbose); - probe_utils->send_msg($outputtarget, "d", "DHCP server reply is wrong") if ($verbose); - $rc = 1; - } - } else { - returncmdoutput($tmp, $outputtarget) if ($verbose); - probe_utils->send_msg($outputtarget, "d", "DHCP server reply is wrong") if ($verbose); - $rc = 1; - } - } else { - - my $tmp = `chdef xcatmntest groups=all ip=$serverip mac=aa:aa:aa:aa:aa:aa`; - if ($?) { - returncmdoutput($tmp, $outputtarget) if ($verbose); - probe_utils->send_msg($outputtarget, "d", "Node simulation by 'chdef' has failed") if ($verbose); - $rc = 1; - last; - } else { - probe_utils->send_msg($outputtarget, "d", "Simulate a node xcatmntest to do dhcp test") if ($verbose); - } - - `cp /etc/hosts /etc/hosts.bak.probe > /dev/null 2>&1`; - - open HOSTFILE, ">> /etc/hosts"; - print HOSTFILE "$serverip xcatmntest xcatmntest.$domain"; - close HOSTFILE; - - probe_utils->send_msg($outputtarget, "d", "To do 'makedhcp xcatmntest'") if ($verbose); - $tmp = `makedhcp xcatmntest 2>&1`; - if ($?) { - returncmdoutput($tmp, $outputtarget) if ($verbose); - probe_utils->send_msg($outputtarget, "d", "makedhcp xcatmntest failed") if ($verbose); - $rc = 1; - `rmdef xcatmntest`; - last; - } - - probe_utils->send_msg($outputtarget, "d", "To do 'makedhcp -q xcatmntest'") if ($verbose); - $tmp = `makedhcp -q xcatmntest`; - if ($?) { - returncmdoutput($tmp, $outputtarget) if ($verbose); - probe_utils->send_msg($outputtarget, "d", "makedhcp -q xcatmntest failed") if ($verbose); - $rc = 1; - `makedhcp -d xcatmntest && rmdef xcatmntest`; - last; - } - chomp($tmp); - if ($tmp !~ /xcatmntest: ip-address = $serverip, hardware-address = aa:aa:aa:aa:aa:aa/) { - returncmdoutput($tmp, $outputtarget) if ($verbose); - probe_utils->send_msg($outputtarget, "d", "DHCP server reply is wrong") if ($verbose); - $rc = 1; - `makedhcp -d xcatmntest && rmdef xcatmntest`; - last; - } - - probe_utils->send_msg($outputtarget, "d", "Start clearing simulation information for dhcp test") if ($verbose); - $tmp = `makedhcp -d xcatmntest && rmdef xcatmntest`; - returncmdoutput($tmp, $outputtarget) if ($verbose); - - `rm /etc/hosts`; - `mv /etc/hosts.bak.probe /etc/hosts`; - } - } - if ($rc) { - probe_utils->send_msg($outputtarget, "f", "$msg"); - probe_utils->send_msg($outputtarget, "d", "Run 'makedhcp -n' if it has not been ran before."); - $rst = 1; - } else { - probe_utils->send_msg($outputtarget, "o", "$msg"); - } - } - - - return $rst; + return $rc; } -#------------------------------------- -# When this command return from all SNs and MN, you need to generate a summary -# All history outpout from SNs and MN are saved in globle hash %summaryoutput. -# $summaryoutput{mn} = @mnhistory -# $summaryoutput{snname1} = @snname1history; -# The entry in each histroy array isn't categorized, the message coming early is arranged before the one coming later. -# A simple example of how to dump %summaryoutput has been written in function -#------------------------------------- sub summary_all_jobs_output { + my $rst = 0; if ($terminal) { return 0; } - print "\n======================do summary=====================\n"; + + #DO SUMMARY DEPENDING ON YOUR SUB_COMMAND NEED + probe_utils->send_msg("$output", "d", "======================do summary====================="); + #print "summaryoutput:\n"; + #print Dumper \%summaryoutput; my @summary; push @summary, "[ok]:[MN]: Check on MN PASS."; + my $ok_section = 0; foreach my $line (@{ $summaryoutput{mn} }) { if ($line =~ /(\[failed\]\s*):\s*(.*)/) { - push @summary, "$1:\t$2"; + push @summary, "$1: $2"; $summary[0] = "[failed]:[MN]: Check on MN FAILED."; + $ok_section = 0; } elsif ($line =~ /(\[warning\]\s*):\s*(.*)/) { - push @summary, "$1:\t$2"; + push @summary, "$1: $2"; + $ok_section = 0; + } elsif($line =~ /(\[debug\]\s*):\s*(.*)/ && ! $ok_section) { + push @summary, "$1: $2"; + } elsif($line =~ /(\[ok\]\s*):\s*(.*)/) { + $ok_section=1; } + } my %summary_sn = (); @@ -679,17 +193,25 @@ sub summary_all_jobs_output { next if ($node eq "mn"); ${ $summary_sn{$node}{"rst"} } = 1; push @{ $summary_sn{$node}{"details"} }, "[ok]:[SN:$node]: Check on SN $node PASS."; + $ok_section = 0; foreach my $log (@{ $summaryoutput{$node} }) { if ($log =~ /(\[failed\]\s*):\s*(.*)/) { - push @{ $summary_sn{$node}{"details"} }, "$1:\t$2"; + push @{ $summary_sn{$node}{"details"} }, "$1: $2"; ${ $summary_sn{$node}{"rst"} } = 0; $summary_sn{$node}{"details"}[0] = "[failed]:[SN:$node]: Check on SN $node FAILED."; + $ok_section = 0; } elsif ($log =~ /(\[warning\]\s*):\s*(.*)/) { - push @{ $summary_sn{$node}{"details"} }, "$1:\t$2"; + push @{ $summary_sn{$node}{"details"} }, "$1: $2"; + $ok_section = 0; + } elsif ($log =~ /(\[debug\]\s*):\s*(.*)/ && ! $ok_section) { + push @{ $summary_sn{$node}{"details"} }, "$1: $2"; + } elsif ($log =~ /(\[ok\]\s*):\s*(.*)/){ + $ok_section = 1; } elsif ($log !~ /^(\[\w+\]\s*):\s*(.*)/) { - push @{ $summary_sn{$node}{"details"} }, "[failed]:\t$log"; + push @{ $summary_sn{$node}{"details"} }, "[failed]: $log"; ${ $summary_sn{$node}{"rst"} } = 0; $summary_sn{$node}{"details"}[0] = "[failed]:[SN:$node]: Check on SN $node FAILED."; + $ok_section = 0; } } } @@ -720,9 +242,587 @@ sub summary_all_jobs_output { } } } + + return $rst; +} + +sub print_check_result{ + my $msg = shift; + my $error_level = shift; + my $check_return_code = shift; + my $error_msg_ref = shift; + + if($check_return_code == 1){ + probe_utils->send_msg("$output", "$error_level", $msg); + probe_utils->send_msg("$output", "d", "$_") foreach (@$error_msg_ref); + }elsif($check_return_code ==2){ + return; + }else{ + probe_utils->send_msg("$output", "o", $msg); + } } +sub check_all_xcat_deamons{ + my $error_ref = shift; + + my $rst = 0; + @$error_ref = (); + my @deamon_list = ("SSL listener", + "DB Access", + "UDP listener", + "install monitor", + "Discovery worker", + "Command log writer"); + + my $output = `ps aux|grep -v grep|grep xcatd`; + foreach my $deamon (@deamon_list){ + if($output !~ /$deamon/){ + push @$error_ref, "Deamon '$deamon' isn't running"; + $rst = 1; + } + } + + return $rst; +} + +sub check_xcatd_receive_request{ + my $error_ref = shift; + my $rst = 0; + @$error_ref = (); + + #check important port + my @port_list = ("xcatdport", "xcatiport"); + foreach my $port_attr (@port_list){ + my $port = `lsdef -t site -i $port_attr -c| awk -F'=' '{print \$2}'`; + chomp($port); + if($port){ + my $cmdoutput = `netstat -ant|grep LISTEN|grep $port`; + if($?){ + push @$error_ref, "Attribute '$port_attr' in site table is set to $port, but xcatd isn't listening on $port"; + $rst=1; + } + }else{ + push @$error_ref, "Attribute '$port_attr' isn't set in 'site' table"; + $rst = 1; + } + } + return $rst if($rst); + + #send one command to test if xcatd can receive request + my $cmdoutput = `lsxcatd -a 2>&1`; + if($?){ + my @lines = split("[\n\r]", $cmdoutput); + push @$error_ref, $_ foreach (@lines); + $rst=1; + } + + return $rst; +} + +sub check_site_table{ + my $sitetable_ref=shift; + my $error_ref = shift; + + my $rst = 0; + @$error_ref = (); + my @attr_list = ("master", "domain", "installdir", "tftpdir"); + foreach my $attr (@attr_list){ + my $value; + $value = `lsdef -t site -i $attr -c | awk -F'=' '{print \$2}'`; + chomp($value); + if($value eq ""){ + push @$error_ref, "There isn't '$attr' definition in 'site' table"; + $rst=1; + }else{ + if (($attr eq "master") and (! probe_utils->is_ip_addr("$value"))) { + push @$error_ref, "The value of 'master' in 'site' table isn't an IP addres"; + $rst=1; + } + $sitetable_ref->{$attr} = $value; + } + } + + return $rst; +} + +sub check_passwd_table{ + my $error_ref = shift; + my $rst = 0; + @$error_ref = (); + + my $passwd = `tabdump passwd |awk -F',' '/system/ { gsub(/"/, "", \$2); gsub(/"/, "", \$3); print \$2,\$3 }'`; + chomp($passwd); + my ($username, $pw) = split(" ", $passwd); + if ($username eq "" || $pw eq "") { + push @$error_ref, "There isn't username or password for 'system' in 'passwd' table"; + $rst = 1; + } + + return $rst; +} + +sub check_network{ + my $sitetable_ref=shift; + my $serverip_ref = shift; #output arguments + my $error_ref = shift; #output arguments + + my $rst = 0; + @$error_ref = (); + + if($is_sn){ + # on SN, get ip address by compare 'master' attribute in 'site' table + # choose the one in the same network with 'master' + my @ipoutput = `ip addr show | grep inet | grep -v inet6 2>&1`; + foreach (@ipoutput) { + if ($_ =~ /inet\s+(.+)\/(.+)\s+brd\s+(.+)\s+scope global/i) { + if (xCAT::NetworkUtils::isInSameSubnet($sitetable_ref->{master}, $1, $2, 1)) { + $$serverip_ref = $1; + } + } + } + if (!defined($$serverip_ref) || ($$serverip_ref eq "")) { + push @$error_ref, "There isn't IP in current server has the same network as master '$sitetable_ref->{master}'"; + $rst = 1; + } + }else{ # on MN + my $nics = `ip addr show $installnic >/dev/null 2>&1`; + if ($?) { + push @$error_ref, "There isn't NIC '$installnic' in current server"; + $rst = 1; + } else { + $$serverip_ref = `ip addr show $installnic | awk -F" " '/inet / {print \$2}'|awk -F"/" '{print \$1}'`; + chomp($$serverip_ref); + if (!defined($$serverip_ref) || ($$serverip_ref eq "")) { + push @$error_ref, "There isn't IP address assigned to NIC $installnic"; + $rst =1; + }else{ + if ($$serverip_ref ne $sitetable_ref->{master}) { + push @$error_ref, "The IP $$serverip_ref of $installnic doesn't equal the value of 'master' in 'site' table"; + $rst = 1; + } + } + } + } + + my $networks = `tabdump networks|grep -v "^#"`; + $networks =~ s/\"//g; + my $netcnt = `echo "$networks"|wc -l`; + my $hit = 0; + for (my $i = 1 ; $i < $netcnt + 1 ; $i++) { + my $line = `echo "$networks" |sed -n ${i}p |awk -F"," '{print \$2,\$3,\$4}'`; + chomp($line); + if ($line =~ /(.+) (.+) (.+)/) { + if ($is_sn) { + $hit = 1 if (probe_utils->is_ip_belong_to_net("$1", "$2", $$serverip_ref)); + } else { + $hit = 1 if (probe_utils->is_ip_belong_to_net("$1", "$2", $$serverip_ref) && ("$3" eq "$installnic")); + } + } + } + if (! $hit) { + push @$error_ref, "IP $$serverip_ref of $installnic doesn't belong to any network defined in 'networks' table"; + $rst = 1; + } + return $rst; +} + +sub check_server_ip_static{ + my $serverip=shift; + my $error_ref = shift; + + my $rst = 0; + @$error_ref = (); + + if (! probe_utils->is_static_ip("$serverip", "$installnic")) { + push @$error_ref, "The value '$serverip' of 'master' in 'site' table isn't a static ip"; + $rst=1; + } + return $rst; +} + + +sub check_directory{ + my $sitetable_ref=shift; + my $error_ref = shift; + my $rst = 0; + @$error_ref = (); + + my @dir_list = ("installdir", "tftpdir"); + foreach my $dir (@dir_list){ + if($sitetable_ref->{$dir} eq ""){ + push @$error_ref, "There isn't '$dir' definition in 'site' table"; + $rst = 1; + }else{ + if (! -e "$sitetable_ref->{$dir}") { + push @$error_ref, "There isn't '$sitetable_ref->{$dir}' directory on current server"; + $rst = 1; + }else{ + if ($is_sn) { + my $mountoutput = `mount | grep '$sitetable_ref->{$dir}'`; + chomp($mountoutput); + + my $mountip; + if ($mountoutput =~ /(.+):$sitetable_ref->{$dir} on $sitetable_ref->{$dir} /) { + my $mountsource = $1; + if (xCAT::NetworkUtils->isIpaddr($mountsource)) { + $mountip = $mountsource; + } else { + $mountip = xCAT::NetworkUtils->getipaddr($mountsource); + } + } + + if ($mountip ne $sitetable_ref->{master}) { + push @$error_ref, "$dir '$sitetable_ref->{$dir}' isn't mounted from the management node"; + $rst = 1; + } + } + } + } + } + + return $rst; +} + + +sub check_disk{ + my $error_ref = shift; + + my $rst = 0; + @$error_ref = (); + + #check the free space of specific directory + #if "/var" is mounted on standalone disk, more than 1G free space is expected + #if "/tmp" is mounted on standalone disk, more than 1G free space is expected + #if installdir is mounted on standalone disk, more than 10G free space is expected. + #if any one of above three directories hasn't standalone disk, "/" directory should cover its space requirement. + my @dir_expectedspace_list = (["/var", "1"], ["/tmp", "1"], ["$installdir", "10"], ["/" , "0"]); + foreach my $dir (@dir_expectedspace_list){ + next if($dir->[0] eq "/" && $dir->[1] == 0); + my $checkrst = probe_utils->is_dir_has_enough_space($dir->[0], $dir->[1]); + if($checkrst == 2){ + $dir_expectedspace_list[$#dir_expectedspace_list][1] += $dir->[1]; +# }elsif($checkrst == 1){ +# probe_utils->send_msg($outputtarget, "o", "The free space of '$dir->[0]' directory is more than $dir->[1] G"); + }elsif($checkrst == 0){ +# probe_utils->send_msg($outputtarget, "w", "The free space of '$dir->[0]' is less than $dir->[1] G"); + push @$error_ref, "The free space of '$dir->[0]' is less than $dir->[1] G"; + $rst = 1; + } + + } + + return $rst; +} + +sub check_selinux{ + my $error_ref = shift; + my $rst = 0; + @$error_ref = (); + + if (probe_utils->is_selinux_enable()) { + push @$error_ref, "SELinux is enabled on current server"; + $rst = 1; + } + + return $rst; +} + +sub check_firewall { + my $error_ref = shift; + my $rst = 0; + @$error_ref = (); + + if (probe_utils->is_firewall_open()) { + push @$error_ref, "Firewall is configured on current server"; + $rst =1; + } + + return $rst; +} + +sub check_http_service{ + my $serverip=shift; + my $error_ref = shift; + my $rst = 0; + @$error_ref = (); + + `which wget > /dev/null 2>&1`; + if ($?) { + push @$error_ref, "HTTP check need 'wget' tool, please install 'wget' tool and try again"; + $rst = 1; + } else { + if (! probe_utils->is_http_ready("$serverip")) { + push @$error_ref, "HTTP service isn't ready on $serverip"; + $rst = 1; + } + } + + return $rst; +} + + +sub check_tftp_service{ + my $serverip = shift; + my $error_ref = shift; + my $rst = 0; + @$error_ref = (); + + my $nodename = `hostname -s`; + chomp($nodename); + + # For sn, 'setuptftp' attribute could be set to '0' or '1'. + # if '0', sn does not need to provie TFTP service, will not check it + my $checktftp = 1; + if ($is_sn) { + $checktftp = `lsdef $nodename -i setuptftp -c | awk -F'=' '{print \$2}' `; + chomp($checktftp); + } + if ($checktftp) { + `which tftp > /dev/null 2>&1`; + if ($?) { + push @$error_ref, "TFTP check need 'tftp' tool, please install 'tftp' tool and try again"; + $rst = 1; + } else { + $msg = "TFTP service is ready on $serverip"; + if (! probe_utils->is_tftp_ready("$serverip")) { + push @$error_ref, "TFTP service isn't ready on $serverip"; + $rst = 1; + } + } + }else{ + $rst = 2; + } + + return $rst; +} + +sub check_dns_service{ + my $sitetable_ref=shift; + my $serverip=shift; + my $error_ref = shift; + my $rst = 0; + @$error_ref = (); + + my $nodename = `hostname -s`; + chomp($nodename); + + # For sn, 'setupdns' attribute could be set to '0' or '1'. + # if '0', sn does not need to provie DNS service, will not check it + my $checkdns = 1; + if ($is_sn) { + $checkdns = `lsdef $nodename -i setupnameserver -c | awk -F'=' '{print \$2}'`; + chomp($checkdns); + } + + if ($checkdns) { + `which nslookup > /dev/null 2>&1`; + if ($?) { + push @$error_ref, "DNS check need 'nslookup' tool, please install 'nslookup' tool and try again"; + $rst = 1; + } else { + if ($is_sn) { + # on sn, nslookup it's ip to check DNS service + if (!probe_utils->is_dns_ready("$serverip", "$sitetable_ref->{master}", "$nodename", "$sitetable_ref->{domain}")) { + push @$error_ref, "DNS service isn't ready on $serverip"; + $rst = 1; + } + }else{ + my $rc=0; + # if this is a hierarchical cluster, nslookup one of sn to check DNS service + my @snlist = xCAT::ServiceNodeUtils->getAllSN(); + my $sntmp = shift(@snlist); + if ($sntmp) { + my $sninfo = `cat /etc/hosts | grep $sntmp`; + if ($sninfo =~ /(\d+).(\d+).(\d+).(\d+)/) { + my $snip = "$1.$2.$3.$4"; + if (!probe_utils->is_dns_ready("$snip", "$serverip", "$sntmp", "$sitetable_ref->{domain}")) { + $rc = 1; + } + } + } else { + + # if there is no sn, nslookup mnip + my $nslkp = `nslookup $serverip $serverip 2>&1`; + chomp($nslkp); + my $tmp = grep {$_ =~ "Server:[\t\s]*$serverip"} split (/\n/, $nslkp); + if (!$tmp) { + $rc = 1; + } + } + if ($rc) { + push @$error_ref, "DNS service isn't ready on $serverip"; + $rst = 1; + } + } + } + }esle{ + $rst =2; + } + + return $rst; +} + + +sub check_dhcp_service{ + my $serverip=shift; + my $error_ref = shift; + + my $rst = 0; + @$error_ref = (); + + + # For sn, 'setupdhcp' attribute could be set to '0' or '1'. + # if '0', sn does not need to provie DHCP service, will not check it + if ($is_sn) { + my $nodename = `hostname -s`; + chomp($nodename); + my $checkdhcp = `lsdef $nodename -i setupdhcp -c | awk -F'=' '{print \$2}'`; + chomp($checkdhcp); + if ($checkdhcp) { + # on sn, just check dhcpd service whether running + my $dhcpoutput = `ps aux | grep dhcpd |grep -v grep`; + if (! $dhcpoutput) { + push @$error_ref, "There isn't 'dhcpd' deamon in current server"; + $rst = 1; + } + } else { + $rst = 2; + } + } else { + my $rc = 0; + { #very important brace to create a block + my @snlist = xCAT::ServiceNodeUtils->getAllSN(); + my $sntmp = shift(@snlist); + if ($sntmp) { + my $tmp = `makedhcp -q $sntmp`; + if ($?) { + push @$error_ref, "makedhcp -q $sntmp failed"; + returncmdoutput($tmp, $error_ref) if ($verbose); + $rc = 1; + last; + } + chomp($tmp); + my $snip = xCAT::NetworkUtils->getipaddr($sntmp); + my $snmac = `lsdef $sntmp -i mac -c | awk -F'=' '{print \$2}'`; + chomp ($snmac); + my $tmpmac; + if ($tmp =~ /$sntmp: ip-address = $snip, hardware-address = (.+)/) { + $tmpmac = $1; + if ($tmpmac !~ $snmac) { + push @$error_ref, "DHCP server reply is wrong"; + returncmdoutput($tmp, $error_ref) if ($verbose); + $rc = 1; + } + } else { + push @$error_ref, "DHCP server reply is wrong"; + returncmdoutput($tmp, $error_ref) if ($verbose); + $rc = 1; + } + } else { + + my $tmp = `chdef xcatmntest groups=all ip=$serverip mac=aa:aa:aa:aa:aa:aa`; + if ($?) { + push @$error_ref, "Node simulation by 'chdef' has failed"; + returncmdoutput($tmp, $error_ref) if ($verbose); + $rc = 1; + last; + } else { + push @$error_ref, "Simulate a node xcatmntest to do dhcp test" if ($verbose); + } + + `cp /etc/hosts /etc/hosts.bak.probe > /dev/null 2>&1`; + + open HOSTFILE, ">> /etc/hosts"; + print HOSTFILE "$serverip xcatmntest xcatmntest.$domain"; + close HOSTFILE; + + probe_utils->send_msg($outputtarget, "d", "To do 'makedhcp xcatmntest'") if ($verbose); + $tmp = `makedhcp xcatmntest 2>&1`; + if ($?) { + push @$error_ref, "makedhcp xcatmntest failed"; + returncmdoutput($tmp, $error_ref) if ($verbose); + $rc = 1; + `rmdef xcatmntest`; + last; + } + + $tmp = `makedhcp -q xcatmntest`; + if ($?) { + push @$error_ref, "makedhcp -q xcatmntest failed"; + returncmdoutput($tmp, $error_ref) if ($verbose); + $rc = 1; + `makedhcp -d xcatmntest && rmdef xcatmntest`; + last; + } + chomp($tmp); + if ($tmp !~ /xcatmntest: ip-address = $serverip, hardware-address = aa:aa:aa:aa:aa:aa/) { + push @$error_ref, "DHCP server reply is wrong"; + returncmdoutput($tmp, $error_ref) if ($verbose); + $rc = 1; + `makedhcp -d xcatmntest && rmdef xcatmntest`; + last; + } + + push @$error_ref, "Start clearing simulation information for dhcp test" if ($verbose); + $tmp = `makedhcp -d xcatmntest && rmdef xcatmntest`; + returncmdoutput($tmp, $error_ref) if ($verbose); + + `rm /etc/hosts`; + `mv /etc/hosts.bak.probe /etc/hosts`; + } + } + if ($rc) { + push @$error_ref, "Run 'makedhcp -n' if it has not been ran before."; + $rst = 1; + } + } + + return $rst; +} + +sub check_dhcp_leases{ + my $error_ref = shift; + + my $rst = 0; + @$error_ref = (); + + my $leasefile = ""; + if (-e "/var/lib/dhcpd/dhcpd.leases") { + $leasefile = "/var/lib/dhcpd/dhcpd.leases"; + } elsif (-e "/var/lib/dhcp/db/dhcpd.leases") { + $leasefile = "/var/lib/dhcp/db/dhcpd.leases"; + } elsif (-e "/var/lib/dhcp/dhcpd.leases") { + $leasefile = "/var/lib/dhcp/dhcpd.leases"; + } + + my $filesizetmp = `du -sb $leasefile`; + if ($?) { + push @$error_ref, "Got the size of '$leasefile' failed"; + returncmdoutput($filesizetmp, $error_ref) if ($verbose); + $rst = 1; + } else { + chomp($filesizetmp); + my ($size, $file) = split(" ", $filesizetmp); + if ($size > 104857600) { + my $size_m = $size / 1048576; + push @$error_ref, "The size of $leasefile is $size_m M, more than 100M"; + $rst =1; + } + } + return $rst; +} + +sub returncmdoutput { + my $rst = shift; + my $error_ref = shift; + + chomp($rst); + my @lines = split("[\n\r]", $rst); + foreach my $line (@lines) { + push @$error_ref, $line; + } +} #------------------------------------- # Clean up test environment # ------------------------------------- @@ -736,90 +836,14 @@ sub cleanup { `mv /etc/hosts.bak.probe /etc/hosts > /dev/null 2>&1`; } } - #------------------------------------- -# Each probe sub command is supposed to support hierarchical. -# This funtion is used to caclulate which SN should be dispatched which command +# main process #------------------------------------- -sub caclulate_dispatch_cmd { - my @snlist = xCAT::ServiceNodeUtils->getAllSN(); - if ($noderange) { - my @nodes = `nodels $noderange 2>&1`; - if ($?) { - my $error = join(" ", @nodes); - if ($error =~ /Error: Invalid nodes and\/or groups in noderange: (.+)/) { - probe_utils->send_msg("$output", "f", "There are invaild nodes ($1) in command line attribute node range"); - } else { - probe_utils->send_msg("$output", "f", "There is an error in command line attribute node range, use 'nodels' to check"); - } - return 1; - } else { - chomp foreach (@nodes); - my $snnodemap = xCAT::ServiceNodeUtils->get_ServiceNode(\@nodes, "xcat", "MN"); - my %newsnnodemap; - foreach my $sn (keys %$snnodemap) { - if (grep(/^$sn$/, @snlist)) { - push(@{ $newsnnodemap{$sn} }, @{ $snnodemap->{$sn} }); - } else { - push(@{ $newsnnodemap{mn} }, @{ $snnodemap->{$sn} }); - } - } - - foreach my $sn (keys %newsnnodemap) { - my $nodes = join(",", @{ $newsnnodemap{$sn} }); - if ($sn eq "mn") { - $noderange = $nodes; - } else { - for (my $i = 0 ; $i <= $#tmpargv ; $i++) { - if ($tmpargv[$i] eq "-n") { - $tmpargv[ $i + 1 ] = $nodes; - last; - } - } - my $args = join(" ", @tmpargv); - $dispatchcmd{$sn} = "$::XCATROOT/probe/subcmds/$program_name $args 2>&1"; - } - } - } - } else { - if (@snlist) { - my $args = join(" ", @tmpargv); - if ($args =~ /\-V/) { - $args = "-V"; - } else { - $args = " "; - } - my $sns = join(",", @snlist); - $dispatchcmd{$sns} = "$::XCATROOT/probe/subcmds/$program_name $args 2>&1" if (!$?); - } - } - return 0; -} - -#------------------------------------ -# print sn's msg after all msg received -#------------------------------------ -sub send_sn_msg { - foreach $node (keys %summaryoutput) { - next if ($node eq "mn"); - foreach my $line (@{ $summaryoutput{$node} }) { - if ($line =~ /^(\[\w+\]\s*):\s*(.*)/) { - print "$1:[SN:$node]: $2\n"; - } else { - print "[failed] :[SN:$node]: $line\n"; - } - } - } -} - -#------------------------------------- -# main process start -#------------------------------------- -@tmpargv = @ARGV; +my @tmpargv = @ARGV; if ( !GetOptions("--help|h" => \$help, "T" => \$test, - "n=s" => \$noderange, + "H" => \$hierarchy, "V" => \$verbose, "i=s" => \$installnic)) { @@ -842,166 +866,75 @@ if ($test) { exit 0; } +#Handle the interrupt signal from STDIN $SIG{TERM} = $SIG{INT} = sub { $terminal = 1; cleanup(); }; -#-------------------------------------------- -# To confirm what current node is, MN or SN -#-------------------------------------------- -$is_sn = 1 if (-e "/etc/xCATSN"); - -if ($is_sn) { +#if it is called by hierarchy template, just run job, not to do dispatch +if ($hierarchy || $is_sn) { $rst = do_main_job(); exit $rst; } -if (!defined($installnic)) { - probe_utils->send_msg("$output", "f", "Option -i is required"); - probe_utils->send_msg("$output", "d", "$::USAGE"); - exit 1; +my @error; +my $hierarchy_instance = hierarchy->new(); +$rst = $hierarchy_instance->dispatch_cmd($noderange, \@tmpargv, \@error); +if($rst){ + probe_utils->send_msg("$output", "f", "Calculate dispatch command failed"); + foreach (@error){ + probe_utils->send_msg("$output", "", "$_"); + } + if($hierarchy_instance->destory(\@error)){ + probe_utils->send_msg("$output", "", "$_") foreach(@error); + } + exit $rst; } -#-------------------------------------------- -# Each probe tool is supposed to support hierarchical. -#-------------------------------------------- -$rst = caclulate_dispatch_cmd(); - -#print Dumper \%dispatchcmd; -#print "node left to mn : $noderange\n"; -exit $rst if ($rst); - -#-------------------------------------------- -# dispatch job to MN and SN -#-------------------------------------------- -my $mnjobpid = 0; -my @snsjobpids = (); -my @snsjobfds = (); -my $pipe_parent_read; -my $pipe_child_write; -pipe $pipe_parent_read, $pipe_child_write; -{ - #handle job in MN - $mnjobpid = fork(); - if (!defined($mnjobpid)) { - probe_utils->send_msg("$output", "f", "fork process to handle MN job failed: $!"); - $rst = 1; - last; - } elsif ($mnjobpid == 0) { - $SIG{TERM} = $SIG{INT} = sub { - exit 1; - }; - - close $pipe_parent_read; - $rst = do_main_job($pipe_child_write); - exit $rst; - } - $SIG{CHLD} = sub { waitpid($mnjobpid, WNOHANG) }; - close $pipe_child_write; - - #handle job dispatch to SN - foreach my $sn (keys %dispatchcmd) { - my $snjobcmd = "xdsh $sn -s \"$dispatchcmd{$sn}\" 2>&1"; - my $snjobfd; - my $snjobpid; - if (!($snjobpid = open($snjobfd, "$snjobcmd |"))) { - probe_utils->send_msg("$output", "f", "fork process to dispatch cmd $snjobcmd to $sn failed: $!"); - next; - } - push(@snsjobpids, $snjobpid); - push(@snsjobfds, $snjobfd); - } - - my $select = new IO::Select; - $select->add(\*$pipe_parent_read) if ($pipe_parent_read); - $select->add(\*$_) foreach (@snsjobfds); - $| = 1; - - my $line; - my %pipeisnonull; - $pipeisnonull{mn} = 1; - $pipeisnonull{$_} = 1 foreach (@snsjobfds); - my $onepipeisnonull = 1; - while ($onepipeisnonull) { - if (@hdls = $select->can_read(0)) { - foreach $hdl (@hdls) { - if ($pipeisnonull{mn} && $hdl == \*$pipe_parent_read) { - if (eof($pipe_parent_read)) { - $pipeisnonull{mn} = 0; - } else { - chomp($line = <$pipe_parent_read>); - if ($line =~ /(\[\w+\]\s*):\s*(.*)/) { - print "$1:[MN]: $2\n"; - } - push @{ $summaryoutput{mn} }, $line; +my %reply_cache; +while($hierarchy_instance->read_reply(\%reply_cache)){ + foreach my $servers (keys %reply_cache){ #Dispatch_cmd may use SN range to dispatch cms to SNs at one time + my @server_array = split(",", $servers); + foreach my $server (@server_array){ + foreach (@{$reply_cache{$servers}}){ + my $msg = ""; + my $logmsg=""; + + #For cases like below: + #c910f02c04p04: [ok] :All xCAT deamons are running + if($reply_cache{$servers}->[$_] =~ /^(\w+)\s*:\s*(\[\w+\]\s*):\s*(.*)/){ + if("$1" eq "$server"){ + $logmsg = "$2: $3"; + $msg = "$2:<$server>: $3"; } - } else { - foreach my $fd (@snsjobfds) { - if ($pipeisnonull{$fd} && $hdl == \*$fd) { - if (eof($fd)) { - $pipeisnonull{$fd} = 0; - } else { - chomp($line = <$fd>); - if ($line =~ /(Error:)\s+(\w+)\s+(.+)/i) { - push @{ $summaryoutput{$2} }, $line; - } elsif ($line =~ /^(\w+)\s*:\s(.*)/) { - push @{ $summaryoutput{$1} }, $2; - $line = "$2:$1: $3" if ($line =~ /^(\w+)\s*:\s*(\[\w+\]\s*):\s*(.*)/); - } - } - } + #For cases like below: + #c910f02c04p05: IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY! + }elsif($reply_cache{$servers}->[$_] =~ /^(\w+)\s*:\s*(.*)/){ + if("$1" eq "$server"){ + $logmsg = "$2"; + $msg = "<$server>: $2"; + } + #For cases like below: + #Unable to open socket connection to xcatd daemon on localhost:3001. + }else{ + if(length($reply_cache{$servers}->[$_])){ + $logmsg = $reply_cache{$servers}->[$_]; + $msg = "[failed] :[$server]: $reply_cache{$servers}->[$_]"; } } + probe_utils->send_msg("$output", "", "$msg") if(length($msg)); + push @{$summaryoutput{$server}},$logmsg if(length($logmsg)); } - $onepipeisnonull = 0; - $onepipeisnonull |= $pipeisnonull{$_} foreach (keys %pipeisnonull); } - last if ($terminal); - sleep 1; } - send_sn_msg(); + if($terminal){ + last; + } } -close($pipe_child_write) if ($pipe_child_write); -close($pipe_parent_read) if ($pipe_parent_read); -close($_) foreach (@snsjobfds); -my %runningpid; -$runningpid{$mnjobpid} = 1 if ($mnjobpid); -$runningpid{$_} = 1 foreach (@snsjobpids); -my $existrunningpid = 0; -$existrunningpid = 1 if (%runningpid); - -my $trytime = 0; -while ($existrunningpid) { - - #try INT 5 up to 5 times - if ($try < 5) { - foreach my $pid (keys %runningpid) { - kill 'INT', $pid if ($runningpid{$pid}); - } - - #try TERM 5 up to 5 times - } elsif ($try < 10) { - foreach my $pid (keys %runningpid) { - kill 'TERM', $pid if ($runningpid{$pid}); - } - - #try KILL 1 time - } else { - foreach my $pid (keys %runningpid) { - kill 'KILL', $pid if ($runningpid{$pid}); - } - } - ++$try; - - sleep 1; - foreach my $pid (keys %runningpid) { - $runningpid{$pid} = 0 if (waitpid($pid, WNOHANG)); - } - $existrunningpid = 0; - $existrunningpid |= $runningpid{$_} foreach (keys %runningpid); - last if ($try > 10); +if($hierarchy_instance->destory(\@error)){ + probe_utils->send_msg("$output", "", "$_") foreach(@error); } #------------------------------------- @@ -1014,3 +947,5 @@ exit $rst; + + From 2751a075b6e2310b18294285438c6e2d02f56796 Mon Sep 17 00:00:00 2001 From: hu-weihua Date: Fri, 4 Nov 2016 02:32:48 -0400 Subject: [PATCH 2/3] fix bug#1528 and modify code depending on comments --- xCAT-probe/subcmds/xcatmn | 461 +++++++++++++++++++------------------- 1 file changed, 235 insertions(+), 226 deletions(-) diff --git a/xCAT-probe/subcmds/xcatmn b/xCAT-probe/subcmds/xcatmn index f09e111b9..9fb7c68c4 100755 --- a/xCAT-probe/subcmds/xcatmn +++ b/xCAT-probe/subcmds/xcatmn @@ -9,14 +9,15 @@ use hierarchy; use xCAT::ServiceNodeUtils; use xCAT::NetworkUtils; use File::Basename; +use File::Copy; use Data::Dumper; use Getopt::Long qw(:config no_ignore_case); -my $help = 0; #command line attribute '-h', get usage information -my $test = 0; #command line attribute '-T' +my $help = 0; #command line attribute '-h', get usage information +my $test = 0; #command line attribute '-T' my $hierarchy = 0; -my $verbose = 0; #command line attribute '-V' -my $noderange; #command line attribute '-n' +my $verbose = 0; #command line attribute '-V' +my $noderange; #command line attribute '-n' my $output = "stdout"; #used by probe_utils->send_msg("$output", "o", "xxxxxxxxxx"); print output to STDOUT my $rst = 0; #the exit code of current command my $terminal = 0; #means get INT signal from STDIN @@ -33,7 +34,7 @@ $is_sn = 1 if (-e "/etc/xCATSN"); #------------------------------------- -# Usage +# Usage #------------------------------------- my $program_name = basename("$0"); #current sub_command name $::USAGE = "Usage: @@ -51,7 +52,7 @@ Options: "; sub do_main_job { - my $rst = 0; + my $rst = 0; my @error = (); my $checkpoint; my $rc = 0; @@ -59,97 +60,98 @@ sub do_main_job { #check if all xcat deamons are running $checkpoint = "All xCAT deamons are running"; - $rst = check_all_xcat_deamons(\@error); + $rst = check_all_xcat_deamons(\@error); print_check_result($checkpoint, "f", $rst, \@error); - return $rst if($rst); + return $rst if ($rst); - #check if xcatd can receive request + #check if xcatd can receive request $checkpoint = "xcatd can receive command request"; - $rst = check_xcatd_receive_request(\@error); + $rst = check_xcatd_receive_request(\@error); print_check_result($checkpoint, "f", $rst, \@error); - return $rst if($rst); + return $rst if ($rst); #check 'site' table configuratiions my %sitetable; $checkpoint = "'site' table is configured correctly"; - $rst = check_site_table(\%sitetable,\@error); + $rst = check_site_table(\%sitetable, \@error); print_check_result($checkpoint, "f", $rst, \@error); - return $rst if($rst); + return $rst if ($rst); #check network configuratiions $checkpoint = "Provision network is configured correctly"; - $rst = check_network(\%sitetable,\$installnicip,\@error); + $rst = check_network(\%sitetable, \$installnicip, \@error); print_check_result($checkpoint, "f", $rst, \@error); - return $rst if($rst); + return $rst if ($rst); #check 'passwd' table configuratiions $checkpoint = "'passwd' table is configured correctly"; - $rst = check_passwd_table(\@error); + $rst = check_passwd_table(\@error); print_check_result($checkpoint, "f", $rst, \@error); $rc |= $rst; #check important directory $checkpoint = "Important directory is configured correctly"; - $rst = check_directory(\%sitetable,\@error); + $rst = check_directory(\%sitetable, \@error); print_check_result($checkpoint, "f", $rst, \@error); $rc |= $rst; - #check if SElinux is disabled + #check if SElinux is disabled $checkpoint = "SELinux is disabled on current server"; - $rst = check_selinux(\@error); + $rst = check_selinux(\@error); print_check_result($checkpoint, "f", $rst, \@error); $rc |= $rst; #check http service $checkpoint = "HTTP service works well"; - $rst = check_http_service($installnicip, \@error); + $rst = check_http_service($installnicip, \@error); print_check_result($checkpoint, "f", $rst, \@error); $rc |= $rst; #check tftp service $checkpoint = "TFTP service works well"; - $rst = check_tftp_service($installnicip,\@error); + $rst = check_tftp_service($installnicip, \@error); print_check_result($checkpoint, "f", $rst, \@error); $rc |= $rst; #check DNS service $checkpoint = "DNS service works well"; - $rst = check_dns_service(\%sitetable, $installnicip,\@error); + $rst = check_dns_service(\%sitetable, $installnicip, \@error); print_check_result($checkpoint, "f", $rst, \@error); $rc |= $rst; #check DHCP service $checkpoint = "DHCP service works well"; - $rst = check_dhcp_service($installnicip, \@error); + $rst = check_dhcp_service($installnicip, \@error); print_check_result($checkpoint, "f", $rst, \@error); $rc |= $rst; #Below are the 'warning` level check points - #check if firewall is close + #check if firewall is close $checkpoint = "Firewall is closed on current server"; - $rst = check_firewall(\@error); + $rst = check_firewall(\@error); print_check_result($checkpoint, "w", $rst, \@error); $rc |= $rst; #check disk space - $checkpoint = "The disk space is enough for xCAT to work"; - $rst = check_disk(\@error); + $checkpoint = "The disk space is enough for xCAT to work"; + $rst = check_disk(\@error); print_check_result($checkpoint, "w", $rst, \@error); $rc |= $rst; - + #some sepecific check points in MN - if(! $is_sn){ + if (!$is_sn) { + #check if server ip is a static ip in MN $checkpoint = "The IP of master is a static IP address"; - $rst = check_server_ip_static($installnicip,\@error); + $rst = check_server_ip_static($installnicip, \@error); print_check_result($checkpoint, "w", $rst, \@error); $rc |= $rst; #check if dhcpd.leases is less than 100M $checkpoint = "The dhcpd.leases file is less than 100M"; - $rst = check_dhcp_leases(\@error); + $rst = check_dhcp_leases(\@error); print_check_result($checkpoint, "w", $rst, \@error); $rc |= $rst; } @@ -166,6 +168,7 @@ sub summary_all_jobs_output { #DO SUMMARY DEPENDING ON YOUR SUB_COMMAND NEED probe_utils->send_msg("$output", "d", "======================do summary====================="); + #print "summaryoutput:\n"; #print Dumper \%summaryoutput; @@ -180,12 +183,12 @@ sub summary_all_jobs_output { } elsif ($line =~ /(\[warning\]\s*):\s*(.*)/) { push @summary, "$1: $2"; $ok_section = 0; - } elsif($line =~ /(\[debug\]\s*):\s*(.*)/ && ! $ok_section) { + } elsif ($line =~ /(\[debug\]\s*):\s*(.*)/ && !$ok_section) { push @summary, "$1: $2"; - } elsif($line =~ /(\[ok\]\s*):\s*(.*)/) { - $ok_section=1; + } elsif ($line =~ /(\[ok\]\s*):\s*(.*)/) { + $ok_section = 1; } - + } my %summary_sn = (); @@ -203,9 +206,9 @@ sub summary_all_jobs_output { } elsif ($log =~ /(\[warning\]\s*):\s*(.*)/) { push @{ $summary_sn{$node}{"details"} }, "$1: $2"; $ok_section = 0; - } elsif ($log =~ /(\[debug\]\s*):\s*(.*)/ && ! $ok_section) { + } elsif ($log =~ /(\[debug\]\s*):\s*(.*)/ && !$ok_section) { push @{ $summary_sn{$node}{"details"} }, "$1: $2"; - } elsif ($log =~ /(\[ok\]\s*):\s*(.*)/){ + } elsif ($log =~ /(\[ok\]\s*):\s*(.*)/) { $ok_section = 1; } elsif ($log !~ /^(\[\w+\]\s*):\s*(.*)/) { push @{ $summary_sn{$node}{"details"} }, "[failed]: $log"; @@ -242,115 +245,115 @@ sub summary_all_jobs_output { } } } - + return $rst; } -sub print_check_result{ - my $msg = shift; - my $error_level = shift; +sub print_check_result { + my $msg = shift; + my $error_level = shift; my $check_return_code = shift; - my $error_msg_ref = shift; + my $error_msg_ref = shift; - if($check_return_code == 1){ + if ($check_return_code == 1) { probe_utils->send_msg("$output", "$error_level", $msg); probe_utils->send_msg("$output", "d", "$_") foreach (@$error_msg_ref); - }elsif($check_return_code ==2){ + } elsif ($check_return_code == 2) { return; - }else{ - probe_utils->send_msg("$output", "o", $msg); + } else { + probe_utils->send_msg("$output", "o", $msg); } } -sub check_all_xcat_deamons{ +sub check_all_xcat_deamons { my $error_ref = shift; my $rst = 0; @$error_ref = (); - my @deamon_list = ("SSL listener", - "DB Access", - "UDP listener", - "install monitor", - "Discovery worker", - "Command log writer"); + my @deamon_list = ("SSL listener", + "DB Access", + "UDP listener", + "install monitor", + "Discovery worker", + "Command log writer"); my $output = `ps aux|grep -v grep|grep xcatd`; - foreach my $deamon (@deamon_list){ - if($output !~ /$deamon/){ - push @$error_ref, "Deamon '$deamon' isn't running"; - $rst = 1; + foreach my $deamon (@deamon_list) { + if ($output !~ /$deamon/) { + push @$error_ref, "Deamon '$deamon' isn't running"; + $rst = 1; } } - + return $rst; } -sub check_xcatd_receive_request{ +sub check_xcatd_receive_request { my $error_ref = shift; - my $rst = 0; + my $rst = 0; @$error_ref = (); #check important port my @port_list = ("xcatdport", "xcatiport"); - foreach my $port_attr (@port_list){ + foreach my $port_attr (@port_list) { my $port = `lsdef -t site -i $port_attr -c| awk -F'=' '{print \$2}'`; - chomp($port); - if($port){ + chomp($port); + if ($port) { my $cmdoutput = `netstat -ant|grep LISTEN|grep $port`; - if($?){ + if ($?) { push @$error_ref, "Attribute '$port_attr' in site table is set to $port, but xcatd isn't listening on $port"; - $rst=1; + $rst = 1; } - }else{ + } else { push @$error_ref, "Attribute '$port_attr' isn't set in 'site' table"; $rst = 1; } } - return $rst if($rst); + return $rst if ($rst); #send one command to test if xcatd can receive request my $cmdoutput = `lsxcatd -a 2>&1`; - if($?){ - my @lines = split("[\n\r]", $cmdoutput); - push @$error_ref, $_ foreach (@lines); - $rst=1; + if ($?) { + my @lines = split("[\n\r]", $cmdoutput); + push @$error_ref, $_ foreach (@lines); + $rst = 1; } - + return $rst; } -sub check_site_table{ - my $sitetable_ref=shift; - my $error_ref = shift; +sub check_site_table { + my $sitetable_ref = shift; + my $error_ref = shift; my $rst = 0; @$error_ref = (); my @attr_list = ("master", "domain", "installdir", "tftpdir"); - foreach my $attr (@attr_list){ + foreach my $attr (@attr_list) { my $value; $value = `lsdef -t site -i $attr -c | awk -F'=' '{print \$2}'`; chomp($value); - if($value eq ""){ + if ($value eq "") { push @$error_ref, "There isn't '$attr' definition in 'site' table"; - $rst=1; - }else{ - if (($attr eq "master") and (! probe_utils->is_ip_addr("$value"))) { + $rst = 1; + } else { + if (($attr eq "master") and (!probe_utils->is_ip_addr("$value"))) { push @$error_ref, "The value of 'master' in 'site' table isn't an IP addres"; - $rst=1; + $rst = 1; } $sitetable_ref->{$attr} = $value; } } - + return $rst; } -sub check_passwd_table{ +sub check_passwd_table { my $error_ref = shift; - my $rst = 0; + my $rst = 0; @$error_ref = (); - + my $passwd = `tabdump passwd |awk -F',' '/system/ { gsub(/"/, "", \$2); gsub(/"/, "", \$3); print \$2,\$3 }'`; chomp($passwd); my ($username, $pw) = split(" ", $passwd); @@ -362,15 +365,16 @@ sub check_passwd_table{ return $rst; } -sub check_network{ - my $sitetable_ref=shift; - my $serverip_ref = shift; #output arguments - my $error_ref = shift; #output arguments +sub check_network { + my $sitetable_ref = shift; + my $serverip_ref = shift; #output arguments + my $error_ref = shift; #output arguments my $rst = 0; @$error_ref = (); - - if($is_sn){ + + if ($is_sn) { + # on SN, get ip address by compare 'master' attribute in 'site' table # choose the one in the same network with 'master' my @ipoutput = `ip addr show | grep inet | grep -v inet6 2>&1`; @@ -382,10 +386,10 @@ sub check_network{ } } if (!defined($$serverip_ref) || ($$serverip_ref eq "")) { - push @$error_ref, "There isn't IP in current server has the same network as master '$sitetable_ref->{master}'"; + push @$error_ref, "There isn't IP in current server has the same network as master '$sitetable_ref->{master}'"; $rst = 1; } - }else{ # on MN + } else { # on MN my $nics = `ip addr show $installnic >/dev/null 2>&1`; if ($?) { push @$error_ref, "There isn't NIC '$installnic' in current server"; @@ -395,8 +399,8 @@ sub check_network{ chomp($$serverip_ref); if (!defined($$serverip_ref) || ($$serverip_ref eq "")) { push @$error_ref, "There isn't IP address assigned to NIC $installnic"; - $rst =1; - }else{ + $rst = 1; + } else { if ($$serverip_ref ne $sitetable_ref->{master}) { push @$error_ref, "The IP $$serverip_ref of $installnic doesn't equal the value of 'master' in 'site' table"; $rst = 1; @@ -404,7 +408,7 @@ sub check_network{ } } } - + my $networks = `tabdump networks|grep -v "^#"`; $networks =~ s/\"//g; my $netcnt = `echo "$networks"|wc -l`; @@ -420,48 +424,48 @@ sub check_network{ } } } - if (! $hit) { + if (!$hit) { push @$error_ref, "IP $$serverip_ref of $installnic doesn't belong to any network defined in 'networks' table"; $rst = 1; } return $rst; } -sub check_server_ip_static{ - my $serverip=shift; +sub check_server_ip_static { + my $serverip = shift; my $error_ref = shift; my $rst = 0; @$error_ref = (); - if (! probe_utils->is_static_ip("$serverip", "$installnic")) { + if (!probe_utils->is_static_ip("$serverip", "$installnic")) { push @$error_ref, "The value '$serverip' of 'master' in 'site' table isn't a static ip"; - $rst=1; - } - return $rst; -} + $rst = 1; + } + return $rst; +} -sub check_directory{ - my $sitetable_ref=shift; - my $error_ref = shift; - my $rst = 0; +sub check_directory { + my $sitetable_ref = shift; + my $error_ref = shift; + my $rst = 0; @$error_ref = (); my @dir_list = ("installdir", "tftpdir"); - foreach my $dir (@dir_list){ - if($sitetable_ref->{$dir} eq ""){ - push @$error_ref, "There isn't '$dir' definition in 'site' table"; + foreach my $dir (@dir_list) { + if ($sitetable_ref->{$dir} eq "") { + push @$error_ref, "There isn't '$dir' definition in 'site' table"; $rst = 1; - }else{ - if (! -e "$sitetable_ref->{$dir}") { + } else { + if (!-e "$sitetable_ref->{$dir}") { push @$error_ref, "There isn't '$sitetable_ref->{$dir}' directory on current server"; $rst = 1; - }else{ + } else { if ($is_sn) { my $mountoutput = `mount | grep '$sitetable_ref->{$dir}'`; chomp($mountoutput); - + my $mountip; if ($mountoutput =~ /(.+):$sitetable_ref->{$dir} on $sitetable_ref->{$dir} /) { my $mountsource = $1; @@ -471,53 +475,58 @@ sub check_directory{ $mountip = xCAT::NetworkUtils->getipaddr($mountsource); } } - + if ($mountip ne $sitetable_ref->{master}) { push @$error_ref, "$dir '$sitetable_ref->{$dir}' isn't mounted from the management node"; $rst = 1; } } } - } + } } return $rst; } -sub check_disk{ +sub check_disk { my $error_ref = shift; my $rst = 0; @$error_ref = (); - #check the free space of specific directory - #if "/var" is mounted on standalone disk, more than 1G free space is expected - #if "/tmp" is mounted on standalone disk, more than 1G free space is expected - #if installdir is mounted on standalone disk, more than 10G free space is expected. - #if any one of above three directories hasn't standalone disk, "/" directory should cover its space requirement. - my @dir_expectedspace_list = (["/var", "1"], ["/tmp", "1"], ["$installdir", "10"], ["/" , "0"]); - foreach my $dir (@dir_expectedspace_list){ - next if($dir->[0] eq "/" && $dir->[1] == 0); - my $checkrst = probe_utils->is_dir_has_enough_space($dir->[0], $dir->[1]); - if($checkrst == 2){ - $dir_expectedspace_list[$#dir_expectedspace_list][1] += $dir->[1]; -# }elsif($checkrst == 1){ -# probe_utils->send_msg($outputtarget, "o", "The free space of '$dir->[0]' directory is more than $dir->[1] G"); - }elsif($checkrst == 0){ -# probe_utils->send_msg($outputtarget, "w", "The free space of '$dir->[0]' is less than $dir->[1] G"); - push @$error_ref, "The free space of '$dir->[0]' is less than $dir->[1] G"; - $rst = 1; - } + my %dir_expectedspace_list = ("/var" => 1, "/tmp" => 1, "/install" => 10); + my %mountpointinfo; + foreach my $dir (keys %dir_expectedspace_list) { + my $output = `df -k $dir|tail -n 1`; + chomp($output); + my @splitoutput = split(" ", $output); + $mountpointinfo{ $splitoutput[5] }{available} = $splitoutput[3]; + $mountpointinfo{ $splitoutput[5] }{need} += $dir_expectedspace_list{$dir} * 1048576; + push @{ $mountpointinfo{ $splitoutput[5] }{mount} }, $dir; + } + my $msg = ""; + foreach $mountpoint (keys %mountpointinfo) { + if ($mountpointinfo{$mountpoint}{need} > $mountpointinfo{$mountpoint}{available}) { + foreach (@{ $mountpointinfo{$mountpoint}{mount} }) { + $msg .= "'$_' needs $dir_expectedspace_list{$_} G, "; + } + my $size_g = $mountpointinfo{$mountpoint}{available} / 1048576; + $msg .= "These directories are mounted on '$mountpoint', the free space of '$mountpoint' is $size_g G, it isn't enough."; + $rst = 1; + } + } + if ($rst) { + push @$error_ref, "$msg"; } return $rst; } -sub check_selinux{ +sub check_selinux { my $error_ref = shift; - my $rst = 0; + my $rst = 0; @$error_ref = (); if (probe_utils->is_selinux_enable()) { @@ -530,21 +539,21 @@ sub check_selinux{ sub check_firewall { my $error_ref = shift; - my $rst = 0; + my $rst = 0; @$error_ref = (); if (probe_utils->is_firewall_open()) { push @$error_ref, "Firewall is configured on current server"; - $rst =1; + $rst = 1; } return $rst; } -sub check_http_service{ - my $serverip=shift; +sub check_http_service { + my $serverip = shift; my $error_ref = shift; - my $rst = 0; + my $rst = 0; @$error_ref = (); `which wget > /dev/null 2>&1`; @@ -552,8 +561,8 @@ sub check_http_service{ push @$error_ref, "HTTP check need 'wget' tool, please install 'wget' tool and try again"; $rst = 1; } else { - if (! probe_utils->is_http_ready("$serverip")) { - push @$error_ref, "HTTP service isn't ready on $serverip"; + if (!probe_utils->is_http_ready("$serverip")) { + push @$error_ref, "HTTP service isn't ready on $serverip"; $rst = 1; } } @@ -562,10 +571,10 @@ sub check_http_service{ } -sub check_tftp_service{ - my $serverip = shift; +sub check_tftp_service { + my $serverip = shift; my $error_ref = shift; - my $rst = 0; + my $rst = 0; @$error_ref = (); my $nodename = `hostname -s`; @@ -585,23 +594,23 @@ sub check_tftp_service{ $rst = 1; } else { $msg = "TFTP service is ready on $serverip"; - if (! probe_utils->is_tftp_ready("$serverip")) { + if (!probe_utils->is_tftp_ready("$serverip")) { push @$error_ref, "TFTP service isn't ready on $serverip"; $rst = 1; } } - }else{ + } else { $rst = 2; } return $rst; } -sub check_dns_service{ - my $sitetable_ref=shift; - my $serverip=shift; - my $error_ref = shift; - my $rst = 0; +sub check_dns_service { + my $sitetable_ref = shift; + my $serverip = shift; + my $error_ref = shift; + my $rst = 0; @$error_ref = (); my $nodename = `hostname -s`; @@ -614,7 +623,7 @@ sub check_dns_service{ $checkdns = `lsdef $nodename -i setupnameserver -c | awk -F'=' '{print \$2}'`; chomp($checkdns); } - + if ($checkdns) { `which nslookup > /dev/null 2>&1`; if ($?) { @@ -622,13 +631,15 @@ sub check_dns_service{ $rst = 1; } else { if ($is_sn) { + # on sn, nslookup it's ip to check DNS service if (!probe_utils->is_dns_ready("$serverip", "$sitetable_ref->{master}", "$nodename", "$sitetable_ref->{domain}")) { push @$error_ref, "DNS service isn't ready on $serverip"; $rst = 1; } - }else{ - my $rc=0; + } else { + my $rc = 0; + # if this is a hierarchical cluster, nslookup one of sn to check DNS service my @snlist = xCAT::ServiceNodeUtils->getAllSN(); my $sntmp = shift(@snlist); @@ -645,7 +656,7 @@ sub check_dns_service{ # if there is no sn, nslookup mnip my $nslkp = `nslookup $serverip $serverip 2>&1`; chomp($nslkp); - my $tmp = grep {$_ =~ "Server:[\t\s]*$serverip"} split (/\n/, $nslkp); + my $tmp = grep { $_ =~ "Server:[\t\s]*$serverip" } split(/\n/, $nslkp); if (!$tmp) { $rc = 1; } @@ -656,23 +667,23 @@ sub check_dns_service{ } } } - }esle{ - $rst =2; - } - + } esle { + $rst = 2; + } + return $rst; } -sub check_dhcp_service{ - my $serverip=shift; +sub check_dhcp_service { + my $serverip = shift; my $error_ref = shift; my $rst = 0; @$error_ref = (); - # For sn, 'setupdhcp' attribute could be set to '0' or '1'. + # For sn, 'setupdhcp' attribute could be set to '0' or '1'. # if '0', sn does not need to provie DHCP service, will not check it if ($is_sn) { my $nodename = `hostname -s`; @@ -680,18 +691,19 @@ sub check_dhcp_service{ my $checkdhcp = `lsdef $nodename -i setupdhcp -c | awk -F'=' '{print \$2}'`; chomp($checkdhcp); if ($checkdhcp) { + # on sn, just check dhcpd service whether running my $dhcpoutput = `ps aux | grep dhcpd |grep -v grep`; - if (! $dhcpoutput) { - push @$error_ref, "There isn't 'dhcpd' deamon in current server"; + if (!$dhcpoutput) { + push @$error_ref, "There isn't 'dhcpd' deamon in current server"; $rst = 1; } } else { $rst = 2; } } else { - my $rc = 0; - { #very important brace to create a block + my $rc = 0; + { #very important brace to create a block my @snlist = xCAT::ServiceNodeUtils->getAllSN(); my $sntmp = shift(@snlist); if ($sntmp) { @@ -705,7 +717,7 @@ sub check_dhcp_service{ chomp($tmp); my $snip = xCAT::NetworkUtils->getipaddr($sntmp); my $snmac = `lsdef $sntmp -i mac -c | awk -F'=' '{print \$2}'`; - chomp ($snmac); + chomp($snmac); my $tmpmac; if ($tmp =~ /$sntmp: ip-address = $snip, hardware-address = (.+)/) { $tmpmac = $1; @@ -749,7 +761,7 @@ sub check_dhcp_service{ $tmp = `makedhcp -q xcatmntest`; if ($?) { - push @$error_ref, "makedhcp -q xcatmntest failed"; + push @$error_ref, "makedhcp -q xcatmntest failed"; returncmdoutput($tmp, $error_ref) if ($verbose); $rc = 1; `makedhcp -d xcatmntest && rmdef xcatmntest`; @@ -768,12 +780,12 @@ sub check_dhcp_service{ $tmp = `makedhcp -d xcatmntest && rmdef xcatmntest`; returncmdoutput($tmp, $error_ref) if ($verbose); - `rm /etc/hosts`; - `mv /etc/hosts.bak.probe /etc/hosts`; + unlink "/etc/hosts"; + move("/etc/hosts.bak.probe", "/etc/hosts"); } } if ($rc) { - push @$error_ref, "Run 'makedhcp -n' if it has not been ran before."; + push @$error_ref, "Run 'makedhcp -n' if it has not been ran before."; $rst = 1; } } @@ -781,7 +793,7 @@ sub check_dhcp_service{ return $rst; } -sub check_dhcp_leases{ +sub check_dhcp_leases { my $error_ref = shift; my $rst = 0; @@ -796,25 +808,18 @@ sub check_dhcp_leases{ $leasefile = "/var/lib/dhcp/dhcpd.leases"; } - my $filesizetmp = `du -sb $leasefile`; - if ($?) { - push @$error_ref, "Got the size of '$leasefile' failed"; - returncmdoutput($filesizetmp, $error_ref) if ($verbose); + my @fileinfo = stat("$leasefile"); + if ($fileinfo[7] > 104857600) { + my $size_m = $fileinfo[7] / 1048576; + push @$error_ref, "The size of $leasefile is $size_m M, more than 100M"; $rst = 1; - } else { - chomp($filesizetmp); - my ($size, $file) = split(" ", $filesizetmp); - if ($size > 104857600) { - my $size_m = $size / 1048576; - push @$error_ref, "The size of $leasefile is $size_m M, more than 100M"; - $rst =1; - } } + return $rst; } sub returncmdoutput { - my $rst = shift; + my $rst = shift; my $error_ref = shift; chomp($rst); @@ -823,6 +828,7 @@ sub returncmdoutput { push @$error_ref, $line; } } + #------------------------------------- # Clean up test environment # ------------------------------------- @@ -832,10 +838,11 @@ sub cleanup { `makedhcp -d xcatmntest && rmdef xcatmntest > /dev/null 2>&1`; } if (-e "/etc/hosts.bak.probe") { - `rm /etc/hosts > /dev/null 2>&1`; - `mv /etc/hosts.bak.probe /etc/hosts > /dev/null 2>&1`; + unlink "/etc/hosts"; + move("/etc/hosts.bak.probe", "/etc/hosts"); } } + #------------------------------------- # main process #------------------------------------- @@ -881,60 +888,62 @@ if ($hierarchy || $is_sn) { my @error; my $hierarchy_instance = hierarchy->new(); $rst = $hierarchy_instance->dispatch_cmd($noderange, \@tmpargv, \@error); -if($rst){ - probe_utils->send_msg("$output", "f", "Calculate dispatch command failed"); - foreach (@error){ - probe_utils->send_msg("$output", "", "$_"); - } - if($hierarchy_instance->destory(\@error)){ - probe_utils->send_msg("$output", "", "$_") foreach(@error); - } - exit $rst; +if ($rst) { + probe_utils->send_msg("$output", "f", "Calculate dispatch command failed"); + foreach (@error) { + probe_utils->send_msg("$output", "", "$_"); + } + if ($hierarchy_instance->destory(\@error)) { + probe_utils->send_msg("$output", "", "$_") foreach (@error); + } + exit $rst; } my %reply_cache; -while($hierarchy_instance->read_reply(\%reply_cache)){ - foreach my $servers (keys %reply_cache){ #Dispatch_cmd may use SN range to dispatch cms to SNs at one time +while ($hierarchy_instance->read_reply(\%reply_cache)) { + foreach my $servers (keys %reply_cache) { #Dispatch_cmd may use SN range to dispatch cms to SNs at one time my @server_array = split(",", $servers); - foreach my $server (@server_array){ - foreach (@{$reply_cache{$servers}}){ - my $msg = ""; - my $logmsg=""; - - #For cases like below: + foreach my $server (@server_array) { + foreach (@{ $reply_cache{$servers} }) { + my $msg = ""; + my $logmsg = ""; + + #For cases like below: #c910f02c04p04: [ok] :All xCAT deamons are running - if($reply_cache{$servers}->[$_] =~ /^(\w+)\s*:\s*(\[\w+\]\s*):\s*(.*)/){ - if("$1" eq "$server"){ + if ($reply_cache{$servers}->[$_] =~ /^(\w+)\s*:\s*(\[\w+\]\s*):\s*(.*)/) { + if ("$1" eq "$server") { $logmsg = "$2: $3"; - $msg = "$2:<$server>: $3"; + $msg = "$2:<$server>: $3"; } - #For cases like below: - #c910f02c04p05: IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY! - }elsif($reply_cache{$servers}->[$_] =~ /^(\w+)\s*:\s*(.*)/){ - if("$1" eq "$server"){ + + #For cases like below: + #c910f02c04p05: IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY! + } elsif ($reply_cache{$servers}->[$_] =~ /^(\w+)\s*:\s*(.*)/) { + if ("$1" eq "$server") { $logmsg = "$2"; - $msg = "<$server>: $2"; + $msg = "<$server>: $2"; } - #For cases like below: - #Unable to open socket connection to xcatd daemon on localhost:3001. - }else{ - if(length($reply_cache{$servers}->[$_])){ + + #For cases like below: + #Unable to open socket connection to xcatd daemon on localhost:3001. + } else { + if (length($reply_cache{$servers}->[$_])) { $logmsg = $reply_cache{$servers}->[$_]; $msg = "[failed] :[$server]: $reply_cache{$servers}->[$_]"; } } - probe_utils->send_msg("$output", "", "$msg") if(length($msg)); - push @{$summaryoutput{$server}},$logmsg if(length($logmsg)); + probe_utils->send_msg("$output", "", "$msg") if (length($msg)); + push @{ $summaryoutput{$server} }, $logmsg if (length($logmsg)); } } } - if($terminal){ - last; + if ($terminal) { + last; } } -if($hierarchy_instance->destory(\@error)){ - probe_utils->send_msg("$output", "", "$_") foreach(@error); +if ($hierarchy_instance->destory(\@error)) { + probe_utils->send_msg("$output", "", "$_") foreach (@error); } #------------------------------------- From 0a0615139c60100c7ffc4e973f76ce66e59367f8 Mon Sep 17 00:00:00 2001 From: hu-weihua Date: Fri, 4 Nov 2016 03:19:38 -0400 Subject: [PATCH 3/3] modify depending on comments --- xCAT-probe/subcmds/xcatmn | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/xCAT-probe/subcmds/xcatmn b/xCAT-probe/subcmds/xcatmn index 9fb7c68c4..e30f54650 100755 --- a/xCAT-probe/subcmds/xcatmn +++ b/xCAT-probe/subcmds/xcatmn @@ -498,11 +498,11 @@ sub check_disk { my %dir_expectedspace_list = ("/var" => 1, "/tmp" => 1, "/install" => 10); my %mountpointinfo; foreach my $dir (keys %dir_expectedspace_list) { - my $output = `df -k $dir|tail -n 1`; + my $output = `df --block-size=1G $dir|tail -n 1`; chomp($output); my @splitoutput = split(" ", $output); $mountpointinfo{ $splitoutput[5] }{available} = $splitoutput[3]; - $mountpointinfo{ $splitoutput[5] }{need} += $dir_expectedspace_list{$dir} * 1048576; + $mountpointinfo{ $splitoutput[5] }{need} += $dir_expectedspace_list{$dir}; push @{ $mountpointinfo{ $splitoutput[5] }{mount} }, $dir; } @@ -512,8 +512,7 @@ sub check_disk { foreach (@{ $mountpointinfo{$mountpoint}{mount} }) { $msg .= "'$_' needs $dir_expectedspace_list{$_} G, "; } - my $size_g = $mountpointinfo{$mountpoint}{available} / 1048576; - $msg .= "These directories are mounted on '$mountpoint', the free space of '$mountpoint' is $size_g G, it isn't enough."; + $msg .= "These directories are mounted on '$mountpoint', the free space of '$mountpoint' is $mountpointinfo{$mountpoint}{available} G, it isn't enough."; $rst = 1; } }