#! /usr/bin/perl # IBM(c) 2016 EPL license http://www.eclipse.org/legal/epl-v10.html BEGIN { $::XCATROOT = $ENV{'XCATROOT'} ? $ENV{'XCATROOT'} : -d '/opt/xcat' ? '/opt/xcat' : '/usr'; } use lib "$::XCATROOT/probe/lib/perl"; use probe_utils; use File::Basename; use Net::Ping; use Getopt::Long qw(:config no_ignore_case); #use Data::Dumper; use warnings; my $program_name = basename("$0"); my $help; my $noderange = ""; my $test; my $output = "stdout"; my $verbose = 0; my $rst = 0; my $large_system = 20; # Number of nodes to treat as a large system $::USAGE = "Usage: $program_name -h $program_name {-c|-d|-u uuid} [-n noderange] [-V] Description: Use this command to check if diskless nodes have the same images installed as defines in xCAT DB. Use this command to check if all diskless nodes have the identical images installed. Options: -h : Get usage information of $program_name -n : Range of nodes to check -d : To verify diskless nodes have the same images installed as defines in xCAT DB. -c : To verify all diskless nodes have the identical images installed. -u : To display a list of diskless nodes running an OS with the specified UUID -V : To print additional debug information. "; #------------------------------------- # main process #------------------------------------- if ( !GetOptions("--help|h" => \$help, "T" => \$test, "V" => \$VERBOSE, "n=s" => \$noderange, "c" => \$CONSISTENCY_CHECK, "d" => \$DEFINITION_CHECK, "u=s" => \$UUID_specified)) { probe_utils->send_msg("$output", "f", "Invalid parameter for $program_name"); probe_utils->send_msg("$output", "d", "$::USAGE"); exit 1; } if ($help) { if ($output ne "stdout") { probe_utils->send_msg("$output", "d", "$::USAGE"); } else { print "$::USAGE"; } exit 0; } if ($test) { probe_utils->send_msg("$output", "o", "Use this command to check if specified diskless nodes have the same images installed or if nodes are installed with the same image as defined on the management node."); exit 0; } unless (defined($CONSISTENCY_CHECK) || defined($DEFINITION_CHECK) || defined($UUID_specified)) { probe_utils->send_msg("$output", "f", "At least one of -c or -d or -u flags is required"); probe_utils->send_msg("$output", "d", "$::USAGE"); exit 1; } if ($UUID_specified) { # Looking for nodes running with specific UUID. Do consistency check processing first to get the data $CONSISTENCY_CHECK = 1; } if (scalar(@ARGV) >= 1) { # After processing all the expected flags and arguments, # there is still left over stuff on the command line probe_utils->send_msg("$output", "f", "Invalid flag or parameter: @ARGV"); probe_utils->send_msg("$output", "d", "$::USAGE"); exit 1; } my @pingable_nodes; my @diskless_nodes; my $na = "N/A"; my $defined_UUID = $na; my %node_defined_image_uuid_hash; my %node_defined_image_name_hash; my %osimage_defined_provmethod_hash; my $all_nodes_provmethod = `lsdef -i provmethod -c $noderange`; my $all_osimage_provmethod = `lsdef -t osimage -i provmethod,rootimgdir -c`; chomp($all_nodes_provmethod); my @all_nodes_provmethod_lines = split("[\n\r]", $all_nodes_provmethod); my @all_osimage_provmethod_lines = split("[\n\r]", $all_osimage_provmethod); if ($all_nodes_provmethod =~ /Usage:/) { # lsdef command displayed a Usage message. Must be some noderange formatting problem. # Issue a warning and exit. probe_utils->send_msg("$output", "w", "Can not get a list of nodes from specified noderange."); exit 1; } if (scalar(@all_nodes_provmethod_lines) <= 0) { # There were no nodes matching the noderange. Issue a warning and exit. probe_utils->send_msg("$output", "w", "No nodes matching the noderange were found."); exit 1; } # Build a hash of key="osimage name + attribute name" value="provmethod and rootimgdir attribute value" foreach (@all_osimage_provmethod_lines) { my ($osimage_name, $values) = split "=", $_; $osimage_defined_provmethod_hash{$osimage_name} = $values; } # First, extract diskless nodes foreach (@all_nodes_provmethod_lines) { # Get osimage name for the node my ($node_name, $junk, $node_osimage_name) = split "[:=]", $_; chomp($node_osimage_name); if (length($node_osimage_name) > 0) { # Get provmethod and rootimgdir for the osimage my $osimage_provmethod_type = $osimage_defined_provmethod_hash{ $node_osimage_name . ": provmethod" }; my $rootimagedir = $osimage_defined_provmethod_hash{ $node_osimage_name . ": rootimgdir" }; chomp($osimage_provmethod_type) if ($osimage_provmethod_type); chomp($rootimagedir) if ($rootimagedir); # Check if it is netboot, meaning diskless if ($osimage_provmethod_type && $osimage_provmethod_type eq 'netboot') { push(@diskless_nodes, $node_name); probe_utils->send_msg("$output", "o", "$node_name is diskless") if ($VERBOSE); if (length($rootimagedir) > 0) { # For this diskless node, get UUID from rootimg directory xcatinfo file of the provmethod osimage my $xcatinfo_file = $rootimagedir . "/rootimg/opt/xcat/xcatinfo"; if (-r $xcatinfo_file) { $defined_UUID = `awk -F"'" '/IMAGEUUID/ {print \$2}' $xcatinfo_file`; chomp($defined_UUID); if (length($defined_UUID) < 1) { $defined_UUID = $na; } } } $node_defined_image_uuid_hash{$node_name} = $defined_UUID; $node_defined_image_name_hash{$node_name} = $node_osimage_name; print "Node $node_name has defined image $node_osimage_name at $rootimagedir with UUID $defined_UUID\n" if ($VERBOSE); } else { probe_utils->send_msg("$output", "w", "$node_name is not diskless. No image consistency verification will be performed.") if ($VERBOSE); } } else { my $nodetype = `lsdef -i nodetype -c $node_name`; my ($node, $type) = split "=", $nodetype; # If node object happens to be switch, do not display this warning message unless ($type =~ /switch/) { probe_utils->send_msg("$output", "w", "$node_name has no provision method defined. No image consistency verification will be performed.") if ($VERBOSE); } } } if (scalar(@diskless_nodes) <= 0) { # There were no diskless nodes found. Issue a warning and exit. probe_utils->send_msg("$output", "w", "No diskless nodes were found."); exit 1; } if (scalar(@diskless_nodes) <= 0) { # There were no diskless nodes found. Issue a warning and exit. probe_utils->send_msg("$output", "w", "No diskless nodes were found"); exit 1; } # Next, check if all diskless nodes are pingable my $ping_hosts = join ",", @diskless_nodes; my $pping_output = `pping $ping_hosts`; chomp($pping_output); my @pping_lines = split("[\n\r]", $pping_output); foreach (@pping_lines) { my ($hostname, $result) = split ":", $_; my ($token, $status) = split ' ', $result; chomp($token); if ($token ne 'ping') { probe_utils->send_msg("$output", "f", "Pinging $hostname"); } else { probe_utils->send_msg("$output", "o", "Pinging $hostname") if ($VERBOSE); push(@pingable_nodes, $hostname); } } if (scalar(@pingable_nodes) <= 0) { # There were no pingable, diskless nodes found. Issue a warning and exit. probe_utils->send_msg("$output", "w", "No diskless, pingable nodes were found"); exit 1; } if ((scalar(@pingable_nodes) == 1) && ($CONSISTENCY_CHECK)) { # There was only one node in noderange and comparison check was requested. # Nothing to compare the single node to. probe_utils->send_msg("$output", "w", "Comparison check for a single diskless node $pingable_nodes[0] will not be performed.\nMinimum of 2 nodes are needed for that."); exit 1; } # Next, from all pingable nodes get the IMAGENAME and IMAGEUUID entries from xcatinfo file probe_utils->send_msg("$output", "d", "---- Gathering information from all diskless nodes (pingable) ----"); my $pingable_hostname_list = join ",", @pingable_nodes; my $all_xdsh_output = `xdsh $pingable_hostname_list "cat /opt/xcat/xcatinfo"`; my $xcatinfo_image_UUID = ` echo "$all_xdsh_output" | awk -F"=" '/IMAGEUUID/ {gsub(/IMAGEUUID/,"",\$1); gsub(/'"'"'/,"",\$2);; print \$1 \$2}'`; # Check to verify xdsh worked and returned some usefull information if (length($xcatinfo_image_UUID) <= 1) { probe_utils->send_msg("$output", "w", "Unable to extract image UUID information from nodes using xdsh command. No image consistency verification will be performed."); exit 1; } my @xdsh_UUID_lines = split("[\n\r]", $xcatinfo_image_UUID); my $xcatinfo_image_name = ` echo "$all_xdsh_output" | awk -F"=" '/IMAGENAME/ {gsub(/IMAGENAME/,"",\$1); gsub(/'"'"'/,"",\$2); print \$1 \$2}'`; # Check to verify xdsh worked and returned some usefull information if (length($xcatinfo_image_name) <= 1) { probe_utils->send_msg("$output", "w", "Unable to extract image name information from nodes using xdsh command. No image consistency verification will be performed."); exit 1; } my @xdsh_name_lines = split("[\n\r]", $xcatinfo_image_name); my %node_running_image_uuid_hash; my %node_running_image_name_hash; # Build a hash of key=hostname, value=running UUID foreach (@xdsh_UUID_lines) { my ($hostname, $uuid) = split ": ", $_; chomp($hostname); chomp($uuid); if (length($uuid) > 0) { $node_running_image_uuid_hash{$hostname} = $uuid; } else { $node_running_image_uuid_hash{$hostname} = $na; } } # It is possible that some older version xCAT nodes will not have an IMAGEUUID line in # the xcatinfo file, for those nodes insert $na as the running UUID value foreach (@pingable_nodes) { unless (exists($node_running_image_uuid_hash{$_})) { $node_running_image_uuid_hash{$_} = $na; } } # Build a hash of key="hostname", value="running OS image name" foreach (@xdsh_name_lines) { my ($hostname, $osimage_name) = split ": ", $_; chomp($hostname); chomp($osimage_name); if (length($osimage_name) > 0) { $node_running_image_name_hash{$hostname} = $osimage_name; } else { $node_running_image_name_hash{$hostname} = $na; } } # print Dumper(\%node_running_image_uuid_hash); # print Dumper(\%node_running_image_name_hash); # Information gathering is done. Now do veification checking. # Probe verification step 1 - make sure all nodes are installed with the osimage name and imageUUID as defined on MN if ($DEFINITION_CHECK) { my $success_nodes = 0; my $msg; foreach (@pingable_nodes) { if (($node_running_image_name_hash{$_} eq $node_defined_image_name_hash{$_}) && ($node_running_image_uuid_hash{$_} eq $node_defined_image_uuid_hash{$_})) { if ($node_running_image_uuid_hash{$_} eq $na) { $msg = "$_: Not able to determine installed os image name or UUID"; } else { $msg = "OS image installed on diskless node $_ matches its image definition on the management node"; probe_utils->send_msg("$output", "o", "$msg") if ($VERBOSE); $success_nodes++; next; } } else { $msg = "$_: Unmatched OS image name or image UUID.\n Defined: name = $node_defined_image_name_hash{$_}" . " uuid = $node_defined_image_uuid_hash{$_}\n Installed: name = $node_running_image_name_hash{$_}" . " uuid = $node_running_image_uuid_hash{$_}"; } probe_utils->send_msg("$output", "f", "$msg"); } if (scalar(@pingable_nodes) eq $success_nodes) { # All pingable nodes were tested with success probe_utils->send_msg("$output", "o", "OS image installed on each diskless node matches its image definition on the management node"); } } # Probe verification step 2 - make sure all nodes are installed with the same osimage name and imageUUID if ($CONSISTENCY_CHECK) { my $msg = "Undefined"; my $status = "f"; my $image_name_and_uuid; my $image_uuid; my %unique_image_hash; # Go through the nodes and build a hash of key=image_name+image_uuid and value of nodename foreach (@pingable_nodes) { $image_name_and_uuid = $node_running_image_name_hash{$_} . ":" . $node_running_image_uuid_hash{$_}; if (exists $unique_image_hash{$image_name_and_uuid}) { $unique_image_hash{$image_name_and_uuid} = $unique_image_hash{$image_name_and_uuid} . "," . $_; } else { $unique_image_hash{$image_name_and_uuid} = $_; } } # print Dumper(\%unique_image_hash); # If there is more then one key in the hash, nodes have multiple images. my $number_of_keys = keys %unique_image_hash; if ($number_of_keys == 1) { my @image_names = keys %unique_image_hash; if ($image_names[0] =~ /$na/) { $msg = "Not able to determine OS image name or UUID of the image installed on any diskless node."; $status = "f"; } else { $msg = "All diskless nodes have the same OS image installed: @image_names."; $status = "o"; } } else { my $node_image_table; if ($UUID_specified) { # Produce list of nodes running with a specified UUID foreach $os_uuid (sort keys %unique_image_hash) { my ($os_name, $uuid) = split(":", $os_uuid); if ($uuid eq $UUID_specified) { # Found UUID match foreach my $node_name (split(",",$unique_image_hash{$os_uuid})) { $node_image_table .= sprintf("$node_name\n"); } $msg = "Compute nodes running OS=>$os_name UUID=>$uuid:\n" . $node_image_table; $status = "d"; last; } } if (length($node_image_table) < 1) { # At the end of the loop, no UUID match found $msg = "No diskless nodes running UUID $UUID_specified were found."; $status = "d"; } } else { if (scalar(@pingable_nodes) > $large_system) { # Produce summary output for a large system foreach $os_uuid (sort keys %unique_image_hash) { my ($os_name, $uuid) = split(":", $os_uuid); my $count = scalar(split(",",$unique_image_hash{$os_uuid})); $node_image_table .= sprintf(" $count diskless nodes running OS=>$os_name UUID=>$uuid\n"); } $node_image_table .= "Run 'xcatprobe image -u UUID' to display a list of diskless nodes that have OS installed with the specified UUID"; } else { # Produce list output for small system foreach $compute_node (sort keys %node_running_image_name_hash) { $node_image_table .= sprintf(" %-15s %-30s : %-20s\n", $compute_node, $node_running_image_name_hash{$compute_node}, $node_running_image_uuid_hash{$compute_node}); } } $msg = "Not all diskless nodes are installed with the same OS image.\n" . $node_image_table; $status = "f"; } } probe_utils->send_msg("$output", "$status", "$msg"); } exit 0;