2
0
mirror of https://github.com/xcat2/xcat-core.git synced 2025-05-21 19:22:05 +00:00
2022-11-29 11:50:18 -05:00

1330 lines
48 KiB
Perl

package xCAT_plugin::nodestat;
BEGIN
{
$::XCATROOT = $ENV{'XCATROOT'} ? $ENV{'XCATROOT'} : '/opt/xcat';
}
use lib "$::XCATROOT/lib/perl";
use strict;
use warnings;
use Socket;
my $inet6support = eval {
require Socket6;
require IO::Socket::INET6;
1;
};
use IO::Handle;
use Getopt::Long;
use Data::Dumper;
use xCAT::GlobalDef;
use xCAT::NetworkUtils;
require xCAT::Utils;
require xCAT::TableUtils;
require xCAT::ServiceNodeUtils;
my %nodesetstats;
my %chainhash;
my %default_ports = (
'ftp' => '21',
'ssh' => '22',
'sshd' => '22',
'pbs' => '15002',
'pbs_mom' => '15002',
'xend' => '8002',
'll' => '9616',
'loadl' => '9616',
'loadl_master' => '9616',
'loadleveler' => '9616',
'gpfs' => '1191',
'rdp' => '3389',
'msrpc' => '135',
);
sub handled_commands {
return {
nodestat => 'nodestat',
nodestat_internal => 'nodestat',
};
}
sub pinghost {
my $node = shift;
my $rc = system("ping -q -n -c 1 -w 1 $node > /dev/null");
if ($rc == 0) {
return 1;
} else {
return 0;
}
}
sub nodesockopen {
my $node = shift;
my $port = shift;
my $socket;
my $addr = gethostbyname($node);
my $sin = sockaddr_in($port, $addr);
my $proto = getprotobyname('tcp');
socket($socket, PF_INET, SOCK_STREAM, $proto) || return 0;
connect($socket, $sin) || return 0;
return 1;
}
sub installer_query {
my $node = shift;
my $destport = 3001;
my $socket;
my $text = "";
if ($inet6support) {
$socket = IO::Socket::INET6->new(PeerAddr => $node, PeerPort => 3001);
unless ($socket) { return 0; }
} else {
my $proto = getprotobyname('tcp');
socket($socket, PF_INET, SOCK_STREAM, $proto) || return 0;
my $addr = gethostbyname($node);
my $sin = sockaddr_in($destport, $addr);
connect($socket, $sin) || return 0;
}
print $socket "stat \n";
$socket->flush;
while (<$socket>) {
$text .= $_;
}
$text =~ s/\n.*//;
return $text;
close($socket);
}
sub getstat {
my $response = shift;
foreach (@{ $response->{node} }) {
$nodesetstats{ $_->{name}->[0] } = $_->{data}->[0];
}
}
#-------------------------------------------------------
=head3 preprocess_request
Check and setup for hierarchy
=cut
#-------------------------------------------------------
sub preprocess_request
{
my $req = shift;
my $cb = shift;
my %sn;
if (defined $req->{_xcatpreprocessed}->[0] && $req->{_xcatpreprocessed}->[0] == 1) { return [$req]; }
#exit if preprocessed
my $command = $req->{command}->[0];
if ($command eq "nodestat") {
@ARGV = ();
my $args = $req->{arg};
if ($args) {
@ARGV = @{$args};
}
# parse the options
$::UPDATE = 0;
$::QUIET = 0;
$::MON = 0;
$::POWER = 0;
#Getopt::Long::Configure("posix_default");
#Getopt::Long::Configure("no_gnu_compat");
Getopt::Long::Configure("bundling");
$Getopt::Long::ignorecase = 0;
if (!GetOptions(
'm|usemon' => \$::MON,
'q|quiet' => \$::QUIET, #this is a internal flag used by monitoring
'u|updatedb' => \$::UPDATE,
'p|powerstat' => \$::POWER,
'h|help' => \$::HELP,
'v|version' => \$::VERSION))
{
&usage($cb,1);
return (1);
}
if ($::HELP) {
&usage($cb);
return (0);
}
if ($::VERSION) {
my $version = xCAT::Utils->Version();
my $rsp = {};
$rsp->{data}->[0] = "$version";
xCAT::MsgUtils->message("I", $rsp, $cb);
return (0);
}
my $nodes = $req->{node};
if (!$nodes)
{
&usage($cb,1);
return (1);
}
$req->{'update'}->[0] = $::UPDATE;
$req->{'quiet'}->[0] = $::QUIET;
$req->{'mon'}->[0] = $::MON;
$req->{'power'}->[0] = $::POWER;
return [$req];
}
#the following is for nodestat_internal command
my $nodes = $req->{node};
my $service = "xcat";
my @requests;
if ($nodes) {
my $usenmapfrommn = 0;
if (-x '/usr/bin/nmap' or -x '/usr/local/bin/nmap') {
#my $sitetab = xCAT::Table->new('site');
#if ($sitetab) {
#(my $ref) = $sitetab->getAttribs({key => 'useNmapfromMN'}, 'value');
my @entries = xCAT::TableUtils->get_site_attribute("useNmapfromMN");
my $t_entry = $entries[0];
if (defined($t_entry)) {
if ($t_entry =~ /1|yes|YES|Y|y/) { $usenmapfrommn = 1; }
}
#}
}
#get monsettings
my %apps = ();
my $mon = $req->{'mon'}->[0];
if ($mon == 1) { %apps = getStatusMonsettings(); }
#if no apps specified in the monsetting table, add sshd, pbs and xend
if (keys(%apps) == 0) {
$apps{'sshd'}->{'group'} = "ALL"; #ALL means anything on the nodelist table, it is different from all
$apps{'sshd'}->{'port'} = "22";
$apps{'https'}->{'group'} = "ALL"; #ALL means anything on the nodelist table, it is different from all
$apps{'https'}->{'port'} = "443";
$apps{'pbs'}->{'group'} = "ALL";
$apps{'pbs'}->{'port'} = "15002";
$apps{'xend'}->{'group'} = "ALL";
$apps{'xend'}->{'port'} = "8002";
$apps{'rdp'}->{'group'} = "ALL";
$apps{'rdp'}->{'port'} = "3389";
$apps{'msrpc'}->{'group'} = "ALL";
$apps{'msrpc'}->{'port'} = "135";
$apps{'APPS'} = [ 'sshd', 'https', 'pbs', 'xend' ];
} else {
#go thorugh the settings and put defaults in
foreach my $app (keys(%apps)) {
if ($app eq 'APPS') { next; }
if (!exists($apps{$app}->{'group'})) { $apps{$app}->{'group'} = "ALL"; }
if (exists($apps{$app}->{'cmd'}) || exists($apps{$app}->{'dcmd'}) || exists($apps{$app}->{'lcmd'})) { next; }
if (exists($apps{$app}->{'port'})) { next; }
#add port number in if nothing is specified
if (exists($default_ports{$app})) { $apps{$app}->{'port'} = $default_ports{$app}; }
else {
my $p = `grep "^$app" /etc/services`;
if ($? == 0) {
my @a_list = sort(split('\n', $p));
my @a_temp = split('/', $a_list[0]);
my @a = split(' ', $a_temp[0]);
$apps{$app}->{'port'} = $a[1];
} else {
my $rsp = {};
$rsp->{data}->[0] = "Cannot find port number for application $app. Please either specify a port number or a command in monsetting table for $app.";
xCAT::MsgUtils->message("I", $rsp, $cb);
return (0);
}
}
}
#always add sshd
if (!exists($apps{'ssh'}) || !exists($apps{'sshd'})) {
$apps{'sshd'}->{'group'} = "ALL";
$apps{'sshd'}->{'port'} = "22";
my $pa = $apps{'APPS'};
push @$pa, 'sshd';
}
}
#print Dumper(%apps);
# find service nodes for requested nodes
# build an individual request for each service node
my $sn = xCAT::ServiceNodeUtils->get_ServiceNode($nodes, $service, "MN");
#get the member for each group
my %groups = ();
foreach my $app (keys %apps) {
if ($app eq 'APPS') { next; }
my $group = $apps{$app}->{'group'};
if (($group) && ($group ne "ALL") && (!exists($groups{$group}))) {
my @tmp_nodes = xCAT::NodeRange::noderange($group);
foreach (@tmp_nodes) {
$groups{$group}->{$_} = 1;
}
}
}
# build each request for each service node
my $all_apps = $apps{'APPS'};
my %all_porthash = (); #stores all the port apps if usenmapfrommn=1
my %lcmdhash = (); #('myapp2,myapp3'=> {
# lcmd=>'/tmp/mycmd1,/usr/bin/date',
# node=>[node1,node2]
# }
#)
foreach my $snkey (keys %$sn)
{
my $reqcopy = {%$req};
$reqcopy->{node} = $sn->{$snkey};
$reqcopy->{'_xcatdest'} = $snkey;
$reqcopy->{_xcatpreprocessed}->[0] = 1;
$reqcopy->{'useNmapfromMN'}->[0] = $usenmapfrommn;
$reqcopy->{'allapps'} = $all_apps;
my %porthash = (); #('sshd,ll'=> {
# port=>'22,5001',
# node=>[node1,node2]
# }
#)
my %cmdhash = (); #('gpfs,myapp'=> {
# cmd=>'/tmp/mycmd1,/usr/bin/date',
# node=>[node1,node2]
# }
#)
my %dcmdhash = (); #('myapp2,myapp3'=> {
# dcmd=>'/tmp/mycmd1,/usr/bin/date',
# node=>[node1,node2]
# }
#)
my @nodes_for_sn = @{ $sn->{$snkey} };
foreach my $node (@nodes_for_sn) {
my @ports;
my @portapps;
my @cmds;
my @cmdapps;
my @dcmds;
my @dcmdapps;
my @lcmdapps;
my @lcmds;
foreach my $app (keys %apps) {
if ($app eq 'APPS') { next; }
my $group = $apps{$app}->{'group'};
if (($group eq "ALL") || ($groups{$group}->{$node})) {
#print "app=$app\n";
if (exists($apps{$app}->{'port'})) {
push @ports, $apps{$app}->{'port'};
push @portapps, $app;
}
elsif (exists($apps{$app}->{'cmd'})) {
push @cmds, $apps{$app}->{'cmd'};
push @cmdapps, $app;
}
elsif (exists($apps{$app}->{'dcmd'})) {
push @dcmds, $apps{$app}->{'dcmd'};
push @dcmdapps, $app;
}
elsif (exists($apps{$app}->{'lcmd'})) {
push @lcmds, $apps{$app}->{'lcmd'};
push @lcmdapps, $app;
}
}
}
#print "ports=@ports\n";
#print "portapps=@portapps\n";
#print "cmds=@cmds\n";
#print "cmdapps=@cmdapps\n";
#print "dcmds=@dcmds\n";
#print "dcmdapps=@dcmdapps\n";
if (@portapps > 0) {
my $tmpapps = join(',', @portapps);
if (($usenmapfrommn == 1) && (@cmdapps == 0) && (@dcmdapps == 0) && (@lcmdapps == 0)) {
#this is the case where mn handles ports for all nodes using nmap
#The current limitation is that when there are cmd or dcmd specified for the node
# nmap has to be done on the service node because if both mn and sn update the appstatus
# one will overwites the other.
if (exists($all_porthash{$tmpapps})) {
my $pa = $all_porthash{$tmpapps}->{'node'};
push @$pa, $node;
} else {
$all_porthash{$tmpapps}->{'node'} = [$node];
$all_porthash{$tmpapps}->{'port'} = join(',', @ports);
}
} else {
if (exists($porthash{$tmpapps})) {
my $pa = $porthash{$tmpapps}->{'node'};
push @$pa, $node;
} else {
$porthash{$tmpapps}->{'node'} = [$node];
$porthash{$tmpapps}->{'port'} = join(',', @ports);
}
}
}
if (@cmdapps > 0) {
my $tmpapps = join(',', @cmdapps);
if (exists($cmdhash{$tmpapps})) {
my $pa = $cmdhash{$tmpapps}->{'node'};
push @$pa, $node;
} else {
$cmdhash{$tmpapps}->{'node'} = [$node];
$cmdhash{$tmpapps}->{'cmd'} = join(',', @cmds);
}
}
if (@dcmdapps > 0) {
my $tmpapps = join(',', @dcmdapps);
if (exists($dcmdhash{$tmpapps})) {
my $pa = $dcmdhash{$tmpapps}->{'node'};
push @$pa, $node;
} else {
$dcmdhash{$tmpapps}->{'node'} = [$node];
$dcmdhash{$tmpapps}->{'dcmd'} = join(',', @dcmds);
}
}
if (@lcmdapps > 0) {
my $i = 0;
foreach my $lapp (@lcmdapps) {
if (exists($lcmdhash{$lapp})) {
my $pa = $lcmdhash{$lapp}->{'node'};
push @$pa, $node;
} else {
$lcmdhash{$lapp}->{'node'} = [$node];
$lcmdhash{$lapp}->{'lcmd'} = $lcmds[$i];
}
$i++;
}
}
} #end foreach (@nodes_for_sn)
#print Dumper(%porthash);
#print "cmdhash=" . Dumper(%cmdhash);
#now push the settings into the requests
my $i = 1;
if ((keys(%porthash) == 0) && (keys(%cmdhash) == 0) && (keys(%dcmdhash) == 0) && (keys(%lcmdhash) == 0)) { next; }
foreach my $tmpapps (keys %porthash) {
$reqcopy->{'portapps'}->[0] = scalar keys %porthash;
$reqcopy->{"portapps$i"}->[0] = $tmpapps;
$reqcopy->{ "portapps$i" . "port" }->[0] = $porthash{$tmpapps}->{'port'};
$reqcopy->{ "portapps$i" . "node" } = $porthash{$tmpapps}->{'node'};
$i++;
}
$i = 1;
foreach my $tmpapps (keys %cmdhash) {
$reqcopy->{'cmdapps'}->[0] = scalar keys %cmdhash;
$reqcopy->{"cmdapps$i"}->[0] = $tmpapps;
$reqcopy->{ "cmdapps$i" . "cmd" }->[0] = $cmdhash{$tmpapps}->{'cmd'};
$reqcopy->{ "cmdapps$i" . "node" } = $cmdhash{$tmpapps}->{'node'};
$i++;
}
$i = 1;
foreach my $tmpapps (keys %dcmdhash) {
$reqcopy->{'dcmdapps'}->[0] = scalar keys %dcmdhash;
$reqcopy->{"dcmdapps$i"}->[0] = $tmpapps;
$reqcopy->{ "dcmdapps$i" . "dcmd" }->[0] = $dcmdhash{$tmpapps}->{'dcmd'};
$reqcopy->{ "dcmdapps$i" . "node" } = $dcmdhash{$tmpapps}->{'node'};
$i++;
}
#done
push @requests, $reqcopy;
} #enf sn_key
#print "apps=" . Dumper(%apps);
#mn handles all nmap when useNmapfromMN=1 on the site table
if (($usenmapfrommn == 1) && (keys(%all_porthash) > 0)) {
my @hostinfo = xCAT::NetworkUtils->determinehostname();
my %iphash = ();
foreach (@hostinfo) { $iphash{$_} = 1; }
my $handled = 0;
foreach my $req (@requests) {
my $currsn = $req->{'_xcatdest'};
if (exists($iphash{$currsn})) {
my $i = 1;
foreach my $tmpapps (keys %all_porthash) {
$req->{'portapps'}->[0] = scalar keys %all_porthash;
$req->{"portapps$i"}->[0] = $tmpapps;
$req->{ "portapps$i" . "port" }->[0] = $all_porthash{$tmpapps}->{'port'};
$req->{ "portapps$i" . "node" } = $all_porthash{$tmpapps}->{'node'};
$i++;
}
$handled = 1;
last;
}
}
if (!$handled) {
my $reqcopy = {%$req};
$reqcopy->{_xcatpreprocessed}->[0] = 1;
$reqcopy->{'useNmapfromMN'}->[0] = $usenmapfrommn;
$reqcopy->{'allapps'} = $all_apps;
my $i = 1;
foreach my $tmpapps (keys %all_porthash) {
$reqcopy->{'portapps'}->[0] = scalar keys %all_porthash;
$reqcopy->{"portapps$i"}->[0] = $tmpapps;
$reqcopy->{ "portapps$i" . "port" }->[0] = $all_porthash{$tmpapps}->{'port'};
$reqcopy->{ "portapps$i" . "node" } = $all_porthash{$tmpapps}->{'node'};
$i++;
}
push @requests, $reqcopy;
}
}
#if ($usenmapfrommn) {
# my $reqcopy = {%$req};
# $reqcopy->{'update'}->[0]=$::UPDATE;
# $reqcopy->{'useNmapfromMN'}->[0]=1;
# if (!$::UPDATE) {
# push @requests, $reqcopy;
# return \@requests; #do not distribute, nodestat seems to lose accuracy and slow down distributed, if using nmap
# }
#}
#now handle local commands
#print "lcmdhash=" . Dumper(%lcmdhash);
if (keys(%lcmdhash) > 0) {
my @hostinfo = xCAT::NetworkUtils->determinehostname();
my %iphash = ();
foreach (@hostinfo) { $iphash{$_} = 1; }
my $handled = 0;
foreach my $req (@requests) {
my $currsn = $req->{'_xcatdest'};
if (exists($iphash{$currsn})) {
my $i = 1;
foreach my $lapp (keys %lcmdhash) {
$req->{'lcmdapps'}->[0] = scalar keys %lcmdhash;
$req->{"lcmdapps$i"}->[0] = $lapp;
$req->{ "lcmdapps$i" . "cmd" }->[0] = $lcmdhash{$lapp}->{'lcmd'};
$req->{ "lcmdapps$i" . "node" } = $lcmdhash{$lapp}->{'node'};
$i++;
}
$handled = 1;
last;
}
}
if (!$handled) {
my $reqcopy = {%$req};
$reqcopy->{_xcatpreprocessed}->[0] = 1;
$reqcopy->{'allapps'} = $all_apps;
my $i = 1;
foreach my $lapp (keys %lcmdhash) {
$reqcopy->{'lcmdapps'}->[0] = scalar keys %lcmdhash;
$reqcopy->{"lcmdapps$i"}->[0] = $lapp;
$reqcopy->{ "lcmdapps$i" . "cmd" }->[0] = $lcmdhash{$lapp}->{'lcmd'};
$reqcopy->{ "lcmdapps$i" . "node" } = $lcmdhash{$lapp}->{'node'};
$i++;
}
push @requests, $reqcopy;
}
}
}
return \@requests;
}
sub interrogate_node { #Meant to run against confirmed up nodes
my $node = shift;
my $doreq = shift;
my $p_tmp = shift;
my %portservices = %$p_tmp;
my $status = "";
my $appsd = ""; #detailed status
my $ret = {};
$ret->{'status'} = "ping";
foreach my $port (keys(%portservices)) {
if (nodesockopen($node, $port)) {
$status .= $portservices{$port} . ",";
$appsd .= $portservices{$port} . "=up,";
} else {
$appsd .= $portservices{$port} . "=down,";
}
}
$status =~ s/,$//;
$appsd =~ s/,$//;
$ret->{'appsd'} = $appsd;
if ($status) {
$ret->{'appstatus'} = $status;
return $ret;
}
if ($status = installer_query($node)) {
$ret->{'status'} = $status;
return $ret;
} else { #pingable, but no *clue* as to what the state may be
$doreq->({ command => ['nodeset'],
node => [$node],
arg => ['stat'] },
\&getstat);
$ret->{'status'} = 'ping ' . $nodesetstats{$node};
return $ret;
}
}
sub process_request_nmap {
my $request = shift;
my $callback = shift;
my $doreq = shift;
my $nodelist = shift;
my $p_tmp = shift;
my %portservices = %$p_tmp;
my @nodes = ();
if ($nodelist) { @nodes = @$nodelist; }
my %nodebyip;
my @livenodes;
my %unknownnodes;
my $chaintab = xCAT::Table->new('chain', -create => 0);
if ($chaintab) {
%chainhash = %{ $chaintab->getNodesAttribs(\@nodes, ['currstate']) };
}
my $hoststab = xCAT::Table->new('hosts', -create => 0);
my %hostsents;
if ($hoststab) { %hostsents = %{ $hoststab->getNodesAttribs(\@nodes, ['ip']) }; }
my @ips;
my @ip6s;
foreach (@nodes) {
$unknownnodes{$_} = 1;
my $ip = undef;
if (($hostsents{$_}) && ($hostsents{$_}->[0]->{ip})) {
$ip = $hostsents{$_}->[0]->{ip};
$nodebyip{$ip} = $_;
$ip = xCAT::NetworkUtils->getipaddr($ip);
$nodebyip{$ip} = $_;
} else {
$ip = xCAT::NetworkUtils->getipaddr($_);
}
if (!defined $ip) {
my %rsp;
$rsp{name} = [$_];
$rsp{data} = ["Please make sure $_ exists in /etc/hosts or DNS or hosts table"];
$callback->({ node => [ \%rsp ] });
} else {
if ($ip =~ /:/) {
push @ip6s, $ip;
} else {
push @ips, $ip;
}
$nodebyip{$ip} = $_;
}
}
my $ret = {};
my $node;
my $fping;
my $ports = join ',', keys %portservices;
my %deadnodes;
foreach (@nodes) {
$deadnodes{$_} = 1;
}
#print "nmap -PE --send-ip -p $ports,3001 ".join(' ',@nodes) . "\n";
# open($fping,"nmap -PE --send-ip -p $ports,3001 ".join(' ',@nodes). " 2> /dev/null|") or die("Can't start nmap: $!");
my $currnode = '';
my $port;
my $state;
my %states;
my %rsp;
my $installquerypossible = 0;
my @nodesetnodes = ();
# get additional options from site table
my @nmap_options = xCAT::TableUtils->get_site_attribute("nmapoptions");
my $more_options = "";
if (defined($nmap_options[0])) {
$more_options = $nmap_options[0];
}
foreach my $ip6 (0, 1) { #first pass, ipv4, second pass ipv6
if ($ip6 and scalar(@ip6s)) {
open($fping, "nmap --unprivileged -6 -PS$ports,3001 -n --send-ip -p $ports,3001 $more_options " . join(' ', @ip6s) . " 2> /dev/null|") or die("Can't start nmap: $!");
} elsif (not $ip6 and scalar(@ips)) {
open($fping, "nmap --unprivileged -PA80,443,22 -PE -n --send-ip -p $ports,3001 $more_options " . join(' ', @ips) . " 2> /dev/null|") or die("Can't start nmap: $!");
} else { next; }
while (<$fping>) {
if (/Interesting ports on ([^ ]*)[: ]/ or /Nmap scan report for ([^ ]*)/) {
my $tmpnode = $1;
if ($currnode) { #if still thinking about last node, flush him out
my $status = join ',', sort keys %states;
my $appsd = "";
foreach my $portnum (keys %portservices) {
my $app_t = $portservices{$portnum};
if ($states{$app_t}) { $appsd .= $app_t . "=up,"; }
else { $appsd .= $app_t . "=down,"; }
}
$appsd =~ s/,$//;
my $target = $currnode;
if ($hostsents{$target} and $hostsents{$target}->[0]->{ip}) { $target = $hostsents{$target}->[0]->{ip}; }
if ($status or ($installquerypossible and $status = installer_query($target))) { #pingable, but no *clue* as to what the state may be
$ret->{$currnode}->{'status'} = "ping";
$ret->{$currnode}->{'appstatus'} = $status;
$ret->{$currnode}->{'appsd'} = $appsd;
$currnode = "";
%states = ();
} else {
push @nodesetnodes, $currnode; #Aggregate call to nodeset
}
}
$currnode = $tmpnode;
$currnode =~ s/:$//;
$currnode =~ s/\n$//;
my $nip;
if ($nip = xCAT::NetworkUtils->getipaddr($currnode)) { #reverse lookup may not resemble the nodename, key by ip
if ($nodebyip{$nip}) {
$currnode = $nodebyip{$nip};
}
}
$installquerypossible = 0; #reset possibility indicator
%rsp = ();
unless ($deadnodes{$currnode}) {
my $shortname;
foreach (keys %deadnodes) {
if (/\./) {
$shortname = $_;
$shortname =~ s/\..*//;
}
if ($currnode =~ /^$_\./ or ($shortname and $shortname eq $currnode)) {
$currnode = $_;
last;
}
}
}
delete $deadnodes{$currnode};
} elsif ($currnode) {
#if (/^MAC/) { #oops not all nmap records end with MAC
if (/^PORT/) { next; }
($port, $state) = split;
if ($port and $port =~ /^(\d*)\// and $state eq 'open') {
if ($1 eq "3001" and defined($chainhash{$currnode}->[0]->{currstate}) and $chainhash{$currnode}->[0]->{currstate} =~ /^install/) {
$installquerypossible = 1; #It is possible to actually query node
} elsif ($1 ne "3001") {
$states{ $portservices{$1} } = 1;
}
}
}
}
}
if ($currnode) {
my $status = join ',', sort keys %states;
my $appsd = "";
foreach my $portnum (keys %portservices) {
my $app_t = $portservices{$portnum};
if ($states{$app_t}) { $appsd .= $app_t . "=up,"; }
else { $appsd .= $app_t . "=down,"; }
}
$appsd =~ s/,$//;
my $target = $currnode;
if ($hostsents{$target} and $hostsents{$target}->[0]->{ip}) { $target = $hostsents{$target}->[0]->{ip}; }
if ($status or ($installquerypossible and $status = installer_query($target))) { #pingable, but no *clue* as to what the state may be
$ret->{$currnode}->{'status'} = "ping";
$ret->{$currnode}->{'appstatus'} = $status;
$ret->{$currnode}->{'appsd'} = $appsd;
$currnode = "";
%states = ();
} else {
push @nodesetnodes, $currnode; #Aggregate call to nodeset
}
}
if (@nodesetnodes) {
$doreq->({ command => ['nodeset'],
node => \@nodesetnodes,
arg => ['stat'] },
\&getstat);
foreach (@nodesetnodes) {
$ret->{$_}->{'status'} = $nodesetstats{$_};
}
}
foreach $currnode (sort keys %deadnodes) {
$ret->{$currnode}->{'status'} = "noping";
}
return $ret;
}
sub process_request_port {
my $request = shift;
my $callback = shift;
my $doreq = shift;
my $nodelist = shift;
my $p_tmp = shift;
my %portservices = %$p_tmp;
my @nodes = ();
if ($nodelist) { @nodes = @$nodelist; }
my %unknownnodes;
foreach (@nodes) {
$unknownnodes{$_} = 1;
my $packed_ip = undef;
$packed_ip = xCAT::NetworkUtils->getipaddr($_);
if (!defined $packed_ip) {
my %rsp;
$rsp{name} = [$_];
$rsp{data} = ["Please make sure $_ exists in /etc/hosts"];
$callback->({ node => [ \%rsp ] });
}
}
my $status = {};
if (@nodes > 0) {
my $node;
my $fping;
open($fping, "fping " . join(' ', @nodes) . " 2> /dev/null|") or die("Can't start fping: $!");
while (<$fping>) {
my %rsp;
my $node = $_;
$node =~ s/ .*//;
chomp $node;
if (/ is alive/) {
$status->{$node} = interrogate_node($node, $doreq, $p_tmp);
} elsif (/is unreachable/) {
$status->{$node}->{'status'} = "noping";
} elsif (/ address not found/) {
$status->{$node}->{'status'} = "nosuchhost";
}
}
}
return $status;
}
sub process_request_local_command {
my $request = shift;
my $callback = shift;
my $doreq = shift;
my $nodelist = shift;
my $p_tmp = shift;
my %cmdhash = %$p_tmp;
my @nodes = ();
if ($nodelist) { @nodes = @$nodelist; }
my $status = {};
if (@nodes > 0) {
foreach my $tmp_cmds (keys %cmdhash) {
my @cmds = split(',', $tmp_cmds);
my @apps = split(',', $cmdhash{$tmp_cmds});
my $index = 0;
foreach my $cmd (@cmds) {
my $nodes_string = join(',', @nodes);
my $ret = `$cmd $nodes_string`;
#print "ret=$ret\n";
if (($? == 0) && ($ret)) {
my @ret_array = split('\n', $ret);
foreach (@ret_array) {
my @a = split(':', $_);
chomp($a[1]);
if (exists($status->{ $a[0] })) {
$status->{ $a[0] } .= "," . $apps[$index] . "=" . $a[1];
} else {
$status->{ $a[0] } = $apps[$index] . "=" . $a[1];
}
}
}
$index++;
}
}
}
return $status;
}
sub process_request_remote_command {
my $request = shift;
my $callback = shift;
my $doreq = shift;
my $nodelist = shift;
my $p_tmp = shift;
my %cmdhash = %$p_tmp;
my @nodes = ();
if ($nodelist) { @nodes = @$nodelist; }
my $status = {};
if (@nodes > 0) {
foreach my $tmp_cmds (keys %cmdhash) {
my @cmds = split(',', $tmp_cmds);
my @apps = split(',', $cmdhash{$tmp_cmds});
my $index = 0;
foreach my $cmd (@cmds) {
my @ret = xCAT::InstUtils->xcmd($callback, $doreq, "xdsh", \@nodes, $cmd, 1);
if (@ret) {
foreach (@ret) {
my @a = split(':', $_, 2);
chomp($a[1]); #remove newline
$a[1] =~ s/^\s+//; #remove leading white spaces
$a[1] =~ s/\s+$//; #remove tailing white spaces
if (exists($status->{ $a[0] })) {
$status->{ $a[0] } .= "," . $apps[$index] . "=" . $a[1];
} else {
$status->{ $a[0] } = $apps[$index] . "=" . $a[1];
}
}
}
$index++;
}
}
}
return $status;
}
sub process_request {
my $request = shift;
my $callback = shift;
my $doreq = shift;
%nodesetstats = ();
my $command = $request->{command}->[0];
my $separator = "XXXXXYYYYYZZZZZ";
my $usefping;
if (ref $request->{arg}) {
@ARGV = @{ $request->{arg} };
GetOptions(
'f|useping' => \$usefping
);
}
if ($command eq "nodestat_internal") {
#if ( -x '/usr/bin/nmap' ) {
# my %portservices = (
# '22' => 'sshd',
# '15002' => 'pbs',
# '8002' => 'xend',
# );
#
# return process_request_nmap($request, $callback, $doreq, $request->{node}, \%portservices);
# }
#handle ports and nodelist.status
my $status = {};
if (exists($request->{'portapps'})) {
for (my $i = 1 ; $i <= $request->{'portapps'}->[0] ; $i++) {
my %portservices = ();
my @apps = split(',', $request->{"portapps$i"}->[0]);
my @ports = split(',', $request->{ "portapps$i" . "port" }->[0]);
my $nodes = $request->{ "portapps$i" . "node" };
for (my $j = 0 ; $j < @ports ; $j++) {
$portservices{ $ports[$j] } = $apps[$j];
}
my $ret = {};
if (not $usefping and -x '/usr/bin/nmap') {
$ret = process_request_nmap($request, $callback, $doreq, $nodes, \%portservices);
} else {
$ret = process_request_port($request, $callback, $doreq, $nodes, \%portservices);
}
%$status = (%$status, %$ret);
}
}
#handle local commands
if (exists($request->{'cmdapps'})) {
for (my $i = 1 ; $i <= $request->{'cmdapps'}->[0] ; $i++) {
my %cmdhash = ();
my @apps = split(',', $request->{"cmdapps$i"}->[0]);
my @cmds = split(',', $request->{ "cmdapps$i" . "cmd" }->[0]);
my $nodes = $request->{ "cmdapps$i" . "node" };
for (my $j = 0 ; $j < @cmds ; $j++) {
$cmdhash{ $cmds[$j] } = $apps[$j];
}
my $ret = process_request_local_command($request, $callback, $doreq, $nodes, \%cmdhash);
#print Dumper($ret);
foreach my $node1 (keys(%$ret)) {
if (exists($status->{$node1})) {
my $appstatus = $status->{$node1}->{'appstatus'};
if ($appstatus) { $status->{$node1}->{'appstatus'} .= "," . $ret->{$node1}; }
else { $status->{$node1}->{'appstatus'} = $ret->{$node1}; }
my $appsd = $status->{$node1}->{'appsd'};
if ($appsd) { $status->{$node1}->{'appsd'} .= "," . $ret->{$node1}; }
else { $status->{$node1}->{'appsd'} = $ret->{$node1}; }
} else {
$status->{$node1}->{'appstatus'} = $ret->{$node1};
$status->{$node1}->{'appsd'} = $ret->{$node1};
}
}
}
}
#handle local l commands
if (exists($request->{'lcmdapps'})) {
for (my $i = 1 ; $i <= $request->{'lcmdapps'}->[0] ; $i++) {
my %cmdhash = ();
my @apps = split(',', $request->{"lcmdapps$i"}->[0]);
my @cmds = split(',', $request->{ "lcmdapps$i" . "cmd" }->[0]);
my $nodes = $request->{ "lcmdapps$i" . "node" };
for (my $j = 0 ; $j < @cmds ; $j++) {
$cmdhash{ $cmds[$j] } = $apps[$j];
}
my $ret = process_request_local_command($request, $callback, $doreq, $nodes, \%cmdhash);
foreach my $node1 (keys(%$ret)) {
if (exists($status->{$node1})) {
my $appstatus = $status->{$node1}->{'appstatus'};
if ($appstatus) { $status->{$node1}->{'appstatus'} .= "," . $ret->{$node1}; }
else { $status->{$node1}->{'appstatus'} = $ret->{$node1}; }
my $appsd = $status->{$node1}->{'appsd'};
if ($appsd) { $status->{$node1}->{'appsd'} .= "," . $ret->{$node1}; }
else { $status->{$node1}->{'appsd'} = $ret->{$node1}; }
} else {
$status->{$node1}->{'appstatus'} = $ret->{$node1};
$status->{$node1}->{'appsd'} = $ret->{$node1};
}
}
}
}
#handle remote commands
if (exists($request->{'dcmdapps'})) {
for (my $i = 1 ; $i <= $request->{'dcmdapps'}->[0] ; $i++) {
my %dcmdhash = ();
my @apps = split(',', $request->{"dcmdapps$i"}->[0]);
my @dcmds = split(',', $request->{ "dcmdapps$i" . "dcmd" }->[0]);
my $nodes = $request->{ "dcmdapps$i" . "node" };
for (my $j = 0 ; $j < @dcmds ; $j++) {
$dcmdhash{ $dcmds[$j] } = $apps[$j];
}
my $ret = process_request_remote_command($request, $callback, $doreq, $nodes, \%dcmdhash);
foreach my $node1 (keys(%$ret)) {
if (exists($status->{$node1})) {
my $appstatus = $status->{$node1}->{'appstatus'};
if ($appstatus) { $status->{$node1}->{'appstatus'} .= "," . $ret->{$node1}; }
else { $status->{$node1}->{'appstatus'} = $ret->{$node1}; }
my $appsd = $status->{$node1}->{'appsd'};
if ($appsd) { $status->{$node1}->{'appsd'} .= "," . $ret->{$node1}; }
else { $status->{$node1}->{'appsd'} = $ret->{$node1}; }
} else {
$status->{$node1}->{'appstatus'} = $ret->{$node1};
$status->{$node1}->{'appsd'} = $ret->{$node1};
}
}
}
}
#nodestat_internal command the output, nodestat command will collect it
foreach my $node1 (sort keys(%$status)) {
my %rsp;
$rsp{name} = [$node1];
my $st = $status->{$node1}->{'status'};
my $ast = $status->{$node1}->{'appstatus'};
my $appsd = $status->{$node1}->{'appsd'};
$st = $st ? $st : '';
$ast = $ast ? $ast : '';
$appsd = $appsd ? $appsd : '';
$rsp{data}->[0] = "$st$separator$ast$separator$appsd";
$callback->({ node => [ \%rsp ] });
}
} else { #nodestat command
#first collect the status from the nodes
my $reqcopy = {%$request};
$reqcopy->{command}->[0] = 'nodestat_internal';
my $ret = xCAT::Utils->runxcmd($reqcopy, $doreq, 0, 1);
#print Dumper($ret);
my $status = {};
my @noping_nodes = ();
my $power = $request->{'power'}->[0];
foreach my $tmpdata (@$ret) {
if ($tmpdata =~ /([^:]+): (.*)$separator(.*)$separator(.*)/) {
#print "node=$1, status=$2, appstatus=$3, appsd=$4\n";
if ($status->{$1}->{'status'}) {
$status->{$1}->{'status'} = $status->{$1}->{'status'} . ",$2";
} else {
$status->{$1}->{'status'} = $2;
}
if ($status->{$1}->{'appstatus'}) {
$status->{$1}->{'appstatus'} = $status->{$1}->{'appstatus'} . ",$3";
} else {
$status->{$1}->{'appstatus'} = $3;
}
if ($status->{$1}->{'appsd'}) {
$status->{$1}->{'appsd'} = $status->{$1}->{'appsd'} . ",$4";
} else {
$status->{$1}->{'appsd'} = $4;
}
if (($power) && ($2 eq "noping")) {
push(@noping_nodes, $1);
}
} else {
my $rsp;
$rsp->{data}->[0] = "$tmpdata";
xCAT::MsgUtils->message("I", $rsp, $callback);
}
}
#print Dumper($status);
#get power status for noping nodes
if (($power) && (@noping_nodes > 0)) {
#print "noping_nodes=@noping_nodes\n";
my $ret = xCAT::Utils->runxcmd(
{
command => ['rpower'],
node => \@noping_nodes,
arg => ['stat']
},
$doreq, 0, 1);
foreach my $tmpdata (@$ret) {
if ($tmpdata =~ /([^:]+): (.*)/) {
$status->{$1}->{'status'} = "noping($2)";
} else {
my $rsp;
$rsp->{data}->[0] = "$tmpdata";
xCAT::MsgUtils->message("I", $rsp, $callback);
}
}
}
#print Dumper($request);
my $update = $request->{'update'}->[0];
my $quiet = $request->{'quiet'}->[0];
#show the output
if (!$quiet) {
foreach my $node1 (sort keys(%$status)) {
my %rsp;
$rsp{name} = [$node1];
my $st = $status->{$node1}->{'status'};
my $ast = $status->{$node1}->{'appstatus'};
if ($st) {
if ($st eq 'ping') { $st = $ast ? "$ast" : "$st"; }
else { $st = $ast ? "$st,$ast" : "$st"; }
} else {
$st = $ast;
}
$rsp{data}->[0] = $st;
$callback->({ node => [ \%rsp ] });
}
}
#update the nodelist table
if ($update) {
my $nodetab = xCAT::Table->new('nodelist', -create => 1);
if ($nodetab) {
my $status1 = {};
#get current values and compare with the new value to decide if update of db is necessary
my @nodes1 = keys(%$status);
my $stuff = $nodetab->getNodesAttribs(\@nodes1, [ 'node', 'status', 'appstatus' ]);
#get current local time
my (
$sec, $min, $hour, $mday, $mon,
$year, $wday, $yday, $isdst
)
= localtime(time);
my $currtime = sprintf("%02d-%02d-%04d %02d:%02d:%02d",
$mon + 1, $mday, $year + 1900,
$hour, $min, $sec);
foreach my $node1 (@nodes1) {
my $oldstatus = $stuff->{$node1}->[0]->{status};
my $newstatus = $status->{$node1}->{status};
if ($newstatus) {
if ((!$oldstatus) || ($newstatus ne $oldstatus)) {
$status1->{$node1}->{status} = $newstatus;
$status1->{$node1}->{statustime} = $currtime;
}
}
else {
if ($oldstatus) {
$status1->{$node1}->{status} = "";
$status1->{$node1}->{statustime} = "";
}
}
if ($newstatus =~ /noping/) {
$status1->{$node1}->{appstatus} = "";
} else {
my $oldappstatus = $stuff->{$node1}->[0]->{'appstatus'};
my $newappstatus = $status->{$node1}->{'appsd'};
while ($newappstatus =~ /(\w+)\=(\w+)/) {
my $tmp1 = $1;
my $tmp2 = $2;
if ($oldappstatus) {
if ($oldappstatus =~ /$tmp1\=/) {
$oldappstatus =~ s/$tmp1\=\w+/$tmp1\=$tmp2/g;
} else {
$oldappstatus = $oldappstatus . "\,$tmp1\=$tmp2";
}
} else {
$oldappstatus = "$tmp1\=$tmp2";
}
$newappstatus =~ s/(\w+)\=(\w+)//;
}
$status1->{$node1}->{appstatus} = $oldappstatus;
}
$status1->{$node1}->{appstatustime} = $currtime;
}
#print Dumper($status1);
$nodetab->setNodesAttribs($status1);
}
}
}
}
sub usage
{
my $cb = shift;
my $retcode=shift;
my $rsp = {};
$rsp->{data}->[0] = "Usage:";
$rsp->{data}->[1] = " nodestat [noderange] [-m|--usemon] [-p|powerstat] [-f|--usefping] [-u|--updatedb]";
$rsp->{data}->[2] = " nodestat [-h|--help|-v|--version]";
if($retcode){
$rsp->{errorcode}->[0]=$retcode;
}
xCAT::MsgUtils->message("I", $rsp, $cb);
}
#--------------------------------------------------------------------------------
=head3 getStatusMonsettings
This function goes to the monsetting table to retrieve the settings related to
the node status and app status monitoring.
Arguments:
none.
Returns:
a hash that has settings from the monsetting table for node status and
app status monitoring. For example:
( 'APPS'=>[ll,gpfs],
'll' =>
{
'group' => 'service,compute',
'port' => '5001'
},
'gpfs' =>
{
'group' => 'service',
'cmd' => '/tmp/gpfscmd'
};
)
=cut
#--------------------------------------------------------------------------------
sub getStatusMonsettings {
my %apps = ();
my $tab = xCAT::Table->new('monsetting');
if (defined($tab)) {
my ($ent) = $tab->getAttribs({ name => 'xcatmon', key => 'apps' }, 'value');
if (defined($ent)) {
my $tmp_list = $ent->{value};
if ($tmp_list) {
my @applist = split(',', $tmp_list);
foreach my $app (@applist) {
$apps{$app} = {};
}
$apps{'APPS'} = \@applist;
my @results = $tab->getAttribs({ name => 'xcatmon' }, 'key', 'value');
if (@results) {
foreach (@results) {
my $key = $_->{key};
my $value = $_->{value};
if (exists($apps{$key})) {
my @tem_value = split(',', $value);
foreach my $pair (@tem_value) {
my @tmp_action = split('=', $pair);
if (exists($apps{$key}->{ $tmp_action[0] })) {
$apps{$key}->{ $tmp_action[0] } = $apps{$key}->{ $tmp_action[0] } . "," . $tmp_action[1];
} else {
$apps{$key}->{ $tmp_action[0] } = $tmp_action[1];
}
}
}
}
}
}
}
}
return %apps;
}
#--------------------------------------------------------------------------------
=head3 getNodeStatusAndAppstatus
This function goes to the xCAT nodelist table to retrieve the saved node status and appstatus
for all the node that are managed by local nodes.
Arguments:
nodelist--- an array of nodes
Returns:
a hash pointer that has the node status and appstatus. The format is:
{ node1=> {
status=>'active',appstatus=>'sshd=up,ll=up,gpfs=down'
} ,
node2=> {
status=>'active',appstatus=>'sshd=up,ll=down,gpfs=down'
}
}
=cut
#--------------------------------------------------------------------------------
sub getMonNodesStatusAndAppStatus {
my @nodes = @_;
my %status = ();
my $table = xCAT::Table->new("nodelist", -create => 1);
my $tabdata = $table->getNodesAttribs(\@nodes, [ 'node', 'status', 'appstatus' ]);
foreach my $node (@nodes) {
my $tmp1 = $tabdata->{$node}->[0];
if ($tmp1) {
$status{$node}->{status} = $tmp1->{status};
$status{$node}->{appstatus} = $tmp1->{appstatus};
}
}
return %status;
}
1;