package xCAT_plugin::nodestat;
BEGIN
{
$::XCATROOT = $ENV{'XCATROOT'} ? $ENV{'XCATROOT'} : '/opt/xcat';
}
use lib "$::XCATROOT/lib/perl";
use strict;
use warnings;
use Socket;
my $inet6support = eval {
require Socket6;
require IO::Socket::INET6;
1;
};
use IO::Handle;
use Getopt::Long;
use Data::Dumper;
use xCAT::GlobalDef;
use xCAT::NetworkUtils;
require xCAT::Utils;
require xCAT::TableUtils;
require xCAT::ServiceNodeUtils;
my %nodesetstats;
my %chainhash;
my %default_ports = (
'ftp' => '21',
'ssh' => '22',
'sshd' => '22',
'pbs' => '15002',
'pbs_mom' => '15002',
'xend' => '8002',
'll' => '9616',
'loadl' => '9616',
'loadl_master' => '9616',
'loadleveler' => '9616',
'gpfs' => '1191',
'rdp' => '3389',
'msrpc' => '135',
);
sub handled_commands {
return {
nodestat => 'nodestat',
nodestat_internal => 'nodestat',
};
}
sub pinghost {
my $node = shift;
my $rc = system("ping -q -n -c 1 -w 1 $node > /dev/null");
if ($rc == 0) {
return 1;
} else {
return 0;
}
}
sub nodesockopen {
my $node = shift;
my $port = shift;
my $socket;
my $addr = gethostbyname($node);
my $sin = sockaddr_in($port,$addr);
my $proto = getprotobyname('tcp');
socket($socket,PF_INET,SOCK_STREAM,$proto) || return 0;
connect($socket,$sin) || return 0;
return 1;
}
sub installer_query {
my $node = shift;
my $destport = 3001;
my $socket;
my $text = "";
if ($inet6support) {
$socket = IO::Socket::INET6->new(PeerAddr => $node, PeerPort => 3001);
unless ($socket) { return 0; }
} else {
my $proto = getprotobyname('tcp');
socket($socket,PF_INET,SOCK_STREAM,$proto) || return 0;
my $addr = gethostbyname($node);
my $sin = sockaddr_in($destport,$addr);
connect($socket,$sin) || return 0;
}
print $socket "stat \n";
$socket->flush;
while (<$socket>) {
$text.=$_;
}
$text =~ s/\n.*//;
return $text;
close($socket);
}
sub getstat {
my $response = shift;
foreach (@{$response->{node}}) {
$nodesetstats{$_->{name}->[0]} = $_->{data}->[0];
}
}
#-------------------------------------------------------
=head3 preprocess_request
Check and setup for hierarchy
=cut
#-------------------------------------------------------
sub preprocess_request
{
my $req = shift;
my $cb = shift;
my %sn;
if (defined $req->{_xcatpreprocessed}->[0] && $req->{_xcatpreprocessed}->[0] == 1) { return [$req]; }
#exit if preprocessed
my $command = $req->{command}->[0];
if ($command eq "nodestat") {
@ARGV=();
my $args=$req->{arg};
if ($args) {
@ARGV = @{$args};
}
# parse the options
$::UPDATE=0;
$::QUITE=0;
$::MON=0;
$::POWER=0;
#Getopt::Long::Configure("posix_default");
#Getopt::Long::Configure("no_gnu_compat");
Getopt::Long::Configure("bundling");
$Getopt::Long::ignorecase=0;
if (!GetOptions(
'm|usemon' => \$::MON,
'q|quite' => \$::QUITE, #this is a internal flag used by monitoring
'u|updatedb' => \$::UPDATE,
'p|powerstat' => \$::POWER,
'h|help' => \$::HELP,
'v|version' => \$::VERSION))
{
&usage($cb);
return(1);
}
if ($::HELP) {
&usage($cb);
return(0);
}
if ($::VERSION) {
my $version = xCAT::Utils->Version();
my $rsp={};
$rsp->{data}->[0] = "$version";
xCAT::MsgUtils->message("I", $rsp, $cb);
return(0);
}
my $nodes = $req->{node};
if (!$nodes)
{
&usage($cb);
return (1);
}
$req->{'update'}->[0]=$::UPDATE;
$req->{'quite'}->[0]=$::QUITE;
$req->{'mon'}->[0]=$::MON;
$req->{'power'}->[0]=$::POWER;
return [$req];
}
#the following is for nodestat_internal command
my $nodes = $req->{node};
my $service = "xcat";
my @requests;
if ($nodes) {
my $usenmapfrommn=0;
if (-x '/usr/bin/nmap' or -x '/usr/local/bin/nmap') {
#my $sitetab = xCAT::Table->new('site');
#if ($sitetab) {
#(my $ref) = $sitetab->getAttribs({key => 'useNmapfromMN'}, 'value');
my @entries = xCAT::TableUtils->get_site_attribute("useNmapfromMN");
my $t_entry = $entries[0];
if (defined($t_entry)) {
if ($t_entry =~ /1|yes|YES|Y|y/) { $usenmapfrommn=1; }
}
#}
}
#get monsettings
my %apps = ();
my $mon=$req->{'mon'}->[0];
if ($mon == 1) { %apps=getStatusMonsettings(); }
#if no apps specified in the monsetting table, add sshd, pbs and xend
if (keys(%apps) == 0) {
$apps{'sshd'}->{'group'} = "ALL"; #ALL means anything on the nodelist table, it is different from all
$apps{'sshd'}->{'port'} = "22";
$apps{'https'}->{'group'} = "ALL"; #ALL means anything on the nodelist table, it is different from all
$apps{'https'}->{'port'} = "443";
$apps{'pbs'}->{'group'} = "ALL";
$apps{'pbs'}->{'port'} = "15002";
$apps{'xend'}->{'group'} = "ALL";
$apps{'xend'}->{'port'} = "8002";
$apps{'rdp'}->{'group'} = "ALL";
$apps{'rdp'}->{'port'} = "3389";
$apps{'msrpc'}->{'group'} = "ALL";
$apps{'msrpc'}->{'port'} = "135";
$apps{'APPS'}=['sshd', 'https', 'pbs', 'xend'];
} else {
#go thorugh the settings and put defaults in
foreach my $app (keys(%apps)) {
if ($app eq 'APPS') { next; }
if (!exists($apps{$app}->{'group'})) { $apps{$app}->{'group'} = "ALL"; }
if (exists($apps{$app}->{'cmd'}) || exists($apps{$app}->{'dcmd'}) || exists($apps{$app}->{'lcmd'})) { next; }
if (exists($apps{$app}->{'port'})) { next; }
#add port number in if nothing is specified
if (exists($default_ports{$app})) { $apps{$app}->{'port'} = $default_ports{$app}; }
else {
my $p=`grep "^$app" /etc/services`;
if ($? == 0) {
my @a_list=sort(split('\n', $p));
my @a_temp=split('/',$a_list[0]);
my @a=split(' ', $a_temp[0]);
$apps{$app}->{'port'}=$a[1];
} else {
my $rsp={};
$rsp->{data}->[0]= "Cannot find port number for application $app. Please either specify a port number or a command in monsetting table for $app.";;
xCAT::MsgUtils->message("I", $rsp, $cb);
return (0);
}
}
}
#always add sshd
if (!exists($apps{'ssh'}) || !exists($apps{'sshd'}) ) {
$apps{'sshd'}->{'group'} = "ALL";
$apps{'sshd'}->{'port'} = "22";
my $pa=$apps{'APPS'};
push @$pa, 'sshd';
}
}
#print Dumper(%apps);
# find service nodes for requested nodes
# build an individual request for each service node
my $sn = xCAT::ServiceNodeUtils->get_ServiceNode($nodes, $service, "MN");
#get the member for each group
my %groups=();
foreach my $app (keys %apps) {
if ($app eq 'APPS') { next; }
my $group=$apps{$app}->{'group'};
if (($group) && ($group ne "ALL") && (! exists($groups{$group}))) {
my @tmp_nodes = xCAT::NodeRange::noderange($group);
foreach (@tmp_nodes) {
$groups{$group}->{$_}=1;
}
}
}
# build each request for each service node
my $all_apps=$apps{'APPS'};
my %all_porthash=(); #stores all the port apps if usenmapfrommn=1
my %lcmdhash=(); #('myapp2,myapp3'=> {
# lcmd=>'/tmp/mycmd1,/usr/bin/date',
# node=>[node1,node2]
# }
#)
foreach my $snkey (keys %$sn)
{
my $reqcopy = {%$req};
$reqcopy->{node} = $sn->{$snkey};
$reqcopy->{'_xcatdest'} = $snkey;
$reqcopy->{_xcatpreprocessed}->[0] = 1;
$reqcopy->{'useNmapfromMN'}->[0]=$usenmapfrommn;
$reqcopy->{'allapps'}=$all_apps;
my %porthash=(); #('sshd,ll'=> {
# port=>'22,5001',
# node=>[node1,node2]
# }
#)
my %cmdhash=(); #('gpfs,myapp'=> {
# cmd=>'/tmp/mycmd1,/usr/bin/date',
# node=>[node1,node2]
# }
#)
my %dcmdhash=(); #('myapp2,myapp3'=> {
# dcmd=>'/tmp/mycmd1,/usr/bin/date',
# node=>[node1,node2]
# }
#)
my @nodes_for_sn=@{$sn->{$snkey}};
foreach my $node (@nodes_for_sn) {
my @ports;
my @portapps;
my @cmds;
my @cmdapps;
my @dcmds;
my @dcmdapps;
my @lcmdapps;
my @lcmds;
foreach my $app (keys %apps) {
if ($app eq 'APPS') { next; }
my $group=$apps{$app}->{'group'};
if (($group eq "ALL") || ($groups{$group}->{$node})) {
#print "app=$app\n";
if (exists($apps{$app}->{'port'})) {
push @ports, $apps{$app}->{'port'};
push @portapps, $app;
}
elsif (exists($apps{$app}->{'cmd'})) {
push @cmds, $apps{$app}->{'cmd'};
push @cmdapps, $app;
}
elsif (exists($apps{$app}->{'dcmd'})) {
push @dcmds, $apps{$app}->{'dcmd'};
push @dcmdapps, $app;
}
elsif (exists($apps{$app}->{'lcmd'})) {
push @lcmds, $apps{$app}->{'lcmd'};
push @lcmdapps, $app;
}
}
}
#print "ports=@ports\n";
#print "portapps=@portapps\n";
#print "cmds=@cmds\n";
#print "cmdapps=@cmdapps\n";
#print "dcmds=@dcmds\n";
#print "dcmdapps=@dcmdapps\n";
if (@portapps>0) {
my $tmpapps=join(',', @portapps);
if (($usenmapfrommn==1) && (@cmdapps==0) && (@dcmdapps==0) && (@lcmdapps==0)) {
#this is the case where mn handles ports for all nodes using nmap
#The current limitation is that when there are cmd or dcmd specified for the node
# nmap has to be done on the service node because if both mn and sn update the appstatus
# one will overwites the other.
if (exists($all_porthash{$tmpapps})) {
my $pa=$all_porthash{$tmpapps}->{'node'};
push @$pa, $node;
} else {
$all_porthash{$tmpapps}->{'node'}=[$node];
$all_porthash{$tmpapps}->{'port'}=join(',', @ports);
}
} else {
if (exists($porthash{$tmpapps})) {
my $pa=$porthash{$tmpapps}->{'node'};
push @$pa, $node;
} else {
$porthash{$tmpapps}->{'node'}=[$node];
$porthash{$tmpapps}->{'port'}=join(',', @ports);
}
}
}
if (@cmdapps>0) {
my $tmpapps=join(',', @cmdapps);
if (exists($cmdhash{$tmpapps})) {
my $pa=$cmdhash{$tmpapps}->{'node'};
push @$pa, $node;
} else {
$cmdhash{$tmpapps}->{'node'}=[$node];
$cmdhash{$tmpapps}->{'cmd'}=join(',', @cmds);
}
}
if (@dcmdapps>0) {
my $tmpapps=join(',', @dcmdapps);
if (exists($dcmdhash{$tmpapps})) {
my $pa=$dcmdhash{$tmpapps}->{'node'};
push @$pa, $node;
} else {
$dcmdhash{$tmpapps}->{'node'}=[$node];
$dcmdhash{$tmpapps}->{'dcmd'}=join(',', @dcmds);
}
}
if (@lcmdapps>0) {
my $i=0;
foreach my $lapp (@lcmdapps) {
if (exists($lcmdhash{$lapp})) {
my $pa=$lcmdhash{$lapp}->{'node'};
push @$pa, $node;
} else {
$lcmdhash{$lapp}->{'node'}=[$node];
$lcmdhash{$lapp}->{'lcmd'}=$lcmds[$i];
}
$i++;
}
}
} #end foreach (@nodes_for_sn)
#print Dumper(%porthash);
#print "cmdhash=" . Dumper(%cmdhash);
#now push the settings into the requests
my $i=1;
if ((keys(%porthash) == 0) && (keys(%cmdhash) == 0) && (keys(%dcmdhash) == 0) && (keys(%lcmdhash) == 0)) { next; }
foreach my $tmpapps (keys %porthash) {
$reqcopy->{'portapps'}->[0]= scalar keys %porthash;
$reqcopy->{"portapps$i"}->[0]= $tmpapps;
$reqcopy->{"portapps$i" . "port"}->[0]= $porthash{$tmpapps}->{'port'};
$reqcopy->{"portapps$i" . "node"} = $porthash{$tmpapps}->{'node'};;
$i++;
}
$i=1;
foreach my $tmpapps (keys %cmdhash) {
$reqcopy->{'cmdapps'}->[0]= scalar keys %cmdhash;
$reqcopy->{"cmdapps$i"}->[0]= $tmpapps;
$reqcopy->{"cmdapps$i" . "cmd"}->[0]= $cmdhash{$tmpapps}->{'cmd'};
$reqcopy->{"cmdapps$i" . "node"} = $cmdhash{$tmpapps}->{'node'};;
$i++;
}
$i=1;
foreach my $tmpapps (keys %dcmdhash) {
$reqcopy->{'dcmdapps'}->[0]= scalar keys %dcmdhash;
$reqcopy->{"dcmdapps$i"}->[0]= $tmpapps;
$reqcopy->{"dcmdapps$i" . "dcmd"}->[0]= $dcmdhash{$tmpapps}->{'dcmd'};
$reqcopy->{"dcmdapps$i" . "node"} = $dcmdhash{$tmpapps}->{'node'};
$i++;
}
#done
push @requests, $reqcopy;
} #enf sn_key
#print "apps=" . Dumper(%apps);
#mn handles all nmap when useNmapfromMN=1 on the site table
if (($usenmapfrommn == 1) && (keys(%all_porthash) > 0)) {
my @hostinfo=xCAT::NetworkUtils->determinehostname();
my %iphash=();
foreach(@hostinfo) {$iphash{$_}=1;}
my $handled=0;
foreach my $req (@requests) {
my $currsn=$req->{'_xcatdest'};
if (exists($iphash{$currsn})) {
my $i=1;
foreach my $tmpapps (keys %all_porthash) {
$req->{'portapps'}->[0]= scalar keys %all_porthash;
$req->{"portapps$i"}->[0]= $tmpapps;
$req->{"portapps$i" . "port"}->[0]= $all_porthash{$tmpapps}->{'port'};
$req->{"portapps$i" . "node"} = $all_porthash{$tmpapps}->{'node'};;
$i++;
}
$handled=1;
last;
}
}
if (!$handled) {
my $reqcopy = {%$req};
$reqcopy->{_xcatpreprocessed}->[0] = 1;
$reqcopy->{'useNmapfromMN'}->[0]=$usenmapfrommn;
$reqcopy->{'allapps'}=$all_apps;
my $i=1;
foreach my $tmpapps (keys %all_porthash) {
$reqcopy->{'portapps'}->[0]= scalar keys %all_porthash;
$reqcopy->{"portapps$i"}->[0]= $tmpapps;
$reqcopy->{"portapps$i" . "port"}->[0]= $all_porthash{$tmpapps}->{'port'};
$reqcopy->{"portapps$i" . "node"} = $all_porthash{$tmpapps}->{'node'};;
$i++;
}
push @requests, $reqcopy;
}
}
#if ($usenmapfrommn) {
# my $reqcopy = {%$req};
# $reqcopy->{'update'}->[0]=$::UPDATE;
# $reqcopy->{'useNmapfromMN'}->[0]=1;
# if (!$::UPDATE) {
# push @requests, $reqcopy;
# return \@requests; #do not distribute, nodestat seems to lose accuracy and slow down distributed, if using nmap
# }
#}
#now handle local commands
#print "lcmdhash=" . Dumper(%lcmdhash);
if (keys(%lcmdhash) > 0) {
my @hostinfo=xCAT::NetworkUtils->determinehostname();
my %iphash=();
foreach(@hostinfo) {$iphash{$_}=1;}
my $handled=0;
foreach my $req (@requests) {
my $currsn=$req->{'_xcatdest'};
if (exists($iphash{$currsn})) {
my $i=1;
foreach my $lapp (keys %lcmdhash) {
$req->{'lcmdapps'}->[0]= scalar keys %lcmdhash;
$req->{"lcmdapps$i"}->[0]= $lapp;
$req->{"lcmdapps$i" . "cmd"}->[0]= $lcmdhash{$lapp}->{'lcmd'};
$req->{"lcmdapps$i" . "node"} = $lcmdhash{$lapp}->{'node'};;
$i++;
}
$handled=1;
last;
}
}
if (!$handled) {
my $reqcopy = {%$req};
$reqcopy->{_xcatpreprocessed}->[0] = 1;
$reqcopy->{'allapps'}=$all_apps;
my $i=1;
foreach my $lapp (keys %lcmdhash) {
$reqcopy->{'lcmdapps'}->[0]= scalar keys %lcmdhash;
$reqcopy->{"lcmdapps$i"}->[0]= $lapp;
$reqcopy->{"lcmdapps$i" . "cmd"}->[0]= $lcmdhash{$lapp}->{'lcmd'};
$reqcopy->{"lcmdapps$i" . "node"} = $lcmdhash{$lapp}->{'node'};;
$i++;
}
push @requests, $reqcopy;
}
}
}
return \@requests;
}
sub interrogate_node { #Meant to run against confirmed up nodes
my $node=shift;
my $doreq=shift;
my $p_tmp=shift;
my %portservices = %$p_tmp;
my $status = "";
my $appsd=""; #detailed status
my $ret={};
$ret->{'status'}="ping";
foreach my $port (keys(%portservices)) {
if (nodesockopen($node,$port)) {
$status.=$portservices{$port} . ",";
$appsd.=$portservices{$port} . "=up,";
} else {
$appsd.=$portservices{$port} . "=down,";
}
}
$status =~ s/,$//;
$appsd =~ s/,$//;
$ret->{'appsd'}=$appsd;
if ($status) {
$ret->{'appstatus'}=$status;
return $ret;
}
if ($status = installer_query($node)) {
$ret->{'status'}=$status;
return $ret;
} else { #pingable, but no *clue* as to what the state may be
$doreq->({command=>['nodeset'],
node=>[$node],
arg=>['stat']},
\&getstat);
$ret->{'status'} = 'ping '.$nodesetstats{$node};
return $ret;
}
}
sub process_request_nmap {
my $request = shift;
my $callback = shift;
my $doreq = shift;
my $nodelist=shift;
my $p_tmp=shift;
my %portservices = %$p_tmp;
my @nodes =();
if ($nodelist) { @nodes=@$nodelist;}
my %nodebyip;
my @livenodes;
my %unknownnodes;
my $chaintab = xCAT::Table->new('chain',-create=>0);
if ($chaintab) {
%chainhash = %{$chaintab->getNodesAttribs(\@nodes,['currstate'])};
}
my $hoststab = xCAT::Table->new('hosts',-create=>0);
my %hostsents;
if ($hoststab) { %hostsents = %{$hoststab->getNodesAttribs(\@nodes,['ip'])}; }
my @ips;
my @ip6s;
foreach (@nodes) {
$unknownnodes{$_}=1;
my $ip = undef;
if (($hostsents{$_}) && ($hostsents{$_}->[0]->{ip})) {
$ip = $hostsents{$_}->[0]->{ip};
$nodebyip{$ip} = $_;
$ip = xCAT::NetworkUtils->getipaddr($ip);
$nodebyip{$ip} = $_;
} else {
$ip = xCAT::NetworkUtils->getipaddr($_);
}
if( !defined $ip) {
my %rsp;
$rsp{name}=[$_];
$rsp{data} = [ "Please make sure $_ exists in /etc/hosts or DNS or hosts table" ];
$callback->({node=>[\%rsp]});
} else {
if ($ip =~ /:/) {
push @ip6s,$ip;
} else {
push @ips,$ip;
}
$nodebyip{$ip} = $_;
}
}
my $ret={};
my $node;
my $fping;
my $ports = join ',',keys %portservices;
my %deadnodes;
foreach (@nodes) {
$deadnodes{$_}=1;
}
#print "nmap -PE --send-ip -p $ports,3001 ".join(' ',@nodes) . "\n";
# open($fping,"nmap -PE --send-ip -p $ports,3001 ".join(' ',@nodes). " 2> /dev/null|") or die("Can't start nmap: $!");
my $currnode='';
my $port;
my $state;
my %states;
my %rsp;
my $installquerypossible=0;
my @nodesetnodes=();
foreach my $ip6 (0,1) { #first pass, ipv4, second pass ipv6
if ($ip6 and scalar(@ip6s)) {
open($fping,"nmap -6 -PS$ports,3001 -n --send-ip -p $ports,3001 ".join(' ',@ip6s). " 2> /dev/null|") or die("Can't start nmap: $!");
} elsif (not $ip6 and scalar(@ips)) {
open($fping,"nmap -PE -n --send-ip -p $ports,3001 ".join(' ',@ips). " 2> /dev/null|") or die("Can't start nmap: $!");
} else { next; }
while (<$fping>) {
if (/Interesting ports on ([^ ]*)[: ]/ or /Nmap scan report for ([^ ]*)/) {
my $tmpnode=$1;
if ($currnode) { #if still thinking about last node, flush him out
my $status = join ',',sort keys %states ;
my $appsd="";
foreach my $portnum(keys %portservices) {
my $app_t=$portservices{$portnum};
if ($states{$app_t}) {$appsd .= $app_t . "=up,";}
else {$appsd .= $app_t . "=down,";}
}
$appsd =~ s/,$//;
my $target=$currnode;
if ($hostsents{$target} and $hostsents{$target}->[0]->{ip}) { $target = $hostsents{$target}->[0]->{ip}; }
if ($status or ($installquerypossible and $status = installer_query($target))) { #pingable, but no *clue* as to what the state may be
$ret->{$currnode}->{'status'}="ping";
$ret->{$currnode}->{'appstatus'}=$status;
$ret->{$currnode}->{'appsd'}=$appsd;
$currnode="";
%states=();
} else {
push @nodesetnodes,$currnode; #Aggregate call to nodeset
}
}
$currnode=$tmpnode;
$currnode =~ s/:$//;
$currnode =~ s/\n$//;
my $nip;
if ($nip = xCAT::NetworkUtils->getipaddr($currnode)) { #reverse lookup may not resemble the nodename, key by ip
if ($nodebyip{$nip}) {
$currnode = $nodebyip{$nip};
}
}
$installquerypossible=0; #reset possibility indicator
%rsp=();
unless ($deadnodes{$1}) {
my $shortname;
foreach (keys %deadnodes) {
if (/\./) {
$shortname = $_;
$shortname =~ s/\..*//;
}
if ($currnode =~ /^$_\./ or ($shortname and $shortname eq $currnode)) {
$currnode = $_;
last;
}
}
}
delete $deadnodes{$currnode};
} elsif ($currnode) {
#if (/^MAC/) { #oops not all nmap records end with MAC
if (/^PORT/) { next; }
($port,$state) = split;
if ($port and $port =~ /^(\d*)\// and $state eq 'open') {
if ($1 eq "3001" and defined($chainhash{$currnode}->[0]->{currstate}) and $chainhash{$currnode}->[0]->{currstate} =~ /^install/) {
$installquerypossible=1; #It is possible to actually query node
} elsif ($1 ne "3001") {
$states{$portservices{$1}}=1;
}
}
}
}
}
if ($currnode) {
my $status = join ',',sort keys %states ;
my $appsd="";
foreach my $portnum(keys %portservices) {
my $app_t=$portservices{$portnum};
if ($states{$app_t}) {$appsd .= $app_t . "=up,";}
else {$appsd .= $app_t . "=down,";}
}
$appsd =~ s/,$//;
my $target=$currnode;
if ($hostsents{$target} and $hostsents{$target}->[0]->{ip}) { $target = $hostsents{$target}->[0]->{ip}; }
if ($status or ($installquerypossible and $status = installer_query($target))) { #pingable, but no *clue* as to what the state may be
$ret->{$currnode}->{'status'}="ping";
$ret->{$currnode}->{'appstatus'}=$status;
$ret->{$currnode}->{'appsd'}=$appsd;
$currnode="";
%states=();
} else {
push @nodesetnodes,$currnode; #Aggregate call to nodeset
}
}
if (@nodesetnodes) {
$doreq->({command=>['nodeset'],
node=>\@nodesetnodes,
arg=>['stat']},
\&getstat);
foreach (@nodesetnodes) {
$ret->{$_}->{'status'}=$nodesetstats{$_};
}
}
foreach $currnode (sort keys %deadnodes) {
$ret->{$currnode}->{'status'}="noping";
}
return $ret;
}
sub process_request_port {
my $request = shift;
my $callback = shift;
my $doreq = shift;
my $nodelist=shift;
my $p_tmp=shift;
my %portservices = %$p_tmp;
my @nodes = ();
if ($nodelist) { @nodes=@$nodelist;}
my %unknownnodes;
foreach (@nodes) {
$unknownnodes{$_}=1;
my $packed_ip = undef;
$packed_ip = xCAT::NetworkUtils->getipaddr($_);
if( !defined $packed_ip) {
my %rsp;
$rsp{name}=[$_];
$rsp{data} = [ "Please make sure $_ exists in /etc/hosts" ];
$callback->({node=>[\%rsp]});
}
}
my $status={};
if (@nodes>0) {
my $node;
my $fping;
open($fping,"fping ".join(' ',@nodes). " 2> /dev/null|") or die("Can't start fping: $!");
while (<$fping>) {
my %rsp;
my $node=$_;
$node =~ s/ .*//;
chomp $node;
if (/ is alive/) {
$status->{$node} = interrogate_node($node,$doreq, $p_tmp);
} elsif (/is unreachable/) {
$status->{$node}->{'status'}="noping";
} elsif (/ address not found/) {
$status->{$node}->{'status'}="nosuchhost";
}
}
}
return $status;
}
sub process_request_local_command {
my $request = shift;
my $callback = shift;
my $doreq = shift;
my $nodelist=shift;
my $p_tmp=shift;
my %cmdhash = %$p_tmp;
my @nodes = ();
if ($nodelist) { @nodes=@$nodelist;}
my $status={};
if (@nodes>0) {
foreach my $tmp_cmds (keys %cmdhash) {
my @cmds=split(',', $tmp_cmds);
my @apps=split(',', $cmdhash{$tmp_cmds});
my $index=0;
foreach my $cmd (@cmds) {
my $nodes_string=join(',', @nodes);
my $ret=`$cmd $nodes_string`;
#print "ret=$ret\n";
if (($? ==0) && ($ret)) {
my @ret_array=split('\n', $ret);
foreach(@ret_array) {
my @a=split(':', $_);
chomp($a[1]);
if (exists($status->{$a[0]})) {
$status->{$a[0]} .= "," . $apps[$index] . "=" . $a[1];
} else {
$status->{$a[0]} = $apps[$index] . "=" . $a[1];;
}
}
}
$index++;
}
}
}
return $status;
}
sub process_request_remote_command {
my $request = shift;
my $callback = shift;
my $doreq = shift;
my $nodelist=shift;
my $p_tmp=shift;
my %cmdhash = %$p_tmp;
my @nodes = ();
if ($nodelist) { @nodes=@$nodelist;}
my $status={};
if (@nodes>0) {
foreach my $tmp_cmds (keys %cmdhash) {
my @cmds=split(',', $tmp_cmds);
my @apps=split(',', $cmdhash{$tmp_cmds});
my $index=0;
foreach my $cmd (@cmds) {
my @ret=xCAT::InstUtils->xcmd($callback,$doreq,"xdsh",\@nodes,$cmd,1);
if (@ret) {
foreach(@ret) {
my @a=split(':', $_, 2);
chomp($a[1]); #remove newline
$a[1] =~ s/^\s+//; #remove leading white spaces
$a[1] =~ s/\s+$//; #remove tailing white spaces
if (exists($status->{$a[0]})) {
$status->{$a[0]} .= "," . $apps[$index] . "=" . $a[1];
} else {
$status->{$a[0]} = $apps[$index] . "=" . $a[1];;
}
}
}
$index++;
}
}
}
return $status;
}
sub process_request {
my $request = shift;
my $callback = shift;
my $doreq = shift;
%nodesetstats=();
my $command = $request->{command}->[0];
my $separator="XXXXXYYYYYZZZZZ";
my $usefping;
if (ref $request->{arg}) {
@ARGV=@{$request->{arg}};
GetOptions(
'f' => \$usefping
);
}
if ($command eq "nodestat_internal") {
#if ( -x '/usr/bin/nmap' ) {
# my %portservices = (
# '22' => 'sshd',
# '15002' => 'pbs',
# '8002' => 'xend',
# );
#
# return process_request_nmap($request, $callback, $doreq, $request->{node}, \%portservices);
# }
#handle ports and nodelist.status
my $status={};
if (exists($request->{'portapps'})) {
for (my $i=1; $i<=$request->{'portapps'}->[0]; $i++) {
my %portservices=();
my @apps=split(',', $request->{"portapps$i"}->[0]);
my @ports=split(',', $request->{"portapps$i" . "port"}->[0]);
my $nodes=$request->{"portapps$i" . "node"};
for (my $j=0; $j <@ports; $j++) {
$portservices{$ports[$j]}=$apps[$j];
}
my $ret={};
if ( not $usefping and -x '/usr/bin/nmap' ) {
$ret=process_request_nmap($request, $callback, $doreq, $nodes, \%portservices);
} else {
$ret=process_request_port($request, $callback, $doreq, $nodes, \%portservices);
}
%$status=(%$status, %$ret);
}
}
#handle local commands
if (exists($request->{'cmdapps'})) {
for (my $i=1; $i<=$request->{'cmdapps'}->[0]; $i++) {
my %cmdhash=();
my @apps=split(',', $request->{"cmdapps$i"}->[0]);
my @cmds=split(',', $request->{"cmdapps$i" . "cmd"}->[0]);
my $nodes=$request->{"cmdapps$i" . "node"};
for (my $j=0; $j <@cmds; $j++) {
$cmdhash{$cmds[$j]}=$apps[$j];
}
my $ret = process_request_local_command($request, $callback, $doreq, $nodes, \%cmdhash);
#print Dumper($ret);
foreach my $node1 (keys(%$ret)) {
if (exists($status->{$node1})) {
my $appstatus=$status->{$node1}->{'appstatus'};
if ($appstatus) { $status->{$node1}->{'appstatus'} .= "," . $ret->{$node1}; }
else { $status->{$node1}->{'appstatus'} = $ret->{$node1}; }
my $appsd=$status->{$node1}->{'appsd'};
if ($appsd) { $status->{$node1}->{'appsd'} .= "," . $ret->{$node1}; }
else { $status->{$node1}->{'appsd'} = $ret->{$node1}; }
} else {
$status->{$node1}->{'appstatus'} = $ret->{$node1};
$status->{$node1}->{'appsd'} = $ret->{$node1};
}
}
}
}
#handle local l commands
if (exists($request->{'lcmdapps'})) {
for (my $i=1; $i<=$request->{'lcmdapps'}->[0]; $i++) {
my %cmdhash=();
my @apps=split(',', $request->{"lcmdapps$i"}->[0]);
my @cmds=split(',', $request->{"lcmdapps$i" . "cmd"}->[0]);
my $nodes=$request->{"lcmdapps$i" . "node"};
for (my $j=0; $j <@cmds; $j++) {
$cmdhash{$cmds[$j]}=$apps[$j];
}
my $ret = process_request_local_command($request, $callback, $doreq, $nodes, \%cmdhash);
foreach my $node1 (keys(%$ret)) {
if (exists($status->{$node1})) {
my $appstatus=$status->{$node1}->{'appstatus'};
if ($appstatus) { $status->{$node1}->{'appstatus'} .= "," . $ret->{$node1}; }
else { $status->{$node1}->{'appstatus'} = $ret->{$node1}; }
my $appsd=$status->{$node1}->{'appsd'};
if ($appsd) { $status->{$node1}->{'appsd'} .= "," . $ret->{$node1}; }
else { $status->{$node1}->{'appsd'} = $ret->{$node1}; }
} else {
$status->{$node1}->{'appstatus'} = $ret->{$node1};
$status->{$node1}->{'appsd'} = $ret->{$node1};
}
}
}
}
#handle remote commands
if (exists($request->{'dcmdapps'})) {
for (my $i=1; $i<=$request->{'dcmdapps'}->[0]; $i++) {
my %dcmdhash=();
my @apps=split(',', $request->{"dcmdapps$i"}->[0]);
my @dcmds=split(',', $request->{"dcmdapps$i" . "dcmd"}->[0]);
my $nodes=$request->{"dcmdapps$i" . "node"};
for (my $j=0; $j <@dcmds; $j++) {
$dcmdhash{$dcmds[$j]}=$apps[$j];
}
my $ret = process_request_remote_command($request, $callback, $doreq, $nodes, \%dcmdhash);
foreach my $node1 (keys(%$ret)) {
if (exists($status->{$node1})) {
my $appstatus=$status->{$node1}->{'appstatus'};
if ($appstatus) { $status->{$node1}->{'appstatus'} .= "," . $ret->{$node1}; }
else { $status->{$node1}->{'appstatus'} = $ret->{$node1}; }
my $appsd=$status->{$node1}->{'appsd'};
if ($appsd) { $status->{$node1}->{'appsd'} .= "," . $ret->{$node1}; }
else { $status->{$node1}->{'appsd'} = $ret->{$node1}; }
} else {
$status->{$node1}->{'appstatus'} = $ret->{$node1};
$status->{$node1}->{'appsd'} = $ret->{$node1};
}
}
}
}
#nodestat_internal command the output, nodestat command will collect it
foreach my $node1 (sort keys(%$status)) {
my %rsp;
$rsp{name}=[$node1];
my $st=$status->{$node1}->{'status'};
my $ast= $status->{$node1}->{'appstatus'};
my $appsd = $status->{$node1}->{'appsd'};
$st=$st?$st:'';
$ast=$ast?$ast:'';
$appsd=$appsd?$appsd:'';
$rsp{data}->[0] = "$st$separator$ast$separator$appsd";
$callback->({node=>[\%rsp]});
}
} else { #nodestat command
#first collect the status from the nodes
my $reqcopy = {%$request};
$reqcopy->{command}->[0]='nodestat_internal';
my $ret = xCAT::Utils->runxcmd($reqcopy, $doreq, 0, 1);
#print Dumper($ret);
my $status={};
my @noping_nodes=();
my $power=$request->{'power'}->[0];
foreach my $tmpdata (@$ret) {
if ($tmpdata =~ /([^:]+): (.*)$separator(.*)$separator(.*)/) {
#print "node=$1, status=$2, appstatus=$3, appsd=$4\n";
if ($status->{$1}->{'status'}) {
$status->{$1}->{'status'}=$status->{$1}->{'status'} . ",$2";
} else {
$status->{$1}->{'status'}=$2;
}
if ($status->{$1}->{'appstatus'}) {
$status->{$1}->{'appstatus'}= $status->{$1}->{'appstatus'} . ",$3";
} else {
$status->{$1}->{'appstatus'}=$3;
}
if ($status->{$1}->{'appsd'}) {
$status->{$1}->{'appsd'}=$status->{$1}->{'appsd'} . ",$4";
} else {
$status->{$1}->{'appsd'}=$4;
}
if (($power) && ($2 eq "noping")) {
push(@noping_nodes, $1);
}
} else {
my $rsp;
$rsp->{data}->[0]= "$tmpdata";
xCAT::MsgUtils->message("I", $rsp, $callback);
}
}
#print Dumper($status);
#get power status for noping nodes
if (($power) && (@noping_nodes > 0)) {
#print "noping_nodes=@noping_nodes\n";
my $ret = xCAT::Utils->runxcmd(
{
command => ['rpower'],
node => \@noping_nodes,
arg => [ 'stat' ]
},
$doreq, 0, 1 );
foreach my $tmpdata (@$ret) {
if ($tmpdata =~ /([^:]+): (.*)/) {
$status->{$1}->{'status'}="noping($2)";
} else {
my $rsp;
$rsp->{data}->[0]= "$tmpdata";
xCAT::MsgUtils->message("I", $rsp, $callback);
}
}
}
#print Dumper($request);
my $update=$request->{'update'}->[0];
my $quite=$request->{'quite'}->[0];
#show the output
if (!$quite) {
foreach my $node1 (sort keys(%$status)) {
my %rsp;
$rsp{name}=[$node1];
my $st=$status->{$node1}->{'status'};
my $ast= $status->{$node1}->{'appstatus'};
if ($st) {
if ($st eq 'ping') { $st = $ast ? "$ast" : "$st"; }
else { $st = $ast ? "$st,$ast" : "$st"; }
} else {
$st=$ast;
}
$rsp{data}->[0] = $st;
$callback->({node=>[\%rsp]});
}
}
#update the nodelist table
if ($update) {
my $nodetab=xCAT::Table->new('nodelist', -create=>1);
if ($nodetab) {
my $status1={};
#get current values and compare with the new value to decide if update of db is necessary
my @nodes1=keys(%$status);
my $stuff = $nodetab->getNodesAttribs(\@nodes1, ['node', 'status', 'appstatus']);
#get current local time
my (
$sec, $min, $hour, $mday, $mon,
$year, $wday, $yday, $isdst
)
= localtime(time);
my $currtime = sprintf("%02d-%02d-%04d %02d:%02d:%02d",
$mon + 1, $mday, $year + 1900,
$hour, $min, $sec);
foreach my $node1 (@nodes1) {
my $oldstatus=$stuff->{$node1}->[0]->{status};
my $newstatus=$status->{$node1}->{status};
if ($newstatus) {
if ((!$oldstatus) || ($newstatus ne $oldstatus)) {
$status1->{$node1}->{status}= $newstatus;
$status1->{$node1}->{statustime}= $currtime;
}
}
else {
if ($oldstatus) {
$status1->{$node1}->{status}= "";
$status1->{$node1}->{statustime}= "";
}
}
my $oldappstatus=$stuff->{$node1}->[0]->{'appstatus'};
my $newappstatus=$status->{$node1}->{'appsd'};
while ($newappstatus =~ /(\w+)\=(\w+)/) {
my $tmp1=$1;
my $tmp2=$2;
if ($oldappstatus) {
if($oldappstatus =~ /$tmp1\=/){
$oldappstatus =~ s/$tmp1\=\w+/$tmp1\=$tmp2/g;
}else{
$oldappstatus = $oldappstatus."\,$tmp1\=$tmp2";
}
} else {
$oldappstatus = "$tmp1\=$tmp2";
}
$newappstatus =~ s/(\w+)\=(\w+)//;
}
$status1->{$node1}->{appstatus}= $oldappstatus;
$status1->{$node1}->{appstatustime}= $currtime;
}
#print Dumper($status1);
$nodetab->setNodesAttribs($status1);
}
}
}
}
sub usage
{
my $cb=shift;
my $rsp={};
$rsp->{data}->[0]= "Usage:";
$rsp->{data}->[1]= " nodestat [noderange] [-m|--usemon] [-p|powerstat] [-u|--updatedb]";
$rsp->{data}->[2]= " nodestat [-h|--help|-v|--version]";
xCAT::MsgUtils->message("I", $rsp, $cb);
}
#--------------------------------------------------------------------------------
=head3 getStatusMonsettings
This function goes to the monsetting table to retrieve the settings related to
the node status and app status monitoring.
Arguments:
none.
Returns:
a hash that has settings from the monsetting table for node status and
app status monitoring. For example:
( 'APPS'=>[ll,gpfs],
'll' =>
{
'group' => 'service,compute',
'port' => '5001'
},
'gpfs' =>
{
'group' => 'service',
'cmd' => '/tmp/gpfscmd'
};
)
=cut
#--------------------------------------------------------------------------------
sub getStatusMonsettings {
my %apps=();
my $tab=xCAT::Table->new('monsetting');
if ( defined($tab)) {
my ($ent) = $tab->getAttribs({name => 'xcatmon', key => 'apps' }, 'value');
if ( defined($ent) ) {
my $tmp_list=$ent->{value};
if ($tmp_list) {
my @applist=split(',', $tmp_list);
foreach my $app (@applist) {
$apps{$app}={};
}
$apps{'APPS'}=\@applist;
my @results = $tab->getAttribs({name => 'xcatmon'}, 'key','value');
if (@results) {
foreach(@results) {
my $key=$_->{key};
my $value=$_->{value};
if (exists($apps{$key})) {
my @tem_value=split(',',$value);
foreach my $pair (@tem_value) {
my @tmp_action=split('=', $pair);
if (exists($apps{$key}->{$tmp_action[0]})) {
$apps{$key}->{$tmp_action[0]} = $apps{$key}->{$tmp_action[0]} . "," . $tmp_action[1];
} else {
$apps{$key}->{$tmp_action[0]}=$tmp_action[1];
}
}
}
}
}
}
}
}
return %apps;
}
#--------------------------------------------------------------------------------
=head3 getNodeStatusAndAppstatus
This function goes to the xCAT nodelist table to retrieve the saved node status and appstatus
for all the node that are managed by local nodes.
Arguments:
nodelist--- an array of nodes
Returns:
a hash pointer that has the node status and appstatus. The format is:
{ node1=> {
status=>'active',appstatus=>'sshd=up,ll=up,gpfs=down'
} ,
node2=> {
status=>'active',appstatus=>'sshd=up,ll=down,gpfs=down'
}
}
=cut
#--------------------------------------------------------------------------------
sub getMonNodesStatusAndAppStatus {
my @nodes=@_;
my %status=();
my $table=xCAT::Table->new("nodelist", -create =>1);
my $tabdata=$table->getNodesAttribs(\@nodes,['node', 'status', 'appstatus']);
foreach my $node (@nodes) {
my $tmp1=$tabdata->{$node}->[0];
if ($tmp1) {
$status{$node}->{status}=$tmp1->{status};
$status{$node}->{appstatus}=$tmp1->{appstatus};
}
}
return %status;
}
1;