arif/xcat-core
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
xcat-core/xCAT-server/lib/xcat/plugins/nodestat.pm
jjohnson2 fb02c18853 Fix erroneous 'noping' in various circumstances 
nmap's raw IP facility is prone to mistakes in various
environments.  Disable the raw IP handling by
telling nmap to run as if it didn't have privilege
2015-03-20 10:58:26 -04:00

1295 lines
39 KiB
Perl
Raw Blame History

package xCAT_plugin::nodestat;
BEGIN
{
$::XCATROOT = $ENV{'XCATROOT'} ? $ENV{'XCATROOT'} : '/opt/xcat';
}
use lib "$::XCATROOT/lib/perl";
use strict;
use warnings;
use Socket;
my $inet6support = eval {
require Socket6;
require IO::Socket::INET6;
1;
};
use IO::Handle;
use Getopt::Long;
use Data::Dumper;
use xCAT::GlobalDef;
use xCAT::NetworkUtils;
require xCAT::Utils;
require xCAT::TableUtils;
require xCAT::ServiceNodeUtils;
my %nodesetstats;
my %chainhash;
my %default_ports = (
'ftp' => '21',
'ssh' => '22',
'sshd' => '22',
'pbs' => '15002',
'pbs_mom' => '15002',
'xend' => '8002',
'll' => '9616',
'loadl' => '9616',
'loadl_master' => '9616',
'loadleveler' => '9616',
'gpfs' => '1191',
'rdp' => '3389',
'msrpc' => '135',
);
sub handled_commands {
return {
nodestat => 'nodestat',
nodestat_internal => 'nodestat',
};
}
sub pinghost {
my $node = shift;
my $rc = system("ping -q -n -c 1 -w 1 $node > /dev/null");
if ($rc == 0) {
return 1;
} else {
return 0;
}
}
sub nodesockopen {
my $node = shift;
my $port = shift;
my $socket;
my $addr = gethostbyname($node);
my $sin = sockaddr_in($port,$addr);
my $proto = getprotobyname('tcp');
socket($socket,PF_INET,SOCK_STREAM,$proto) || return 0;
connect($socket,$sin) || return 0;
return 1;
}
sub installer_query {
my $node = shift;
my $destport = 3001;
my $socket;
my $text = "";
if ($inet6support) {
$socket = IO::Socket::INET6->new(PeerAddr => $node, PeerPort => 3001);
unless ($socket) { return 0; }
} else {
my $proto = getprotobyname('tcp');
socket($socket,PF_INET,SOCK_STREAM,$proto) || return 0;
my $addr = gethostbyname($node);
my $sin = sockaddr_in($destport,$addr);
connect($socket,$sin) || return 0;
}
print $socket "stat \n";
$socket->flush;
while (<$socket>) {
$text.=$_;
}
$text =~ s/\n.*//;
return $text;
close($socket);
}
sub getstat {
my $response = shift;
foreach (@{$response->{node}}) {
$nodesetstats{$_->{name}->[0]} = $_->{data}->[0];
}
}
#-------------------------------------------------------
=head3 preprocess_request
Check and setup for hierarchy
=cut
#-------------------------------------------------------
sub preprocess_request
{
my $req = shift;
my $cb = shift;
my %sn;
if (defined $req->{_xcatpreprocessed}->[0] && $req->{_xcatpreprocessed}->[0] == 1) { return [$req]; }
#exit if preprocessed
my $command = $req->{command}->[0];
if ($command eq "nodestat") {
@ARGV=();
my $args=$req->{arg};
if ($args) {
@ARGV = @{$args};
}
# parse the options
$::UPDATE=0;
$::QUITE=0;
$::MON=0;
$::POWER=0;
#Getopt::Long::Configure("posix_default");
#Getopt::Long::Configure("no_gnu_compat");
Getopt::Long::Configure("bundling");
$Getopt::Long::ignorecase=0;
if (!GetOptions(
'm|usemon' => \$::MON,
'q|quite' => \$::QUITE, #this is a internal flag used by monitoring
'u|updatedb' => \$::UPDATE,
'p|powerstat' => \$::POWER,
'h|help' => \$::HELP,
'v|version' => \$::VERSION))
{
&usage($cb);
return(1);
}
if ($::HELP) {
&usage($cb);
return(0);
}
if ($::VERSION) {
my $version = xCAT::Utils->Version();
my $rsp={};
$rsp->{data}->[0] = "$version";
xCAT::MsgUtils->message("I", $rsp, $cb);
return(0);
}
my $nodes = $req->{node};
if (!$nodes)
{
&usage($cb);
return (1);
}
$req->{'update'}->[0]=$::UPDATE;
$req->{'quite'}->[0]=$::QUITE;
$req->{'mon'}->[0]=$::MON;
$req->{'power'}->[0]=$::POWER;
return [$req];
}
#the following is for nodestat_internal command
my $nodes = $req->{node};
my $service = "xcat";
my @requests;
if ($nodes) {
my $usenmapfrommn=0;
if (-x '/usr/bin/nmap' or -x '/usr/local/bin/nmap') {
#my $sitetab = xCAT::Table->new('site');
#if ($sitetab) {
#(my $ref) = $sitetab->getAttribs({key => 'useNmapfromMN'}, 'value');
my @entries = xCAT::TableUtils->get_site_attribute("useNmapfromMN");
my $t_entry = $entries[0];
if (defined($t_entry)) {
if ($t_entry =~ /1|yes|YES|Y|y/) { $usenmapfrommn=1; }
}
#}
}
#get monsettings
my %apps = ();
my $mon=$req->{'mon'}->[0];
if ($mon == 1) { %apps=getStatusMonsettings(); }
#if no apps specified in the monsetting table, add sshd, pbs and xend
if (keys(%apps) == 0) {
$apps{'sshd'}->{'group'} = "ALL"; #ALL means anything on the nodelist table, it is different from all
$apps{'sshd'}->{'port'} = "22";
$apps{'https'}->{'group'} = "ALL"; #ALL means anything on the nodelist table, it is different from all
$apps{'https'}->{'port'} = "443";
$apps{'pbs'}->{'group'} = "ALL";
$apps{'pbs'}->{'port'} = "15002";
$apps{'xend'}->{'group'} = "ALL";
$apps{'xend'}->{'port'} = "8002";
$apps{'rdp'}->{'group'} = "ALL";
$apps{'rdp'}->{'port'} = "3389";
$apps{'msrpc'}->{'group'} = "ALL";
$apps{'msrpc'}->{'port'} = "135";
$apps{'APPS'}=['sshd', 'https', 'pbs', 'xend'];
} else {
#go thorugh the settings and put defaults in
foreach my $app (keys(%apps)) {
if ($app eq 'APPS') { next; }
if (!exists($apps{$app}->{'group'})) { $apps{$app}->{'group'} = "ALL"; }
if (exists($apps{$app}->{'cmd'}) || exists($apps{$app}->{'dcmd'}) || exists($apps{$app}->{'lcmd'})) { next; }
if (exists($apps{$app}->{'port'})) { next; }
#add port number in if nothing is specified
if (exists($default_ports{$app})) { $apps{$app}->{'port'} = $default_ports{$app}; }
else {
my $p=`grep "^$app" /etc/services`;
if ($? == 0) {
my @a_list=sort(split('\n', $p));
my @a_temp=split('/',$a_list[0]);
my @a=split(' ', $a_temp[0]);
$apps{$app}->{'port'}=$a[1];
} else {
my $rsp={};
$rsp->{data}->[0]= "Cannot find port number for application $app. Please either specify a port number or a command in monsetting table for $app.";;
xCAT::MsgUtils->message("I", $rsp, $cb);
return (0);
}
}
}
#always add sshd
if (!exists($apps{'ssh'}) || !exists($apps{'sshd'}) ) {
$apps{'sshd'}->{'group'} = "ALL";
$apps{'sshd'}->{'port'} = "22";
my $pa=$apps{'APPS'};
push @$pa, 'sshd';
}
}
#print Dumper(%apps);
# find service nodes for requested nodes
# build an individual request for each service node
my $sn = xCAT::ServiceNodeUtils->get_ServiceNode($nodes, $service, "MN");
#get the member for each group
my %groups=();
foreach my $app (keys %apps) {
if ($app eq 'APPS') { next; }
my $group=$apps{$app}->{'group'};
if (($group) && ($group ne "ALL") && (! exists($groups{$group}))) {
my @tmp_nodes = xCAT::NodeRange::noderange($group);
foreach (@tmp_nodes) {
$groups{$group}->{$_}=1;
}
}
}
# build each request for each service node
my $all_apps=$apps{'APPS'};
my %all_porthash=(); #stores all the port apps if usenmapfrommn=1
my %lcmdhash=(); #('myapp2,myapp3'=> {
# lcmd=>'/tmp/mycmd1,/usr/bin/date',
# node=>[node1,node2]
# }
#)
foreach my $snkey (keys %$sn)
{
my $reqcopy = {%$req};
$reqcopy->{node} = $sn->{$snkey};
$reqcopy->{'_xcatdest'} = $snkey;
$reqcopy->{_xcatpreprocessed}->[0] = 1;
$reqcopy->{'useNmapfromMN'}->[0]=$usenmapfrommn;
$reqcopy->{'allapps'}=$all_apps;
my %porthash=(); #('sshd,ll'=> {
# port=>'22,5001',
# node=>[node1,node2]
# }
#)
my %cmdhash=(); #('gpfs,myapp'=> {
# cmd=>'/tmp/mycmd1,/usr/bin/date',
# node=>[node1,node2]
# }
#)
my %dcmdhash=(); #('myapp2,myapp3'=> {
# dcmd=>'/tmp/mycmd1,/usr/bin/date',
# node=>[node1,node2]
# }
#)
my @nodes_for_sn=@{$sn->{$snkey}};
foreach my $node (@nodes_for_sn) {
my @ports;
my @portapps;
my @cmds;
my @cmdapps;
my @dcmds;
my @dcmdapps;
my @lcmdapps;
my @lcmds;
foreach my $app (keys %apps) {
if ($app eq 'APPS') { next; }
my $group=$apps{$app}->{'group'};
if (($group eq "ALL") || ($groups{$group}->{$node})) {
#print "app=$app\n";
if (exists($apps{$app}->{'port'})) {
push @ports, $apps{$app}->{'port'};
push @portapps, $app;
}
elsif (exists($apps{$app}->{'cmd'})) {
push @cmds, $apps{$app}->{'cmd'};
push @cmdapps, $app;
}
elsif (exists($apps{$app}->{'dcmd'})) {
push @dcmds, $apps{$app}->{'dcmd'};
push @dcmdapps, $app;
}
elsif (exists($apps{$app}->{'lcmd'})) {
push @lcmds, $apps{$app}->{'lcmd'};
push @lcmdapps, $app;
}
}
}
#print "ports=@ports\n";
#print "portapps=@portapps\n";
#print "cmds=@cmds\n";
#print "cmdapps=@cmdapps\n";
#print "dcmds=@dcmds\n";
#print "dcmdapps=@dcmdapps\n";
if (@portapps>0) {
my $tmpapps=join(',', @portapps);
if (($usenmapfrommn==1) && (@cmdapps==0) && (@dcmdapps==0) && (@lcmdapps==0)) {
#this is the case where mn handles ports for all nodes using nmap
#The current limitation is that when there are cmd or dcmd specified for the node
# nmap has to be done on the service node because if both mn and sn update the appstatus
# one will overwites the other.
if (exists($all_porthash{$tmpapps})) {
my $pa=$all_porthash{$tmpapps}->{'node'};
push @$pa, $node;
} else {
$all_porthash{$tmpapps}->{'node'}=[$node];
$all_porthash{$tmpapps}->{'port'}=join(',', @ports);
}
} else {
if (exists($porthash{$tmpapps})) {
my $pa=$porthash{$tmpapps}->{'node'};
push @$pa, $node;
} else {
$porthash{$tmpapps}->{'node'}=[$node];
$porthash{$tmpapps}->{'port'}=join(',', @ports);
}
}
}
if (@cmdapps>0) {
my $tmpapps=join(',', @cmdapps);
if (exists($cmdhash{$tmpapps})) {
my $pa=$cmdhash{$tmpapps}->{'node'};
push @$pa, $node;
} else {
$cmdhash{$tmpapps}->{'node'}=[$node];
$cmdhash{$tmpapps}->{'cmd'}=join(',', @cmds);
}
}
if (@dcmdapps>0) {
my $tmpapps=join(',', @dcmdapps);
if (exists($dcmdhash{$tmpapps})) {
my $pa=$dcmdhash{$tmpapps}->{'node'};
push @$pa, $node;
} else {
$dcmdhash{$tmpapps}->{'node'}=[$node];
$dcmdhash{$tmpapps}->{'dcmd'}=join(',', @dcmds);
}
}
if (@lcmdapps>0) {
my $i=0;
foreach my $lapp (@lcmdapps) {
if (exists($lcmdhash{$lapp})) {
my $pa=$lcmdhash{$lapp}->{'node'};
push @$pa, $node;
} else {
$lcmdhash{$lapp}->{'node'}=[$node];
$lcmdhash{$lapp}->{'lcmd'}=$lcmds[$i];
}
$i++;
}
}
} #end foreach (@nodes_for_sn)
#print Dumper(%porthash);
#print "cmdhash=" . Dumper(%cmdhash);
#now push the settings into the requests
my $i=1;
if ((keys(%porthash) == 0) && (keys(%cmdhash) == 0) && (keys(%dcmdhash) == 0) && (keys(%lcmdhash) == 0)) { next; }
foreach my $tmpapps (keys %porthash) {
$reqcopy->{'portapps'}->[0]= scalar keys %porthash;
$reqcopy->{"portapps$i"}->[0]= $tmpapps;
$reqcopy->{"portapps$i" . "port"}->[0]= $porthash{$tmpapps}->{'port'};
$reqcopy->{"portapps$i" . "node"} = $porthash{$tmpapps}->{'node'};;
$i++;
}
$i=1;
foreach my $tmpapps (keys %cmdhash) {
$reqcopy->{'cmdapps'}->[0]= scalar keys %cmdhash;
$reqcopy->{"cmdapps$i"}->[0]= $tmpapps;
$reqcopy->{"cmdapps$i" . "cmd"}->[0]= $cmdhash{$tmpapps}->{'cmd'};
$reqcopy->{"cmdapps$i" . "node"} = $cmdhash{$tmpapps}->{'node'};;
$i++;
}
$i=1;
foreach my $tmpapps (keys %dcmdhash) {
$reqcopy->{'dcmdapps'}->[0]= scalar keys %dcmdhash;
$reqcopy->{"dcmdapps$i"}->[0]= $tmpapps;
$reqcopy->{"dcmdapps$i" . "dcmd"}->[0]= $dcmdhash{$tmpapps}->{'dcmd'};
$reqcopy->{"dcmdapps$i" . "node"} = $dcmdhash{$tmpapps}->{'node'};
$i++;
}
#done
push @requests, $reqcopy;
} #enf sn_key
#print "apps=" . Dumper(%apps);
#mn handles all nmap when useNmapfromMN=1 on the site table
if (($usenmapfrommn == 1) && (keys(%all_porthash) > 0)) {
my @hostinfo=xCAT::NetworkUtils->determinehostname();
my %iphash=();
foreach(@hostinfo) {$iphash{$_}=1;}
my $handled=0;
foreach my $req (@requests) {
my $currsn=$req->{'_xcatdest'};
if (exists($iphash{$currsn})) {
my $i=1;
foreach my $tmpapps (keys %all_porthash) {
$req->{'portapps'}->[0]= scalar keys %all_porthash;
$req->{"portapps$i"}->[0]= $tmpapps;
$req->{"portapps$i" . "port"}->[0]= $all_porthash{$tmpapps}->{'port'};
$req->{"portapps$i" . "node"} = $all_porthash{$tmpapps}->{'node'};;
$i++;
}
$handled=1;
last;
}
}
if (!$handled) {
my $reqcopy = {%$req};
$reqcopy->{_xcatpreprocessed}->[0] = 1;
$reqcopy->{'useNmapfromMN'}->[0]=$usenmapfrommn;
$reqcopy->{'allapps'}=$all_apps;
my $i=1;
foreach my $tmpapps (keys %all_porthash) {
$reqcopy->{'portapps'}->[0]= scalar keys %all_porthash;
$reqcopy->{"portapps$i"}->[0]= $tmpapps;
$reqcopy->{"portapps$i" . "port"}->[0]= $all_porthash{$tmpapps}->{'port'};
$reqcopy->{"portapps$i" . "node"} = $all_porthash{$tmpapps}->{'node'};;
$i++;
}
push @requests, $reqcopy;
}
}
#if ($usenmapfrommn) {
# my $reqcopy = {%$req};
# $reqcopy->{'update'}->[0]=$::UPDATE;
# $reqcopy->{'useNmapfromMN'}->[0]=1;
# if (!$::UPDATE) {
# push @requests, $reqcopy;
# return \@requests; #do not distribute, nodestat seems to lose accuracy and slow down distributed, if using nmap
# }
#}
#now handle local commands
#print "lcmdhash=" . Dumper(%lcmdhash);
if (keys(%lcmdhash) > 0) {
my @hostinfo=xCAT::NetworkUtils->determinehostname();
my %iphash=();
foreach(@hostinfo) {$iphash{$_}=1;}
my $handled=0;
foreach my $req (@requests) {
my $currsn=$req->{'_xcatdest'};
if (exists($iphash{$currsn})) {
my $i=1;
foreach my $lapp (keys %lcmdhash) {
$req->{'lcmdapps'}->[0]= scalar keys %lcmdhash;
$req->{"lcmdapps$i"}->[0]= $lapp;
$req->{"lcmdapps$i" . "cmd"}->[0]= $lcmdhash{$lapp}->{'lcmd'};
$req->{"lcmdapps$i" . "node"} = $lcmdhash{$lapp}->{'node'};;
$i++;
}
$handled=1;
last;
}
}
if (!$handled) {
my $reqcopy = {%$req};
$reqcopy->{_xcatpreprocessed}->[0] = 1;
$reqcopy->{'allapps'}=$all_apps;
my $i=1;
foreach my $lapp (keys %lcmdhash) {
$reqcopy->{'lcmdapps'}->[0]= scalar keys %lcmdhash;
$reqcopy->{"lcmdapps$i"}->[0]= $lapp;
$reqcopy->{"lcmdapps$i" . "cmd"}->[0]= $lcmdhash{$lapp}->{'lcmd'};
$reqcopy->{"lcmdapps$i" . "node"} = $lcmdhash{$lapp}->{'node'};;
$i++;
}
push @requests, $reqcopy;
}
}
}
return \@requests;
}
sub interrogate_node { #Meant to run against confirmed up nodes
my $node=shift;
my $doreq=shift;
my $p_tmp=shift;
my %portservices = %$p_tmp;
my $status = "";
my $appsd=""; #detailed status
my $ret={};
$ret->{'status'}="ping";
foreach my $port (keys(%portservices)) {
if (nodesockopen($node,$port)) {
$status.=$portservices{$port} . ",";
$appsd.=$portservices{$port} . "=up,";
} else {
$appsd.=$portservices{$port} . "=down,";
}
}
$status =~ s/,$//;
$appsd =~ s/,$//;
$ret->{'appsd'}=$appsd;
if ($status) {
$ret->{'appstatus'}=$status;
return $ret;
}
if ($status = installer_query($node)) {
$ret->{'status'}=$status;
return $ret;
} else { #pingable, but no *clue* as to what the state may be
$doreq->({command=>['nodeset'],
node=>[$node],
arg=>['stat']},
\&getstat);
$ret->{'status'} = 'ping '.$nodesetstats{$node};
return $ret;
}
}
sub process_request_nmap {
my $request = shift;
my $callback = shift;
my $doreq = shift;
my $nodelist=shift;
my $p_tmp=shift;
my %portservices = %$p_tmp;
my @nodes =();
if ($nodelist) { @nodes=@$nodelist;}
my %nodebyip;
my @livenodes;
my %unknownnodes;
my $chaintab = xCAT::Table->new('chain',-create=>0);
if ($chaintab) {
%chainhash = %{$chaintab->getNodesAttribs(\@nodes,['currstate'])};
}
my $hoststab = xCAT::Table->new('hosts',-create=>0);
my %hostsents;
if ($hoststab) { %hostsents = %{$hoststab->getNodesAttribs(\@nodes,['ip'])}; }
my @ips;
my @ip6s;
foreach (@nodes) {
$unknownnodes{$_}=1;
my $ip = undef;
if (($hostsents{$_}) && ($hostsents{$_}->[0]->{ip})) {
$ip = $hostsents{$_}->[0]->{ip};
$nodebyip{$ip} = $_;
$ip = xCAT::NetworkUtils->getipaddr($ip);
$nodebyip{$ip} = $_;
} else {
$ip = xCAT::NetworkUtils->getipaddr($_);
}
if( !defined $ip) {
my %rsp;
$rsp{name}=[$_];
$rsp{data} = [ "Please make sure $_ exists in /etc/hosts or DNS or hosts table" ];
$callback->({node=>[\%rsp]});
} else {
if ($ip =~ /:/) {
push @ip6s,$ip;
} else {
push @ips,$ip;
}
$nodebyip{$ip} = $_;
}
}
my $ret={};
my $node;
my $fping;
my $ports = join ',',keys %portservices;
my %deadnodes;
foreach (@nodes) {
$deadnodes{$_}=1;
}
#print "nmap -PE --send-ip -p $ports,3001 ".join(' ',@nodes) . "\n";
# open($fping,"nmap -PE --send-ip -p $ports,3001 ".join(' ',@nodes). " 2> /dev/null|") or die("Can't start nmap: $!");
my $currnode='';
my $port;
my $state;
my %states;
my %rsp;
my $installquerypossible=0;
my @nodesetnodes=();
# get additional options from site table
my @nmap_options = xCAT::TableUtils->get_site_attribute("nmapoptions");
my $more_options = $nmap_options[0];
foreach my $ip6 (0,1) { #first pass, ipv4, second pass ipv6
if ($ip6 and scalar(@ip6s)) {
open($fping,"nmap --unprivileged -6 -PS$ports,3001 -n --send-ip -p $ports,3001 $more_options ".join(' ',@ip6s). " 2> /dev/null|") or die("Can't start nmap: $!");
} elsif (not $ip6 and scalar(@ips)) {
open($fping,"nmap --unprivileged -PE -n --send-ip -p $ports,3001 $more_options ".join(' ',@ips). " 2> /dev/null|") or die("Can't start nmap: $!");
} else { next; }
while (<$fping>) {
if (/Interesting ports on ([^ ]*)[: ]/ or /Nmap scan report for ([^ ]*)/) {
my $tmpnode=$1;
if ($currnode) { #if still thinking about last node, flush him out
my $status = join ',',sort keys %states ;
my $appsd="";
foreach my $portnum(keys %portservices) {
my $app_t=$portservices{$portnum};
if ($states{$app_t}) {$appsd .= $app_t . "=up,";}
else {$appsd .= $app_t . "=down,";}
}
$appsd =~ s/,$//;
my $target=$currnode;
if ($hostsents{$target} and $hostsents{$target}->[0]->{ip}) { $target = $hostsents{$target}->[0]->{ip}; }
if ($status or ($installquerypossible and $status = installer_query($target))) { #pingable, but no *clue* as to what the state may be
$ret->{$currnode}->{'status'}="ping";
$ret->{$currnode}->{'appstatus'}=$status;
$ret->{$currnode}->{'appsd'}=$appsd;
$currnode="";
%states=();
} else {
push @nodesetnodes,$currnode; #Aggregate call to nodeset
}
}
$currnode=$tmpnode;
$currnode =~ s/:$//;
$currnode =~ s/\n$//;
my $nip;
if ($nip = xCAT::NetworkUtils->getipaddr($currnode)) { #reverse lookup may not resemble the nodename, key by ip
if ($nodebyip{$nip}) {
$currnode = $nodebyip{$nip};
}
}
$installquerypossible=0; #reset possibility indicator
%rsp=();
unless ($deadnodes{$currnode}) {
my $shortname;
foreach (keys %deadnodes) {
if (/\./) {
$shortname = $_;
$shortname =~ s/\..*//;
}
if ($currnode =~ /^$_\./ or ($shortname and $shortname eq $currnode)) {
$currnode = $_;
last;
}
}
}
delete $deadnodes{$currnode};
} elsif ($currnode) {
#if (/^MAC/) { #oops not all nmap records end with MAC
if (/^PORT/) { next; }
($port,$state) = split;
if ($port and $port =~ /^(\d*)\// and $state eq 'open') {
if ($1 eq "3001" and defined($chainhash{$currnode}->[0]->{currstate}) and $chainhash{$currnode}->[0]->{currstate} =~ /^install/) {
$installquerypossible=1; #It is possible to actually query node
} elsif ($1 ne "3001") {
$states{$portservices{$1}}=1;
}
}
}
}
}
if ($currnode) {
my $status = join ',',sort keys %states ;
my $appsd="";
foreach my $portnum(keys %portservices) {
my $app_t=$portservices{$portnum};
if ($states{$app_t}) {$appsd .= $app_t . "=up,";}
else {$appsd .= $app_t . "=down,";}
}
$appsd =~ s/,$//;
my $target=$currnode;
if ($hostsents{$target} and $hostsents{$target}->[0]->{ip}) { $target = $hostsents{$target}->[0]->{ip}; }
if ($status or ($installquerypossible and $status = installer_query($target))) { #pingable, but no *clue* as to what the state may be
$ret->{$currnode}->{'status'}="ping";
$ret->{$currnode}->{'appstatus'}=$status;
$ret->{$currnode}->{'appsd'}=$appsd;
$currnode="";
%states=();
} else {
push @nodesetnodes,$currnode; #Aggregate call to nodeset
}
}
if (@nodesetnodes) {
$doreq->({command=>['nodeset'],
node=>\@nodesetnodes,
arg=>['stat']},
\&getstat);
foreach (@nodesetnodes) {
$ret->{$_}->{'status'}=$nodesetstats{$_};
}
}
foreach $currnode (sort keys %deadnodes) {
$ret->{$currnode}->{'status'}="noping";
}
return $ret;
}
sub process_request_port {
my $request = shift;
my $callback = shift;
my $doreq = shift;
my $nodelist=shift;
my $p_tmp=shift;
my %portservices = %$p_tmp;
my @nodes = ();
if ($nodelist) { @nodes=@$nodelist;}
my %unknownnodes;
foreach (@nodes) {
$unknownnodes{$_}=1;
my $packed_ip = undef;
$packed_ip = xCAT::NetworkUtils->getipaddr($_);
if( !defined $packed_ip) {
my %rsp;
$rsp{name}=[$_];
$rsp{data} = [ "Please make sure $_ exists in /etc/hosts" ];
$callback->({node=>[\%rsp]});
}
}
my $status={};
if (@nodes>0) {
my $node;
my $fping;
open($fping,"fping ".join(' ',@nodes). " 2> /dev/null|") or die("Can't start fping: $!");
while (<$fping>) {
my %rsp;
my $node=$_;
$node =~ s/ .*//;
chomp $node;
if (/ is alive/) {
$status->{$node} = interrogate_node($node,$doreq, $p_tmp);
} elsif (/is unreachable/) {
$status->{$node}->{'status'}="noping";
} elsif (/ address not found/) {
$status->{$node}->{'status'}="nosuchhost";
}
}
}
return $status;
}
sub process_request_local_command {
my $request = shift;
my $callback = shift;
my $doreq = shift;
my $nodelist=shift;
my $p_tmp=shift;
my %cmdhash = %$p_tmp;
my @nodes = ();
if ($nodelist) { @nodes=@$nodelist;}
my $status={};
if (@nodes>0) {
foreach my $tmp_cmds (keys %cmdhash) {
my @cmds=split(',', $tmp_cmds);
my @apps=split(',', $cmdhash{$tmp_cmds});
my $index=0;
foreach my $cmd (@cmds) {
my $nodes_string=join(',', @nodes);
my $ret=`$cmd $nodes_string`;
#print "ret=$ret\n";
if (($? ==0) && ($ret)) {
my @ret_array=split('\n', $ret);
foreach(@ret_array) {
my @a=split(':', $_);
chomp($a[1]);
if (exists($status->{$a[0]})) {
$status->{$a[0]} .= "," . $apps[$index] . "=" . $a[1];
} else {
$status->{$a[0]} = $apps[$index] . "=" . $a[1];;
}
}
}
$index++;
}
}
}
return $status;
}
sub process_request_remote_command {
my $request = shift;
my $callback = shift;
my $doreq = shift;
my $nodelist=shift;
my $p_tmp=shift;
my %cmdhash = %$p_tmp;
my @nodes = ();
if ($nodelist) { @nodes=@$nodelist;}
my $status={};
if (@nodes>0) {
foreach my $tmp_cmds (keys %cmdhash) {
my @cmds=split(',', $tmp_cmds);
my @apps=split(',', $cmdhash{$tmp_cmds});
my $index=0;
foreach my $cmd (@cmds) {
my @ret=xCAT::InstUtils->xcmd($callback,$doreq,"xdsh",\@nodes,$cmd,1);
if (@ret) {
foreach(@ret) {
my @a=split(':', $_, 2);
chomp($a[1]); #remove newline
$a[1] =~ s/^\s+//; #remove leading white spaces
$a[1] =~ s/\s+$//; #remove tailing white spaces
if (exists($status->{$a[0]})) {
$status->{$a[0]} .= "," . $apps[$index] . "=" . $a[1];
} else {
$status->{$a[0]} = $apps[$index] . "=" . $a[1];;
}
}
}
$index++;
}
}
}
return $status;
}
sub process_request {
my $request = shift;
my $callback = shift;
my $doreq = shift;
%nodesetstats=();
my $command = $request->{command}->[0];
my $separator="XXXXXYYYYYZZZZZ";
my $usefping;
if (ref $request->{arg}) {
@ARGV=@{$request->{arg}};
GetOptions(
'f' => \$usefping
);
}
if ($command eq "nodestat_internal") {
#if ( -x '/usr/bin/nmap' ) {
# my %portservices = (
# '22' => 'sshd',
# '15002' => 'pbs',
# '8002' => 'xend',
# );
#
# return process_request_nmap($request, $callback, $doreq, $request->{node}, \%portservices);
# }
#handle ports and nodelist.status
my $status={};
if (exists($request->{'portapps'})) {
for (my $i=1; $i<=$request->{'portapps'}->[0]; $i++) {
my %portservices=();
my @apps=split(',', $request->{"portapps$i"}->[0]);
my @ports=split(',', $request->{"portapps$i" . "port"}->[0]);
my $nodes=$request->{"portapps$i" . "node"};
for (my $j=0; $j <@ports; $j++) {
$portservices{$ports[$j]}=$apps[$j];
}
my $ret={};
if ( not $usefping and -x '/usr/bin/nmap' ) {
$ret=process_request_nmap($request, $callback, $doreq, $nodes, \%portservices);
} else {
$ret=process_request_port($request, $callback, $doreq, $nodes, \%portservices);
}
%$status=(%$status, %$ret);
}
}
#handle local commands
if (exists($request->{'cmdapps'})) {
for (my $i=1; $i<=$request->{'cmdapps'}->[0]; $i++) {
my %cmdhash=();
my @apps=split(',', $request->{"cmdapps$i"}->[0]);
my @cmds=split(',', $request->{"cmdapps$i" . "cmd"}->[0]);
my $nodes=$request->{"cmdapps$i" . "node"};
for (my $j=0; $j <@cmds; $j++) {
$cmdhash{$cmds[$j]}=$apps[$j];
}
my $ret = process_request_local_command($request, $callback, $doreq, $nodes, \%cmdhash);
#print Dumper($ret);
foreach my $node1 (keys(%$ret)) {
if (exists($status->{$node1})) {
my $appstatus=$status->{$node1}->{'appstatus'};
if ($appstatus) { $status->{$node1}->{'appstatus'} .= "," . $ret->{$node1}; }
else { $status->{$node1}->{'appstatus'} = $ret->{$node1}; }
my $appsd=$status->{$node1}->{'appsd'};
if ($appsd) { $status->{$node1}->{'appsd'} .= "," . $ret->{$node1}; }
else { $status->{$node1}->{'appsd'} = $ret->{$node1}; }
} else {
$status->{$node1}->{'appstatus'} = $ret->{$node1};
$status->{$node1}->{'appsd'} = $ret->{$node1};
}
}
}
}
#handle local l commands
if (exists($request->{'lcmdapps'})) {
for (my $i=1; $i<=$request->{'lcmdapps'}->[0]; $i++) {
my %cmdhash=();
my @apps=split(',', $request->{"lcmdapps$i"}->[0]);
my @cmds=split(',', $request->{"lcmdapps$i" . "cmd"}->[0]);
my $nodes=$request->{"lcmdapps$i" . "node"};
for (my $j=0; $j <@cmds; $j++) {
$cmdhash{$cmds[$j]}=$apps[$j];
}
my $ret = process_request_local_command($request, $callback, $doreq, $nodes, \%cmdhash);
foreach my $node1 (keys(%$ret)) {
if (exists($status->{$node1})) {
my $appstatus=$status->{$node1}->{'appstatus'};
if ($appstatus) { $status->{$node1}->{'appstatus'} .= "," . $ret->{$node1}; }
else { $status->{$node1}->{'appstatus'} = $ret->{$node1}; }
my $appsd=$status->{$node1}->{'appsd'};
if ($appsd) { $status->{$node1}->{'appsd'} .= "," . $ret->{$node1}; }
else { $status->{$node1}->{'appsd'} = $ret->{$node1}; }
} else {
$status->{$node1}->{'appstatus'} = $ret->{$node1};
$status->{$node1}->{'appsd'} = $ret->{$node1};
}
}
}
}
#handle remote commands
if (exists($request->{'dcmdapps'})) {
for (my $i=1; $i<=$request->{'dcmdapps'}->[0]; $i++) {
my %dcmdhash=();
my @apps=split(',', $request->{"dcmdapps$i"}->[0]);
my @dcmds=split(',', $request->{"dcmdapps$i" . "dcmd"}->[0]);
my $nodes=$request->{"dcmdapps$i" . "node"};
for (my $j=0; $j <@dcmds; $j++) {
$dcmdhash{$dcmds[$j]}=$apps[$j];
}
my $ret = process_request_remote_command($request, $callback, $doreq, $nodes, \%dcmdhash);
foreach my $node1 (keys(%$ret)) {
if (exists($status->{$node1})) {
my $appstatus=$status->{$node1}->{'appstatus'};
if ($appstatus) { $status->{$node1}->{'appstatus'} .= "," . $ret->{$node1}; }
else { $status->{$node1}->{'appstatus'} = $ret->{$node1}; }
my $appsd=$status->{$node1}->{'appsd'};
if ($appsd) { $status->{$node1}->{'appsd'} .= "," . $ret->{$node1}; }
else { $status->{$node1}->{'appsd'} = $ret->{$node1}; }
} else {
$status->{$node1}->{'appstatus'} = $ret->{$node1};
$status->{$node1}->{'appsd'} = $ret->{$node1};
}
}
}
}
#nodestat_internal command the output, nodestat command will collect it
foreach my $node1 (sort keys(%$status)) {
my %rsp;
$rsp{name}=[$node1];
my $st=$status->{$node1}->{'status'};
my $ast= $status->{$node1}->{'appstatus'};
my $appsd = $status->{$node1}->{'appsd'};
$st=$st?$st:'';
$ast=$ast?$ast:'';
$appsd=$appsd?$appsd:'';
$rsp{data}->[0] = "$st$separator$ast$separator$appsd";
$callback->({node=>[\%rsp]});
}
} else { #nodestat command
#first collect the status from the nodes
my $reqcopy = {%$request};
$reqcopy->{command}->[0]='nodestat_internal';
my $ret = xCAT::Utils->runxcmd($reqcopy, $doreq, 0, 1);
#print Dumper($ret);
my $status={};
my @noping_nodes=();
my $power=$request->{'power'}->[0];
foreach my $tmpdata (@$ret) {
if ($tmpdata =~ /([^:]+): (.*)$separator(.*)$separator(.*)/) {
#print "node=$1, status=$2, appstatus=$3, appsd=$4\n";
if ($status->{$1}->{'status'}) {
$status->{$1}->{'status'}=$status->{$1}->{'status'} . ",$2";
} else {
$status->{$1}->{'status'}=$2;
}
if ($status->{$1}->{'appstatus'}) {
$status->{$1}->{'appstatus'}= $status->{$1}->{'appstatus'} . ",$3";
} else {
$status->{$1}->{'appstatus'}=$3;
}
if ($status->{$1}->{'appsd'}) {
$status->{$1}->{'appsd'}=$status->{$1}->{'appsd'} . ",$4";
} else {
$status->{$1}->{'appsd'}=$4;
}
if (($power) && ($2 eq "noping")) {
push(@noping_nodes, $1);
}
} else {
my $rsp;
$rsp->{data}->[0]= "$tmpdata";
xCAT::MsgUtils->message("I", $rsp, $callback);
}
}
#print Dumper($status);
#get power status for noping nodes
if (($power) && (@noping_nodes > 0)) {
#print "noping_nodes=@noping_nodes\n";
my $ret = xCAT::Utils->runxcmd(
{
command => ['rpower'],
node => \@noping_nodes,
arg => [ 'stat' ]
},
$doreq, 0, 1 );
foreach my $tmpdata (@$ret) {
if ($tmpdata =~ /([^:]+): (.*)/) {
$status->{$1}->{'status'}="noping($2)";
} else {
my $rsp;
$rsp->{data}->[0]= "$tmpdata";
xCAT::MsgUtils->message("I", $rsp, $callback);
}
}
}
#print Dumper($request);
my $update=$request->{'update'}->[0];
my $quite=$request->{'quite'}->[0];
#show the output
if (!$quite) {
foreach my $node1 (sort keys(%$status)) {
my %rsp;
$rsp{name}=[$node1];
my $st=$status->{$node1}->{'status'};
my $ast= $status->{$node1}->{'appstatus'};
if ($st) {
if ($st eq 'ping') { $st = $ast ? "$ast" : "$st"; }
else { $st = $ast ? "$st,$ast" : "$st"; }
} else {
$st=$ast;
}
$rsp{data}->[0] = $st;
$callback->({node=>[\%rsp]});
}
}
#update the nodelist table
if ($update) {
my $nodetab=xCAT::Table->new('nodelist', -create=>1);
if ($nodetab) {
my $status1={};
#get current values and compare with the new value to decide if update of db is necessary
my @nodes1=keys(%$status);
my $stuff = $nodetab->getNodesAttribs(\@nodes1, ['node', 'status', 'appstatus']);
#get current local time
my (
$sec, $min, $hour, $mday, $mon,
$year, $wday, $yday, $isdst
)
= localtime(time);
my $currtime = sprintf("%02d-%02d-%04d %02d:%02d:%02d",
$mon + 1, $mday, $year + 1900,
$hour, $min, $sec);
foreach my $node1 (@nodes1) {
my $oldstatus=$stuff->{$node1}->[0]->{status};
my $newstatus=$status->{$node1}->{status};
if ($newstatus) {
if ((!$oldstatus) || ($newstatus ne $oldstatus)) {
$status1->{$node1}->{status}= $newstatus;
$status1->{$node1}->{statustime}= $currtime;
}
}
else {
if ($oldstatus) {
$status1->{$node1}->{status}= "";
$status1->{$node1}->{statustime}= "";
}
}
my $oldappstatus=$stuff->{$node1}->[0]->{'appstatus'};
my $newappstatus=$status->{$node1}->{'appsd'};
while ($newappstatus =~ /(\w+)\=(\w+)/) {
my $tmp1=$1;
my $tmp2=$2;
if ($oldappstatus) {
if($oldappstatus =~ /$tmp1\=/){
$oldappstatus =~ s/$tmp1\=\w+/$tmp1\=$tmp2/g;
}else{
$oldappstatus = $oldappstatus."\,$tmp1\=$tmp2";
}
} else {
$oldappstatus = "$tmp1\=$tmp2";
}
$newappstatus =~ s/(\w+)\=(\w+)//;
}
$status1->{$node1}->{appstatus}= $oldappstatus;
$status1->{$node1}->{appstatustime}= $currtime;
}
#print Dumper($status1);
$nodetab->setNodesAttribs($status1);
}
}
}
}
sub usage
{
my $cb=shift;
my $rsp={};
$rsp->{data}->[0]= "Usage:";
$rsp->{data}->[1]= " nodestat [noderange] [-m|--usemon] [-p|powerstat] [-u|--updatedb]";
$rsp->{data}->[2]= " nodestat [-h|--help|-v|--version]";
xCAT::MsgUtils->message("I", $rsp, $cb);
}
#--------------------------------------------------------------------------------
=head3 getStatusMonsettings
This function goes to the monsetting table to retrieve the settings related to
the node status and app status monitoring.
Arguments:
none.
Returns:
a hash that has settings from the monsetting table for node status and
app status monitoring. For example:
( 'APPS'=>[ll,gpfs],
'll' =>
{
'group' => 'service,compute',
'port' => '5001'
},
'gpfs' =>
{
'group' => 'service',
'cmd' => '/tmp/gpfscmd'
};
)
=cut
#--------------------------------------------------------------------------------
sub getStatusMonsettings {
my %apps=();
my $tab=xCAT::Table->new('monsetting');
if ( defined($tab)) {
my ($ent) = $tab->getAttribs({name => 'xcatmon', key => 'apps' }, 'value');
if ( defined($ent) ) {
my $tmp_list=$ent->{value};
if ($tmp_list) {
my @applist=split(',', $tmp_list);
foreach my $app (@applist) {
$apps{$app}={};
}
$apps{'APPS'}=\@applist;
my @results = $tab->getAttribs({name => 'xcatmon'}, 'key','value');
if (@results) {
foreach(@results) {
my $key=$_->{key};
my $value=$_->{value};
if (exists($apps{$key})) {
my @tem_value=split(',',$value);
foreach my $pair (@tem_value) {
my @tmp_action=split('=', $pair);
if (exists($apps{$key}->{$tmp_action[0]})) {
$apps{$key}->{$tmp_action[0]} = $apps{$key}->{$tmp_action[0]} . "," . $tmp_action[1];
} else {
$apps{$key}->{$tmp_action[0]}=$tmp_action[1];
}
}
}
}
}
}
}
}
return %apps;
}
#--------------------------------------------------------------------------------
=head3 getNodeStatusAndAppstatus
This function goes to the xCAT nodelist table to retrieve the saved node status and appstatus
for all the node that are managed by local nodes.
Arguments:
nodelist--- an array of nodes
Returns:
a hash pointer that has the node status and appstatus. The format is:
{ node1=> {
status=>'active',appstatus=>'sshd=up,ll=up,gpfs=down'
} ,
node2=> {
status=>'active',appstatus=>'sshd=up,ll=down,gpfs=down'
}
}
=cut
#--------------------------------------------------------------------------------
sub getMonNodesStatusAndAppStatus {
my @nodes=@_;
my %status=();
my $table=xCAT::Table->new("nodelist", -create =>1);
my $tabdata=$table->getNodesAttribs(\@nodes,['node', 'status', 'appstatus']);
foreach my $node (@nodes) {
my $tmp1=$tabdata->{$node}->[0];
if ($tmp1) {
$status{$node}->{status}=$tmp1->{status};
$status{$node}->{appstatus}=$tmp1->{appstatus};
}
}
return %status;
}
1;
Reference in New Issue View Git Blame Copy Permalink