appstatus support, xcatmon updates appstatus, nodestat command supporting updating the db
git-svn-id: https://svn.code.sf.net/p/xcat/code/xcat-core/trunk@5100 8638fb3e-16cb-4fca-ae20-7b5d299a9bcd
This commit is contained in:
linggao
parent
d661dca0e2
commit
ee1eec6f95
@ -109,6 +109,8 @@ sub supportNodeStatusMon {
|
||||
sub startNodeStatusMon
|
||||
{
|
||||
print "xcatmon.startNodeStatusMon\n";
|
||||
if (! -e "/etc/xCATMN") { return (0, ""); } #only run the cron job on mn
|
||||
|
||||
my $noderef=shift;
|
||||
if ($noderef =~ /xCAT_monitoring::xcatmon/) {
|
||||
$noderef=shift;
|
||||
@ -117,7 +119,8 @@ sub startNodeStatusMon
|
||||
my $callback=shift;
|
||||
|
||||
#run the command first to update the status,
|
||||
my $cmd="$::XCATROOT/sbin/xcatnodemon";
|
||||
#my $cmd="$::XCATROOT/sbin/xcatnodemon";
|
||||
my $cmd="$::XCATROOT/bin/nodestat all -m -u -q";
|
||||
#$output=`$cmd 2>&1`;
|
||||
#if ($?) {
|
||||
# print "xcatmon: $output\n";
|
||||
@ -194,6 +197,8 @@ sub startNodeStatusMon
|
||||
#--------------------------------------------------------------------------------
|
||||
sub stopNodeStatusMon {
|
||||
print "xcatmon.stopNodeStatusMon\n";
|
||||
if (! -e "/etc/xCATMN") { return (0, ""); } #only run the cron job on mn
|
||||
|
||||
my $noderef=shift;
|
||||
if ($noderef =~ /xCAT_monitoring::xcatmon/) {
|
||||
$noderef=shift;
|
||||
@ -201,7 +206,8 @@ sub stopNodeStatusMon {
|
||||
my $scope=shift;
|
||||
my $callback=shift;
|
||||
|
||||
my $job="$::XCATROOT/sbin/xcatnodemon";
|
||||
#my $job="$::XCATROOT/sbin/xcatnodemon";
|
||||
my $job="$::XCATROOT/bin/nodestat all -m -u -q";
|
||||
my ($code, $msg)=xCAT::Utils::remove_cron_job($job);
|
||||
my $localhostname=hostname();
|
||||
if ($code==0) {
|
||||
@ -291,49 +297,49 @@ sub getMonNodesStatus {
|
||||
return %status;
|
||||
}
|
||||
|
||||
|
||||
my @mon_servers=keys(%$hierachy);
|
||||
my $isSV=xCAT::Utils->isServiceNode();
|
||||
|
||||
#on a service node or on ms, get the nodes that has local host as the server node
|
||||
my $monnodes;
|
||||
my @hostinfo=xCAT::Utils->determinehostname();
|
||||
my %iphash=();
|
||||
foreach(@hostinfo) {$iphash{$_}=1;}
|
||||
#if this is mn, include the ones that has no service nodes
|
||||
if (!$isSV) { $iphash{'noservicenode'}=1;}
|
||||
|
||||
|
||||
my %processed=();
|
||||
foreach(@mon_servers) {
|
||||
#service node come in pairs, the first one is the monserver adapter that facing the mn,
|
||||
# the second one is facing the cn. we use the first one here
|
||||
my @server_pair=split(':', $_);
|
||||
my $sv=$server_pair[0];
|
||||
if ($iphash{$sv}) {
|
||||
$monnodes=$hierachy->{$_};
|
||||
}
|
||||
|
||||
|
||||
foreach(@$monnodes) {
|
||||
my $node=$_->[0];
|
||||
my $status=$_->[2];
|
||||
my $type=$_[1];
|
||||
if (!$status) { $status=$::STATUS_DEFINED;} #default
|
||||
if (!$processed{$sv}) { $processed{$sv}=1;}
|
||||
else { next; }
|
||||
|
||||
if ($status eq $::STATUS_ACTIVE) { push(@active_nodes, $node);}
|
||||
elsif ($status eq $::STATUS_INACTIVE) { push(@inactive_nodes, $node);}
|
||||
else {
|
||||
my $need_active=0;
|
||||
my $need_inactive=0;
|
||||
if ($::NEXT_NODESTAT_VAL{$status}->{$::STATUS_ACTIVE}==1) { $need_active=1;}
|
||||
if ($::NEXT_NODESTAT_VAL{$status}->{$::STATUS_INACTIVE}==1) { $need_inactive=1;}
|
||||
if (($need_active==1) && ($need_inactive==0)) { push(@inactive_nodes, $node); } #put it into the inactive list so that the monitoring code can switch it to active.
|
||||
elsif (($need_active==0) && ($need_inactive==1)) { push(@active_nodes, $node); } #put it into the active list so that the monitoring code can chane it to inactive.
|
||||
elsif (($need_active==1) && ($need_inactive==1)) { push(@unknown_nodes, $node);} #unknow list so that the monitoring code can change it to active or inactive
|
||||
else {
|
||||
#if it is non-osi node, check it anyway
|
||||
if ($type !~ /osi/) {push(@unknown_nodes, $node);}
|
||||
}
|
||||
if ($iphash{$sv}) {
|
||||
my $monnodes=$hierachy->{$_};
|
||||
|
||||
foreach(@$monnodes) {
|
||||
my $node=$_->[0];
|
||||
my $status=$_->[2];
|
||||
my $type=$_[1];
|
||||
if (!$status) { $status=$::STATUS_DEFINED;} #default
|
||||
|
||||
if ($status eq $::STATUS_ACTIVE) { push(@active_nodes, $node);}
|
||||
elsif ($status eq $::STATUS_INACTIVE) { push(@inactive_nodes, $node);}
|
||||
else {
|
||||
my $need_active=0;
|
||||
my $need_inactive=0;
|
||||
if ($::NEXT_NODESTAT_VAL{$status}->{$::STATUS_ACTIVE}==1) { $need_active=1;}
|
||||
if ($::NEXT_NODESTAT_VAL{$status}->{$::STATUS_INACTIVE}==1) { $need_inactive=1;}
|
||||
if (($need_active==1) && ($need_inactive==0)) { push(@inactive_nodes, $node); } #put it into the inactive list so that the monitoring code can switch it to active.
|
||||
elsif (($need_active==0) && ($need_inactive==1)) { push(@active_nodes, $node); } #put it into the active list so that the monitoring code can chane it to inactive.
|
||||
elsif (($need_active==1) && ($need_inactive==1)) { push(@unknown_nodes, $node);} #unknow list so that the monitoring code can change it to active or inactive
|
||||
else {
|
||||
#if it is non-osi node, check it anyway
|
||||
if ($type !~ /osi/) {push(@unknown_nodes, $node);}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -346,6 +352,8 @@ sub getMonNodesStatus {
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
#--------------------------------------------------------------------------------
|
||||
=head3 setNodeStatusAttributes
|
||||
This function will update the status column of the nodelist table with the new node status.
|
||||
|
||||
@ -1,11 +1,28 @@
|
||||
package xCAT_plugin::nodestat;
|
||||
BEGIN
|
||||
{
|
||||
$::XCATROOT = $ENV{'XCATROOT'} ? $ENV{'XCATROOT'} : '/opt/xcat';
|
||||
}
|
||||
use lib "$::XCATROOT/lib/perl";
|
||||
|
||||
|
||||
use strict;
|
||||
use warnings;
|
||||
|
||||
|
||||
|
||||
use Socket;
|
||||
use IO::Handle;
|
||||
use Getopt::Long;
|
||||
use Data::Dumper;
|
||||
use xCAT::GlobalDef;
|
||||
|
||||
|
||||
my %nodesetstats;
|
||||
my %default_ports = (
|
||||
'ssh' => '22',
|
||||
'sshd' => '22'
|
||||
);
|
||||
|
||||
sub handled_commands {
|
||||
return {
|
||||
@ -81,99 +98,349 @@ sub preprocess_request
|
||||
($req->{_xcatpreprocessed}->[0] == 1)) {
|
||||
return [$req];
|
||||
}
|
||||
|
||||
@ARGV=();
|
||||
my $args=$req->{arg};
|
||||
if ($args) {
|
||||
@ARGV = @{$args};
|
||||
}
|
||||
|
||||
# parse the options
|
||||
$::UPDATE=0;
|
||||
$::QUITE=0;
|
||||
$::MON=0;
|
||||
Getopt::Long::Configure("posix_default");
|
||||
Getopt::Long::Configure("no_gnu_compat");
|
||||
Getopt::Long::Configure("bundling");
|
||||
if (!GetOptions(
|
||||
'm|usemon' => \$::MON,
|
||||
'q|quite' => \$::QUITE, #this is a internal flag used by monitoring
|
||||
'u|update' => \$::UPDATE,
|
||||
'h|help' => \$::HELP,
|
||||
'v|version' => \$::VERSION))
|
||||
{
|
||||
&usage($cb);
|
||||
return(1);
|
||||
}
|
||||
if ($::HELP) {
|
||||
&usage($cb);
|
||||
return(0);
|
||||
}
|
||||
if ($::VERSION) {
|
||||
my $version = xCAT::Utils->Version();
|
||||
my $rsp={};
|
||||
$rsp->{data}->[0] = "$version";
|
||||
xCAT::MsgUtils->message("I", $rsp, $cb);
|
||||
return(0);
|
||||
}
|
||||
|
||||
|
||||
my $nodes = $req->{node};
|
||||
my $service = "xcat";
|
||||
my @requests;
|
||||
if ($nodes){
|
||||
if (-x '/usr/bin/nmap' or -x '/usr/local/bin/nmap') {
|
||||
my $usenmapfrommn=0;
|
||||
my $sitetab = xCAT::Table->new('site');
|
||||
if ($sitetab) {
|
||||
(my $ref) = $sitetab->getAttribs({key => 'useNmapfromMN'}, 'value');
|
||||
if ($ref) {
|
||||
if ($ref->{value} =~ /1|yes|YES|Y|y/) { $usenmapfrommn=1; }
|
||||
}
|
||||
}
|
||||
if ($usenmapfrommn) {
|
||||
return [$req]; #do not distribute, nodestat seems to lose accuracy and slow down distributed, if using nmap
|
||||
}
|
||||
}
|
||||
# find service nodes for requested nodes
|
||||
# build an individual request for each service node
|
||||
my $sn = xCAT::Utils->get_ServiceNode($nodes, $service, "MN");
|
||||
if ($nodes) {
|
||||
my $usenmapfrommn=0;
|
||||
if (-x '/usr/bin/nmap' or -x '/usr/local/bin/nmap') {
|
||||
my $sitetab = xCAT::Table->new('site');
|
||||
if ($sitetab) {
|
||||
(my $ref) = $sitetab->getAttribs({key => 'useNmapfromMN'}, 'value');
|
||||
if ($ref) {
|
||||
if ($ref->{value} =~ /1|yes|YES|Y|y/) { $usenmapfrommn=1; }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# build each request for each service node
|
||||
foreach my $snkey (keys %$sn)
|
||||
{
|
||||
my $reqcopy = {%$req};
|
||||
$reqcopy->{node} = $sn->{$snkey};
|
||||
#get monsettings
|
||||
my %apps = ();
|
||||
if ($::MON == 1) { %apps=getStatusMonsettings(); }
|
||||
|
||||
#if no apps specified in the monsetting table, add sshd, pbs and xend
|
||||
if (keys(%apps) == 0) {
|
||||
$apps{'sshd'}->{'group'} = "ALL"; #ALL means anything on the nodelist table, it is different from all
|
||||
$apps{'sshd'}->{'port'} = "22";
|
||||
$apps{'pbs'}->{'group'} = "ALL";
|
||||
$apps{'pbs'}->{'port'} = "15002";
|
||||
$apps{'xend'}->{'group'} = "ALL";
|
||||
$apps{'xend'}->{'port'} = "8002";
|
||||
$apps{'APPS'}=['sshd', 'pbs', 'xend'];
|
||||
} else {
|
||||
#go thorugh the settings and put defaults in
|
||||
foreach my $app (keys(%apps)) {
|
||||
if ($app eq 'APPS') { next; }
|
||||
if (!exists($apps{$app}->{'group'})) { $apps{$app}->{'group'} = "ALL"; }
|
||||
if (exists($apps{$app}->{'cmd'}) || exists($apps{$app}->{'dcmd'})) { next; }
|
||||
if (exists($apps{$app}->{'port'})) { next; }
|
||||
#add port number in if nothing is specified
|
||||
if (exists($default_ports{$app})) { $apps{$app}->{'port'} = $default_ports{$app}; }
|
||||
else {
|
||||
print "need to get the port number form /etc/services\n";
|
||||
}
|
||||
}
|
||||
|
||||
#always add sshd
|
||||
if (!exists($apps{'ssh'}) || !exists($apps{'sshd'}) ) {
|
||||
$apps{'sshd'}->{'group'} = "ALL";
|
||||
$apps{'sshd'}->{'port'} = "22";
|
||||
my $pa=$apps{'APPS'};
|
||||
push @$pa, 'sshd';
|
||||
}
|
||||
}
|
||||
|
||||
#print Dumper(%apps);
|
||||
# find service nodes for requested nodes
|
||||
# build an individual request for each service node
|
||||
my $sn = xCAT::Utils->get_ServiceNode($nodes, $service, "MN");
|
||||
|
||||
#get the member for each group
|
||||
my %groups=();
|
||||
foreach my $app (keys %apps) {
|
||||
if ($app eq 'APPS') { next; }
|
||||
my $group=$apps{$app}->{'group'};
|
||||
if (($group) && ($group ne "ALL") && (! exists($groups{$group}))) {
|
||||
my @tmp_nodes = xCAT::NodeRange::noderange($group);
|
||||
foreach (@tmp_nodes) {
|
||||
$groups{$group}->{$_}=1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# build each request for each service node
|
||||
my $all_apps=$apps{'APPS'};
|
||||
my %all_porthash=(); #stores all the port apps if usenmapfrommn=1
|
||||
foreach my $snkey (keys %$sn)
|
||||
{
|
||||
my $reqcopy = {%$req};
|
||||
$reqcopy->{node} = $sn->{$snkey};
|
||||
$reqcopy->{'_xcatdest'} = $snkey;
|
||||
$reqcopy->{_xcatpreprocessed}->[0] = 1;
|
||||
push @requests, $reqcopy;
|
||||
|
||||
}
|
||||
}
|
||||
else
|
||||
{ # non node options like -h
|
||||
@ARGV=();
|
||||
my $args=$req->{arg};
|
||||
if ($args) {
|
||||
@ARGV = @{$args};
|
||||
} else {
|
||||
&usage($cb);
|
||||
return(1);
|
||||
}
|
||||
# parse the options
|
||||
Getopt::Long::Configure("posix_default");
|
||||
Getopt::Long::Configure("no_gnu_compat");
|
||||
Getopt::Long::Configure("bundling");
|
||||
if (!GetOptions(
|
||||
'h|help' => \$::HELP,
|
||||
'v|version' => \$::VERSION))
|
||||
{
|
||||
&usage($cb);
|
||||
return(1);
|
||||
}
|
||||
if ($::HELP) {
|
||||
&usage($cb);
|
||||
return(0);
|
||||
}
|
||||
if ($::VERSION) {
|
||||
my $version = xCAT::Utils->Version();
|
||||
my $rsp={};
|
||||
$rsp->{data}->[0] = "$version";
|
||||
xCAT::MsgUtils->message("I", $rsp, $cb);
|
||||
return(0);
|
||||
}
|
||||
|
||||
$reqcopy->{'update'}->[0]=$::UPDATE;
|
||||
$reqcopy->{'quite'}->[0]=$::QUITE;
|
||||
$reqcopy->{'useNmapfromMN'}->[0]=$usenmapfrommn;
|
||||
$reqcopy->{'allapps'}=$all_apps;
|
||||
|
||||
my %porthash=(); #('sshd,ll'=> {
|
||||
# port=>'22,5001',
|
||||
# node=>[node1,node2]
|
||||
# }
|
||||
#)
|
||||
my %cmdhash=(); #('gpfs,myapp'=> {
|
||||
# cmd=>'/tmp/mycmd1,/usr/bin/date',
|
||||
# node=>[node1,node2]
|
||||
# }
|
||||
#)
|
||||
my %dcmdhash=(); #('myapp2,myapp3'=> {
|
||||
# dcmd=>'/tmp/mycmd1,/usr/bin/date',
|
||||
# node=>[node1,node2]
|
||||
# }
|
||||
#)
|
||||
my @nodes_for_sn=@{$sn->{$snkey}};
|
||||
foreach my $node (@nodes_for_sn) {
|
||||
my @ports;
|
||||
my @portapps;
|
||||
my @cmds;
|
||||
my @cmdapps;
|
||||
my @dcmds;
|
||||
my @dcmdapps;
|
||||
foreach my $app (keys %apps) {
|
||||
if ($app eq 'APPS') { next; }
|
||||
my $group=$apps{$app}->{'group'};
|
||||
if (($group eq "ALL") || ($groups{$group}->{$node})) {
|
||||
#print "app=$app\n";
|
||||
if (exists($apps{$app}->{'port'})) {
|
||||
push @ports, $apps{$app}->{'port'};
|
||||
push @portapps, $app;
|
||||
}
|
||||
elsif (exists($apps{$app}->{'cmd'})) {
|
||||
push @cmds, $apps{$app}->{'cmd'};
|
||||
push @cmdapps, $app;
|
||||
}
|
||||
elsif (exists($apps{$app}->{'dcmd'})) {
|
||||
push @dcmds, $apps{$app}->{'dcmd'};
|
||||
push @dcmdapps, $app;
|
||||
}
|
||||
}
|
||||
}
|
||||
#print "ports=@ports\n";
|
||||
#print "portapps=@portapps\n";
|
||||
#print "cmds=@cmds\n";
|
||||
#print "cmdapps=@cmdapps\n";
|
||||
#print "dcmds=@dcmds\n";
|
||||
#print "dcmdapps=@dcmdapps\n";
|
||||
if (@portapps>0) {
|
||||
my $tmpapps=join(',', @portapps);
|
||||
if (($usenmapfrommn==1) && (@cmdapps==0) && (@dcmdapps==0)) {
|
||||
#this is the case where mn handles ports for all nodes using nmap
|
||||
#The current limitation is that when there are cmd or dcmd specified for the node
|
||||
# nmap has to be done on the service node because if both mn and sn update the appstatus
|
||||
# one will overwites the other.
|
||||
if (exists($all_porthash{$tmpapps})) {
|
||||
my $pa=$all_porthash{$tmpapps}->{'node'};
|
||||
push @$pa, $node;
|
||||
} else {
|
||||
$all_porthash{$tmpapps}->{'node'}=[$node];
|
||||
$all_porthash{$tmpapps}->{'port'}=join(',', @ports);
|
||||
}
|
||||
} else {
|
||||
if (exists($porthash{$tmpapps})) {
|
||||
my $pa=$porthash{$tmpapps}->{'node'};
|
||||
push @$pa, $node;
|
||||
} else {
|
||||
$porthash{$tmpapps}->{'node'}=[$node];
|
||||
$porthash{$tmpapps}->{'port'}=join(',', @ports);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (@cmdapps>0) {
|
||||
my $tmpapps=join(',', @cmdapps);
|
||||
if (exists($cmdhash{$tmpapps})) {
|
||||
my $pa=$cmdhash{$tmpapps}->{'node'};
|
||||
push @$pa, $node;
|
||||
} else {
|
||||
$cmdhash{$tmpapps}->{'node'}=[$node];
|
||||
$cmdhash{$tmpapps}->{'cmd'}=join(',', @cmds);
|
||||
}
|
||||
}
|
||||
if (@dcmdapps>0) {
|
||||
my $tmpapps=join(',', @dcmdapps);
|
||||
if (exists($dcmdhash{$tmpapps})) {
|
||||
my $pa=$dcmdhash{$tmpapps}->{'node'};
|
||||
push @$pa, $node;
|
||||
} else {
|
||||
$dcmdhash{$tmpapps}->{'node'}=[$node];
|
||||
$dcmdhash{$tmpapps}->{'dcmd'}=join(',', @dcmds);
|
||||
}
|
||||
}
|
||||
} #end foreach (@nodes_for_sn)
|
||||
|
||||
#print Dumper(%porthash);
|
||||
#now push the settings into the requests
|
||||
my $i=1;
|
||||
if ((keys(%porthash) == 0) && (keys(%cmdhash) == 0) && (keys(%dcmdhash) == 0)) { next; }
|
||||
foreach my $tmpapps (keys %porthash) {
|
||||
$reqcopy->{'portapps'}->[0]= scalar keys %porthash;
|
||||
$reqcopy->{"portapps$i"}->[0]= $tmpapps;
|
||||
$reqcopy->{"portapps$i" . "port"}->[0]= $porthash{$tmpapps}->{'port'};
|
||||
$reqcopy->{"portapps$i" . "node"} = $porthash{$tmpapps}->{'node'};;
|
||||
$i++;
|
||||
}
|
||||
$i=1;
|
||||
foreach my $tmpapps (keys %cmdhash) {
|
||||
$reqcopy->{'cmdapps'}->[0]= scalar keys %cmdhash;
|
||||
$reqcopy->{"cmdapps$i"}->[0]= $tmpapps;
|
||||
$reqcopy->{"cmdapps$i" . "cmd"}->[0]= $cmdhash{$tmpapps}->{'cmd'};
|
||||
$reqcopy->{"cmdapps$i" . "node"} = $cmdhash{$tmpapps}->{'node'};;
|
||||
$i++;
|
||||
}
|
||||
$i=1;
|
||||
foreach my $tmpapps (keys %dcmdhash) {
|
||||
$reqcopy->{'dcmdapps'}->[0]= scalar keys %dcmdhash;
|
||||
$reqcopy->{"dcmdapps$i"}->[0]= $tmpapps;
|
||||
$reqcopy->{"dcmdapps$i" . "dcmd"}->[0]= $dcmdhash{$tmpapps}->{'dcmd'};
|
||||
$reqcopy->{"dcmdapps$i" . "node"} = $dcmdhash{$tmpapps}->{'node'};
|
||||
$i++;
|
||||
}
|
||||
|
||||
|
||||
#done
|
||||
push @requests, $reqcopy;
|
||||
}
|
||||
|
||||
#mn handles all nmap when useNmapfromMN=1 on the site table
|
||||
if (($usenmapfrommn == 1) && (keys(%all_porthash) > 0)) {
|
||||
my @hostinfo=xCAT::Utils->determinehostname();
|
||||
my %iphash=();
|
||||
foreach(@hostinfo) {$iphash{$_}=1;}
|
||||
my $handled=0;
|
||||
foreach my $req (@requests) {
|
||||
my $currsn=$req->{'_xcatdest'};
|
||||
if (exists($iphash{$currsn})) {
|
||||
my $i=1;
|
||||
foreach my $tmpapps (keys %all_porthash) {
|
||||
$req->{'portapps'}->[0]= scalar keys %all_porthash;
|
||||
$req->{"portapps$i"}->[0]= $tmpapps;
|
||||
$req->{"portapps$i" . "port"}->[0]= $all_porthash{$tmpapps}->{'port'};
|
||||
$req->{"portapps$i" . "node"} = $all_porthash{$tmpapps}->{'node'};;
|
||||
$i++;
|
||||
}
|
||||
$handled=1;
|
||||
last;
|
||||
}
|
||||
}
|
||||
|
||||
if (!$handled) {
|
||||
my $reqcopy = {%$req};
|
||||
$reqcopy->{_xcatpreprocessed}->[0] = 1;
|
||||
$reqcopy->{'update'}->[0]=$::UPDATE;
|
||||
$reqcopy->{'quite'}->[0]=$::QUITE;
|
||||
$reqcopy->{'useNmapfromMN'}->[0]=$usenmapfrommn;
|
||||
$reqcopy->{'allapps'}=$all_apps;
|
||||
my $i=1;
|
||||
foreach my $tmpapps (keys %all_porthash) {
|
||||
$reqcopy->{'portapps'}->[0]= scalar keys %all_porthash;
|
||||
$reqcopy->{"portapps$i"}->[0]= $tmpapps;
|
||||
$reqcopy->{"portapps$i" . "port"}->[0]= $all_porthash{$tmpapps}->{'port'};
|
||||
$reqcopy->{"portapps$i" . "node"} = $all_porthash{$tmpapps}->{'node'};;
|
||||
$i++;
|
||||
}
|
||||
push @requests, $reqcopy;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
#if ($usenmapfrommn) {
|
||||
# my $reqcopy = {%$req};
|
||||
# $reqcopy->{'update'}->[0]=$::UPDATE;
|
||||
# $reqcopy->{'useNmapfromMN'}->[0]=1;
|
||||
# if (!$::UPDATE) {
|
||||
# push @requests, $reqcopy;
|
||||
# return \@requests; #do not distribute, nodestat seems to lose accuracy and slow down distributed, if using nmap
|
||||
# }
|
||||
#}
|
||||
|
||||
}
|
||||
|
||||
return \@requests;
|
||||
}
|
||||
|
||||
sub interrogate_node { #Meant to run against confirmed up nodes
|
||||
my $node=shift;
|
||||
my $doreq=shift;
|
||||
my $p_tmp=shift;
|
||||
my %portservices = %$p_tmp;
|
||||
|
||||
my $status = "";
|
||||
if (nodesockopen($node,15002)) {
|
||||
$status.="pbs,"
|
||||
}
|
||||
if (nodesockopen($node,8002)) {
|
||||
$status.="xend,"
|
||||
}
|
||||
if (nodesockopen($node,22)) {
|
||||
$status.="sshd,"
|
||||
my $appsd=""; #detailed status
|
||||
my $ret={};
|
||||
$ret->{'status'}=$::STATUS_ACTIVE;
|
||||
|
||||
foreach my $port (keys(%portservices)) {
|
||||
if (nodesockopen($node,$port)) {
|
||||
$status.=$portservices{$port} . ",";
|
||||
$appsd.=$portservices{$port} . "=up,";
|
||||
} else {
|
||||
$appsd.=$portservices{$port} . "=down,";
|
||||
}
|
||||
}
|
||||
|
||||
$status =~ s/,$//;
|
||||
$appsd =~ s/,$//;
|
||||
$ret->{'appsd'}=$appsd;
|
||||
if ($status) {
|
||||
return $status;
|
||||
$ret->{'appstatus'}=$status;
|
||||
return $ret;
|
||||
}
|
||||
if ($status = installer_query($node)) {
|
||||
return $status;
|
||||
$ret->{'status'}=$status;
|
||||
return $ret;
|
||||
} else { #pingable, but no *clue* as to what the state may be
|
||||
$doreq->({command=>['nodeset'],
|
||||
node=>[$node],
|
||||
arg=>['stat']},
|
||||
\&getstat);
|
||||
return 'ping '.$nodesetstats{$node};
|
||||
$ret->{'status'} = 'ping '.$nodesetstats{$node};
|
||||
return $ret;
|
||||
}
|
||||
}
|
||||
|
||||
@ -181,14 +448,16 @@ sub process_request_nmap {
|
||||
my $request = shift;
|
||||
my $callback = shift;
|
||||
my $doreq = shift;
|
||||
my $nodelist=shift;
|
||||
my $p_tmp=shift;
|
||||
my %portservices = %$p_tmp;
|
||||
my @nodes =();
|
||||
if ($nodelist) { @nodes=@$nodelist;}
|
||||
|
||||
my $update=$request->{'update'}->[0];
|
||||
|
||||
my %nodebyip;
|
||||
my %portservices = (
|
||||
'22' => 'sshd',
|
||||
'15002' => 'pbs',
|
||||
'8002' => 'xend',
|
||||
);
|
||||
my @livenodes;
|
||||
my @nodes = @{$request->{node}};
|
||||
my %unknownnodes;
|
||||
foreach (@nodes) {
|
||||
$unknownnodes{$_}=1;
|
||||
@ -204,6 +473,7 @@ sub process_request_nmap {
|
||||
}
|
||||
}
|
||||
|
||||
my $ret={};
|
||||
my $node;
|
||||
my $fping;
|
||||
my $ports = join ',',keys %portservices;
|
||||
@ -211,6 +481,7 @@ sub process_request_nmap {
|
||||
foreach (@nodes) {
|
||||
$deadnodes{$_}=1;
|
||||
}
|
||||
#print "nmap -PE --send-ip -p $ports,3001 ".join(' ',@nodes) . "\n";
|
||||
open($fping,"nmap -PE --send-ip -p $ports,3001 ".join(' ',@nodes). " 2> /dev/null|") or die("Can't start nmap: $!");
|
||||
my $currnode='';
|
||||
my $port;
|
||||
@ -248,13 +519,21 @@ sub process_request_nmap {
|
||||
} elsif ($currnode) {
|
||||
if (/^MAC/) {
|
||||
my $status = join ',',sort keys %states ;
|
||||
my $appsd="";
|
||||
foreach my $portnum(keys %portservices) {
|
||||
my $app_t=$portservices{$portnum};
|
||||
if ($states{$app_t}) {$appsd .= $app_t . "=up,";}
|
||||
else {$appsd .= $app_t . "=down,";}
|
||||
}
|
||||
$appsd =~ s/,$//;
|
||||
|
||||
unless ($status or ($installquerypossible and $status = installer_query($currnode))) { #pingable, but no *clue* as to what the state may be
|
||||
push @nodesetnodes,$currnode; #Aggregate call to nodeset
|
||||
next;
|
||||
}
|
||||
$rsp{name}=[$currnode];
|
||||
$rsp{data} = [ $status ];
|
||||
$callback->({node=>[\%rsp]});
|
||||
$ret->{$currnode}->{'status'}=$::STATUS_ACTIVE;
|
||||
$ret->{$currnode}->{'appstatus'}=$status;
|
||||
$ret->{$currnode}->{'appsd'}=$appsd;
|
||||
$currnode="";
|
||||
%states=();
|
||||
next;
|
||||
@ -276,28 +555,27 @@ sub process_request_nmap {
|
||||
arg=>['stat']},
|
||||
\&getstat);
|
||||
foreach (@nodesetnodes) {
|
||||
$rsp{name}=[$_];
|
||||
$rsp{data} = [ "ping ".$nodesetstats{$_} ];
|
||||
$callback->({node=>[\%rsp]});
|
||||
$ret->{$_}->{'status'}=$nodesetstats{$_};
|
||||
}
|
||||
}
|
||||
foreach $currnode (sort keys %deadnodes) {
|
||||
$rsp{name}=[$currnode];
|
||||
$rsp{data} = [ 'noping' ];
|
||||
$callback->({node=>[\%rsp]});
|
||||
$ret->{$currnode}->{'status'}="noping";
|
||||
}
|
||||
|
||||
return $ret;
|
||||
}
|
||||
|
||||
sub process_request {
|
||||
%nodesetstats=();
|
||||
if ( -x '/usr/bin/nmap' ) {
|
||||
return process_request_nmap(@_);
|
||||
}
|
||||
|
||||
sub process_request_port {
|
||||
my $request = shift;
|
||||
my $callback = shift;
|
||||
my $doreq = shift;
|
||||
my $nodelist=shift;
|
||||
my $p_tmp=shift;
|
||||
my %portservices = %$p_tmp;
|
||||
my @nodes = ();
|
||||
if ($nodelist) { @nodes=@$nodelist;}
|
||||
|
||||
my @nodes = @{$request->{node}};
|
||||
my %unknownnodes;
|
||||
foreach (@nodes) {
|
||||
$unknownnodes{$_}=1;
|
||||
@ -311,28 +589,275 @@ sub process_request {
|
||||
}
|
||||
}
|
||||
|
||||
my $node;
|
||||
my $fping;
|
||||
open($fping,"fping ".join(' ',@nodes). " 2> /dev/null|") or die("Can't start fping: $!");
|
||||
while (<$fping>) {
|
||||
my %rsp;
|
||||
my $node=$_;
|
||||
$node =~ s/ .*//;
|
||||
chomp $node;
|
||||
if (/ is alive/) {
|
||||
$rsp{name}=[$node];
|
||||
$rsp{data} = [ interrogate_node($node,$doreq) ];
|
||||
$callback->({node=>[\%rsp]});
|
||||
} elsif (/is unreachable/) {
|
||||
$rsp{name}=[$node];
|
||||
$rsp{data} = [ 'noping' ];
|
||||
$callback->({node=>[\%rsp]});
|
||||
} elsif (/ address not found/) {
|
||||
$rsp{name}=[$node];
|
||||
$rsp{data} = [ 'nosuchhost' ];
|
||||
$callback->({node=>[\%rsp]});
|
||||
my $status={};
|
||||
|
||||
if (@nodes>0) {
|
||||
my $node;
|
||||
my $fping;
|
||||
open($fping,"fping ".join(' ',@nodes). " 2> /dev/null|") or die("Can't start fping: $!");
|
||||
while (<$fping>) {
|
||||
my %rsp;
|
||||
my $node=$_;
|
||||
$node =~ s/ .*//;
|
||||
chomp $node;
|
||||
if (/ is alive/) {
|
||||
$status->{$node} = interrogate_node($node,$doreq, $p_tmp);
|
||||
} elsif (/is unreachable/) {
|
||||
$status->{$node}->{'status'}="noping";
|
||||
} elsif (/ address not found/) {
|
||||
$status->{$node}->{'status'}="nosuchhost";
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return $status;
|
||||
}
|
||||
|
||||
|
||||
sub process_request_local_command {
|
||||
my $request = shift;
|
||||
my $callback = shift;
|
||||
my $doreq = shift;
|
||||
my $nodelist=shift;
|
||||
my $p_tmp=shift;
|
||||
my %cmdhash = %$p_tmp;
|
||||
|
||||
my @nodes = ();
|
||||
if ($nodelist) { @nodes=@$nodelist;}
|
||||
|
||||
my $status={};
|
||||
|
||||
if (@nodes>0) {
|
||||
foreach my $tmp_cmds (keys %cmdhash) {
|
||||
my @cmds=split(',', $tmp_cmds);
|
||||
my @apps=split(',', $cmdhash{$tmp_cmds});
|
||||
my $index=0;
|
||||
foreach my $cmd (@cmds) {
|
||||
my $nodes_string=join(',', @nodes);
|
||||
my $ret=`$cmd $nodes_string`;
|
||||
if (($? ==0) && ($ret)) {
|
||||
my @ret_array=split('\n', $ret);
|
||||
foreach(@ret_array) {
|
||||
my @a=split(':', $_);
|
||||
chomp($a[1]);
|
||||
if (exists($status->{$a[0]})) {
|
||||
$status->{$a[0]} .= "," . $apps[$index] . "=" . $a[1];
|
||||
} else {
|
||||
$status->{$a[0]} = $apps[$index] . "=" . $a[1];;
|
||||
}
|
||||
}
|
||||
}
|
||||
$index++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return $status;
|
||||
}
|
||||
|
||||
|
||||
sub process_request_remote_command {
|
||||
my $request = shift;
|
||||
my $callback = shift;
|
||||
my $doreq = shift;
|
||||
my $nodelist=shift;
|
||||
my $p_tmp=shift;
|
||||
my %cmdhash = %$p_tmp;
|
||||
my @nodes = ();
|
||||
if ($nodelist) { @nodes=@$nodelist;}
|
||||
|
||||
my $status={};
|
||||
|
||||
if (@nodes>0) {
|
||||
foreach my $tmp_cmds (keys %cmdhash) {
|
||||
my @cmds=split(',', $tmp_cmds);
|
||||
my @apps=split(',', $cmdhash{$tmp_cmds});
|
||||
my $index=0;
|
||||
foreach my $cmd (@cmds) {
|
||||
my $nodes_string=join(',', @nodes);
|
||||
#print "XCATBYPASS=Y xdsh $nodes_string $cmd\n";
|
||||
my $ret=`XCATBYPASS=Y xdsh $nodes_string $cmd`;
|
||||
if ($ret) {
|
||||
my @ret_array=split('\n', $ret);
|
||||
foreach(@ret_array) {
|
||||
my @a=split(':', $_, 2);
|
||||
chomp($a[1]); #remove newline
|
||||
$a[1] =~ s/^\s+//; #remove leading white spaces
|
||||
$a[1] =~ s/\s+$//; #remove tailing white spaces
|
||||
|
||||
if (exists($status->{$a[0]})) {
|
||||
$status->{$a[0]} .= "," . $apps[$index] . "=" . $a[1];
|
||||
} else {
|
||||
$status->{$a[0]} = $apps[$index] . "=" . $a[1];;
|
||||
}
|
||||
}
|
||||
}
|
||||
$index++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return $status;
|
||||
}
|
||||
|
||||
sub process_request {
|
||||
my $request = shift;
|
||||
my $callback = shift;
|
||||
my $doreq = shift;
|
||||
%nodesetstats=();
|
||||
|
||||
#print Dumper($request);
|
||||
my $update=$request->{'update'}->[0];
|
||||
my $quite=$request->{'quite'}->[0];
|
||||
my $all_apps=$request->{'allapps'};
|
||||
|
||||
#if ( -x '/usr/bin/nmap' ) {
|
||||
# my %portservices = (
|
||||
# '22' => 'sshd',
|
||||
# '15002' => 'pbs',
|
||||
# '8002' => 'xend',
|
||||
# );
|
||||
#
|
||||
# return process_request_nmap($request, $callback, $doreq, $request->{node}, \%portservices);
|
||||
# }
|
||||
|
||||
#handle ports and nodelist.status
|
||||
my $status={};
|
||||
if (exists($request->{'portapps'})) {
|
||||
for (my $i=1; $i<=$request->{'portapps'}->[0]; $i++) {
|
||||
my %portservices=();
|
||||
my @apps=split(',', $request->{"portapps$i"}->[0]);
|
||||
my @ports=split(',', $request->{"portapps$i" . "port"}->[0]);
|
||||
my $nodes=$request->{"portapps$i" . "node"};
|
||||
for (my $j=0; $j <@ports; $j++) {
|
||||
$portservices{$ports[$j]}=$apps[$j];
|
||||
}
|
||||
|
||||
my $ret={};
|
||||
if ( -x '/usr/bin/nmap' ) {
|
||||
$ret=process_request_nmap($request, $callback, $doreq, $nodes, \%portservices);
|
||||
} else {
|
||||
$ret=process_request_port($request, $callback, $doreq, $nodes, \%portservices);
|
||||
}
|
||||
%$status=(%$status, %$ret);
|
||||
}
|
||||
}
|
||||
|
||||
#print Dumper($status);
|
||||
|
||||
#handle local commands
|
||||
if (exists($request->{'cmdapps'})) {
|
||||
for (my $i=1; $i<=$request->{'cmdapps'}->[0]; $i++) {
|
||||
my %cmdhash=();
|
||||
my @apps=split(',', $request->{"cmdapps$i"}->[0]);
|
||||
my @cmds=split(',', $request->{"cmdapps$i" . "cmd"}->[0]);
|
||||
my $nodes=$request->{"cmdapps$i" . "node"};
|
||||
for (my $j=0; $j <@cmds; $j++) {
|
||||
$cmdhash{$cmds[$j]}=$apps[$j];
|
||||
}
|
||||
|
||||
my $ret = process_request_local_command($request, $callback, $doreq, $nodes, \%cmdhash);
|
||||
foreach my $node1 (keys(%$ret)) {
|
||||
if (exists($status->{$node1})) {
|
||||
my $appstatus=$status->{$node1}->{'appstatus'};
|
||||
if ($appstatus) { $status->{$node1}->{'appstatus'} .= "," . $ret->{$node1}; }
|
||||
else { $status->{$node1}->{'appstatus'} = $ret->{$node1}; }
|
||||
my $appsd=$status->{$node1}->{'appsd'};
|
||||
if ($appsd) { $status->{$node1}->{'appsd'} .= "," . $ret->{$node1}; }
|
||||
else { $status->{$node1}->{'appsd'} = $ret->{$node1}; }
|
||||
} else {
|
||||
$status->{$node1}->{'appstatus'} = $ret->{$node1};
|
||||
$status->{$node1}->{'appsd'} = $ret->{$node1};
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#handle remote commands
|
||||
if (exists($request->{'dcmdapps'})) {
|
||||
for (my $i=1; $i<=$request->{'dcmdapps'}->[0]; $i++) {
|
||||
my %dcmdhash=();
|
||||
my @apps=split(',', $request->{"dcmdapps$i"}->[0]);
|
||||
my @dcmds=split(',', $request->{"dcmdapps$i" . "dcmd"}->[0]);
|
||||
my $nodes=$request->{"dcmdapps$i" . "node"};
|
||||
for (my $j=0; $j <@dcmds; $j++) {
|
||||
$dcmdhash{$dcmds[$j]}=$apps[$j];
|
||||
}
|
||||
|
||||
my $ret = process_request_remote_command($request, $callback, $doreq, $nodes, \%dcmdhash);
|
||||
foreach my $node1 (keys(%$ret)) {
|
||||
if (exists($status->{$node1})) {
|
||||
my $appstatus=$status->{$node1}->{'appstatus'};
|
||||
if ($appstatus) { $status->{$node1}->{'appstatus'} .= "," . $ret->{$node1}; }
|
||||
else { $status->{$node1}->{'appstatus'} = $ret->{$node1}; }
|
||||
my $appsd=$status->{$node1}->{'appsd'};
|
||||
if ($appsd) { $status->{$node1}->{'appsd'} .= "," . $ret->{$node1}; }
|
||||
else { $status->{$node1}->{'appsd'} = $ret->{$node1}; }
|
||||
} else {
|
||||
$status->{$node1}->{'appstatus'} = $ret->{$node1};
|
||||
$status->{$node1}->{'appsd'} = $ret->{$node1};
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#show the output
|
||||
if (!$quite) {
|
||||
foreach my $node1 (sort keys(%$status)) {
|
||||
my %rsp;
|
||||
$rsp{name}=[$node1];
|
||||
my $st=$status->{$node1}->{'status'};
|
||||
my $ast= $status->{$node1}->{'appstatus'};
|
||||
if ($st) {
|
||||
$st = $ast ? "$st,$ast" : "$st";
|
||||
} else {
|
||||
$st=$ast;
|
||||
}
|
||||
$rsp{data}->[0] = $st;
|
||||
$callback->({node=>[\%rsp]});
|
||||
}
|
||||
}
|
||||
|
||||
#update the nodelist table
|
||||
if ($update) {
|
||||
my $nodetab=xCAT::Table->new('nodelist', -create=>1);
|
||||
if ($nodetab) {
|
||||
my $status1={};
|
||||
#get current values and compare with the new value to decide if update of db is necessary
|
||||
my @nodes1=keys(%$status);
|
||||
my $stuff = $nodetab->getNodesAttribs(\@nodes1, ['node', 'status', 'appstatus']);
|
||||
foreach my $node1 (@nodes1) {
|
||||
my $oldstatus=$stuff->{$node1}->[0]->{status};
|
||||
my $newstatus=$status->{$node1}->{status};
|
||||
if ($newstatus) {
|
||||
if ((!$oldstatus) || ($newstatus ne $oldstatus)) {
|
||||
$status1->{$node1}->{status}= $newstatus;
|
||||
}
|
||||
}
|
||||
else {
|
||||
if ($oldstatus) {
|
||||
$status1->{$node1}->{status}= "";
|
||||
}
|
||||
}
|
||||
|
||||
my $oldappstatus=$stuff->{$node1}->[0]->{appstatus};
|
||||
my $newappstatus=$status->{$node1}->{appsd};
|
||||
if ($newappstatus) {
|
||||
if ((!$oldappstatus) || ($newappstatus ne $oldappstatus)) {
|
||||
$status1->{$node1}->{appstatus}= $newappstatus;
|
||||
}
|
||||
}
|
||||
else {
|
||||
if ($oldappstatus) {
|
||||
$status1->{$node1}->{appstatus}= "";
|
||||
}
|
||||
}
|
||||
}
|
||||
#print Dumper($status1);
|
||||
$nodetab->setNodesAttribs($status1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sub usage
|
||||
@ -340,9 +865,106 @@ sub usage
|
||||
my $cb=shift;
|
||||
my $rsp={};
|
||||
$rsp->{data}->[0]= "Usage:";
|
||||
$rsp->{data}->[1]= " nodestat [noderange]";
|
||||
$rsp->{data}->[1]= " nodestat [noderange] [-m|--usemon] [-u|update]";
|
||||
$rsp->{data}->[2]= " nodestat [-h|--help|-v|--version]";
|
||||
xCAT::MsgUtils->message("I", $rsp, $cb);
|
||||
}
|
||||
|
||||
#--------------------------------------------------------------------------------
|
||||
=head3 getStatusMonsettings
|
||||
This function goes to the monsetting table to retrieve the settings related to
|
||||
the node status and app status monitoring.
|
||||
Arguments:
|
||||
none.
|
||||
Returns:
|
||||
a hash that has settings from the monsetting table for node status and
|
||||
app status monitoring. For example:
|
||||
( 'APPS'=>[ll,gpfs],
|
||||
'll' =>
|
||||
{
|
||||
'group' => 'service,compute',
|
||||
'port' => '5001'
|
||||
},
|
||||
'gpfs' =>
|
||||
{
|
||||
'group' => 'service',
|
||||
'cmd' => '/tmp/gpfscmd'
|
||||
};
|
||||
)
|
||||
=cut
|
||||
#--------------------------------------------------------------------------------
|
||||
sub getStatusMonsettings {
|
||||
my %apps=();
|
||||
my $tab=xCAT::Table->new('monsetting');
|
||||
if ( defined($tab)) {
|
||||
my ($ent) = $tab->getAttribs({name => 'xcatmon', key => 'apps' }, 'value');
|
||||
if ( defined($ent) ) {
|
||||
my $tmp_list=$ent->{value};
|
||||
if ($tmp_list) {
|
||||
my @applist=split(',', $tmp_list);
|
||||
foreach my $app (@applist) {
|
||||
$apps{$app}={};
|
||||
}
|
||||
$apps{'APPS'}=\@applist;
|
||||
my @results = $tab->getAttribs({name => 'xcatmon'}, 'key','value');
|
||||
if (@results) {
|
||||
foreach(@results) {
|
||||
my $key=$_->{key};
|
||||
my $value=$_->{value};
|
||||
if (exists($apps{$key})) {
|
||||
my @tem_value=split(',',$value);
|
||||
foreach my $pair (@tem_value) {
|
||||
my @tmp_action=split('=', $pair);
|
||||
if (exists($apps{$key}->{$tmp_action[0]})) {
|
||||
$apps{$key}->{$tmp_action[0]} = $apps{$key}->{$tmp_action[0]} . "," . $tmp_action[1];
|
||||
} else {
|
||||
$apps{$key}->{$tmp_action[0]}=$tmp_action[1];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return %apps;
|
||||
|
||||
}
|
||||
|
||||
#--------------------------------------------------------------------------------
|
||||
=head3 getNodeStatusAndAppstatus
|
||||
This function goes to the xCAT nodelist table to retrieve the saved node status and appstatus
|
||||
for all the node that are managed by local nodes.
|
||||
Arguments:
|
||||
nodelist--- an array of nodes
|
||||
Returns:
|
||||
a hash pointer that has the node status and appstatus. The format is:
|
||||
{ node1=> {
|
||||
status=>'active',appstatus=>'sshd=up,ll=up,gpfs=down'
|
||||
} ,
|
||||
node2=> {
|
||||
status=>'active',appstatus=>'sshd=up,ll=down,gpfs=down'
|
||||
}
|
||||
}
|
||||
|
||||
=cut
|
||||
#--------------------------------------------------------------------------------
|
||||
sub getMonNodesStatusAndAppStatus {
|
||||
my @nodes=@_;
|
||||
|
||||
my %status=();
|
||||
my $table=xCAT::Table->new("nodelist", -create =>1);
|
||||
my $tabdata=$table->getNodesAttribs(\@nodes,['node', 'status', 'appstatus']);
|
||||
foreach my $node (@nodes) {
|
||||
my $tmp1=$tabdata->{$node}->[0];
|
||||
if ($tmp1) {
|
||||
$status{$node}->{status}=$tmp1->{status};
|
||||
$status{$node}->{appstatus}=$tmp1->{appstatus};
|
||||
}
|
||||
}
|
||||
return %status;
|
||||
}
|
||||
|
||||
|
||||
|
||||
1;
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user