From ee1eec6f95764a68887f3a6a00feb83587320dc1 Mon Sep 17 00:00:00 2001 From: linggao Date: Mon, 1 Feb 2010 16:18:49 +0000 Subject: [PATCH] appstatus support, xcatmon updates appstatus, nodestat command supporting updating the db git-svn-id: https://svn.code.sf.net/p/xcat/code/xcat-core/trunk@5100 8638fb3e-16cb-4fca-ae20-7b5d299a9bcd --- xCAT-server/lib/xcat/monitoring/xcatmon.pm | 66 +- xCAT-server/lib/xcat/plugins/nodestat.pm | 844 ++++++++++++++++++--- 2 files changed, 770 insertions(+), 140 deletions(-) diff --git a/xCAT-server/lib/xcat/monitoring/xcatmon.pm b/xCAT-server/lib/xcat/monitoring/xcatmon.pm index 3d80c2be3..da2484465 100644 --- a/xCAT-server/lib/xcat/monitoring/xcatmon.pm +++ b/xCAT-server/lib/xcat/monitoring/xcatmon.pm @@ -109,6 +109,8 @@ sub supportNodeStatusMon { sub startNodeStatusMon { print "xcatmon.startNodeStatusMon\n"; + if (! -e "/etc/xCATMN") { return (0, ""); } #only run the cron job on mn + my $noderef=shift; if ($noderef =~ /xCAT_monitoring::xcatmon/) { $noderef=shift; @@ -117,7 +119,8 @@ sub startNodeStatusMon my $callback=shift; #run the command first to update the status, - my $cmd="$::XCATROOT/sbin/xcatnodemon"; + #my $cmd="$::XCATROOT/sbin/xcatnodemon"; + my $cmd="$::XCATROOT/bin/nodestat all -m -u -q"; #$output=`$cmd 2>&1`; #if ($?) { # print "xcatmon: $output\n"; @@ -194,6 +197,8 @@ sub startNodeStatusMon #-------------------------------------------------------------------------------- sub stopNodeStatusMon { print "xcatmon.stopNodeStatusMon\n"; + if (! -e "/etc/xCATMN") { return (0, ""); } #only run the cron job on mn + my $noderef=shift; if ($noderef =~ /xCAT_monitoring::xcatmon/) { $noderef=shift; @@ -201,7 +206,8 @@ sub stopNodeStatusMon { my $scope=shift; my $callback=shift; - my $job="$::XCATROOT/sbin/xcatnodemon"; + #my $job="$::XCATROOT/sbin/xcatnodemon"; + my $job="$::XCATROOT/bin/nodestat all -m -u -q"; my ($code, $msg)=xCAT::Utils::remove_cron_job($job); my $localhostname=hostname(); if ($code==0) { @@ -291,49 +297,49 @@ sub getMonNodesStatus { return %status; } - my @mon_servers=keys(%$hierachy); my $isSV=xCAT::Utils->isServiceNode(); #on a service node or on ms, get the nodes that has local host as the server node - my $monnodes; my @hostinfo=xCAT::Utils->determinehostname(); my %iphash=(); foreach(@hostinfo) {$iphash{$_}=1;} #if this is mn, include the ones that has no service nodes if (!$isSV) { $iphash{'noservicenode'}=1;} - + my %processed=(); foreach(@mon_servers) { #service node come in pairs, the first one is the monserver adapter that facing the mn, # the second one is facing the cn. we use the first one here my @server_pair=split(':', $_); my $sv=$server_pair[0]; - if ($iphash{$sv}) { - $monnodes=$hierachy->{$_}; - } - - - foreach(@$monnodes) { - my $node=$_->[0]; - my $status=$_->[2]; - my $type=$_[1]; - if (!$status) { $status=$::STATUS_DEFINED;} #default + if (!$processed{$sv}) { $processed{$sv}=1;} + else { next; } - if ($status eq $::STATUS_ACTIVE) { push(@active_nodes, $node);} - elsif ($status eq $::STATUS_INACTIVE) { push(@inactive_nodes, $node);} - else { - my $need_active=0; - my $need_inactive=0; - if ($::NEXT_NODESTAT_VAL{$status}->{$::STATUS_ACTIVE}==1) { $need_active=1;} - if ($::NEXT_NODESTAT_VAL{$status}->{$::STATUS_INACTIVE}==1) { $need_inactive=1;} - if (($need_active==1) && ($need_inactive==0)) { push(@inactive_nodes, $node); } #put it into the inactive list so that the monitoring code can switch it to active. - elsif (($need_active==0) && ($need_inactive==1)) { push(@active_nodes, $node); } #put it into the active list so that the monitoring code can chane it to inactive. - elsif (($need_active==1) && ($need_inactive==1)) { push(@unknown_nodes, $node);} #unknow list so that the monitoring code can change it to active or inactive - else { - #if it is non-osi node, check it anyway - if ($type !~ /osi/) {push(@unknown_nodes, $node);} - } + if ($iphash{$sv}) { + my $monnodes=$hierachy->{$_}; + + foreach(@$monnodes) { + my $node=$_->[0]; + my $status=$_->[2]; + my $type=$_[1]; + if (!$status) { $status=$::STATUS_DEFINED;} #default + + if ($status eq $::STATUS_ACTIVE) { push(@active_nodes, $node);} + elsif ($status eq $::STATUS_INACTIVE) { push(@inactive_nodes, $node);} + else { + my $need_active=0; + my $need_inactive=0; + if ($::NEXT_NODESTAT_VAL{$status}->{$::STATUS_ACTIVE}==1) { $need_active=1;} + if ($::NEXT_NODESTAT_VAL{$status}->{$::STATUS_INACTIVE}==1) { $need_inactive=1;} + if (($need_active==1) && ($need_inactive==0)) { push(@inactive_nodes, $node); } #put it into the inactive list so that the monitoring code can switch it to active. + elsif (($need_active==0) && ($need_inactive==1)) { push(@active_nodes, $node); } #put it into the active list so that the monitoring code can chane it to inactive. + elsif (($need_active==1) && ($need_inactive==1)) { push(@unknown_nodes, $node);} #unknow list so that the monitoring code can change it to active or inactive + else { + #if it is non-osi node, check it anyway + if ($type !~ /osi/) {push(@unknown_nodes, $node);} + } + } } } } @@ -346,6 +352,8 @@ sub getMonNodesStatus { } + + #-------------------------------------------------------------------------------- =head3 setNodeStatusAttributes This function will update the status column of the nodelist table with the new node status. diff --git a/xCAT-server/lib/xcat/plugins/nodestat.pm b/xCAT-server/lib/xcat/plugins/nodestat.pm index 58e53a60f..d01d5766e 100644 --- a/xCAT-server/lib/xcat/plugins/nodestat.pm +++ b/xCAT-server/lib/xcat/plugins/nodestat.pm @@ -1,11 +1,28 @@ package xCAT_plugin::nodestat; +BEGIN +{ + $::XCATROOT = $ENV{'XCATROOT'} ? $ENV{'XCATROOT'} : '/opt/xcat'; +} +use lib "$::XCATROOT/lib/perl"; + + use strict; use warnings; + + use Socket; use IO::Handle; use Getopt::Long; +use Data::Dumper; +use xCAT::GlobalDef; + + my %nodesetstats; +my %default_ports = ( + 'ssh' => '22', + 'sshd' => '22' + ); sub handled_commands { return { @@ -81,99 +98,349 @@ sub preprocess_request ($req->{_xcatpreprocessed}->[0] == 1)) { return [$req]; } + + @ARGV=(); + my $args=$req->{arg}; + if ($args) { + @ARGV = @{$args}; + } + + # parse the options + $::UPDATE=0; + $::QUITE=0; + $::MON=0; + Getopt::Long::Configure("posix_default"); + Getopt::Long::Configure("no_gnu_compat"); + Getopt::Long::Configure("bundling"); + if (!GetOptions( + 'm|usemon' => \$::MON, + 'q|quite' => \$::QUITE, #this is a internal flag used by monitoring + 'u|update' => \$::UPDATE, + 'h|help' => \$::HELP, + 'v|version' => \$::VERSION)) + { + &usage($cb); + return(1); + } + if ($::HELP) { + &usage($cb); + return(0); + } + if ($::VERSION) { + my $version = xCAT::Utils->Version(); + my $rsp={}; + $rsp->{data}->[0] = "$version"; + xCAT::MsgUtils->message("I", $rsp, $cb); + return(0); + } + + my $nodes = $req->{node}; my $service = "xcat"; my @requests; - if ($nodes){ - if (-x '/usr/bin/nmap' or -x '/usr/local/bin/nmap') { - my $usenmapfrommn=0; - my $sitetab = xCAT::Table->new('site'); - if ($sitetab) { - (my $ref) = $sitetab->getAttribs({key => 'useNmapfromMN'}, 'value'); - if ($ref) { - if ($ref->{value} =~ /1|yes|YES|Y|y/) { $usenmapfrommn=1; } - } - } - if ($usenmapfrommn) { - return [$req]; #do not distribute, nodestat seems to lose accuracy and slow down distributed, if using nmap - } - } - # find service nodes for requested nodes - # build an individual request for each service node - my $sn = xCAT::Utils->get_ServiceNode($nodes, $service, "MN"); + if ($nodes) { + my $usenmapfrommn=0; + if (-x '/usr/bin/nmap' or -x '/usr/local/bin/nmap') { + my $sitetab = xCAT::Table->new('site'); + if ($sitetab) { + (my $ref) = $sitetab->getAttribs({key => 'useNmapfromMN'}, 'value'); + if ($ref) { + if ($ref->{value} =~ /1|yes|YES|Y|y/) { $usenmapfrommn=1; } + } + } + } - # build each request for each service node - foreach my $snkey (keys %$sn) - { - my $reqcopy = {%$req}; - $reqcopy->{node} = $sn->{$snkey}; + #get monsettings + my %apps = (); + if ($::MON == 1) { %apps=getStatusMonsettings(); } + + #if no apps specified in the monsetting table, add sshd, pbs and xend + if (keys(%apps) == 0) { + $apps{'sshd'}->{'group'} = "ALL"; #ALL means anything on the nodelist table, it is different from all + $apps{'sshd'}->{'port'} = "22"; + $apps{'pbs'}->{'group'} = "ALL"; + $apps{'pbs'}->{'port'} = "15002"; + $apps{'xend'}->{'group'} = "ALL"; + $apps{'xend'}->{'port'} = "8002"; + $apps{'APPS'}=['sshd', 'pbs', 'xend']; + } else { + #go thorugh the settings and put defaults in + foreach my $app (keys(%apps)) { + if ($app eq 'APPS') { next; } + if (!exists($apps{$app}->{'group'})) { $apps{$app}->{'group'} = "ALL"; } + if (exists($apps{$app}->{'cmd'}) || exists($apps{$app}->{'dcmd'})) { next; } + if (exists($apps{$app}->{'port'})) { next; } + #add port number in if nothing is specified + if (exists($default_ports{$app})) { $apps{$app}->{'port'} = $default_ports{$app}; } + else { + print "need to get the port number form /etc/services\n"; + } + } + + #always add sshd + if (!exists($apps{'ssh'}) || !exists($apps{'sshd'}) ) { + $apps{'sshd'}->{'group'} = "ALL"; + $apps{'sshd'}->{'port'} = "22"; + my $pa=$apps{'APPS'}; + push @$pa, 'sshd'; + } + } + + #print Dumper(%apps); + # find service nodes for requested nodes + # build an individual request for each service node + my $sn = xCAT::Utils->get_ServiceNode($nodes, $service, "MN"); + + #get the member for each group + my %groups=(); + foreach my $app (keys %apps) { + if ($app eq 'APPS') { next; } + my $group=$apps{$app}->{'group'}; + if (($group) && ($group ne "ALL") && (! exists($groups{$group}))) { + my @tmp_nodes = xCAT::NodeRange::noderange($group); + foreach (@tmp_nodes) { + $groups{$group}->{$_}=1; + } + } + } + + # build each request for each service node + my $all_apps=$apps{'APPS'}; + my %all_porthash=(); #stores all the port apps if usenmapfrommn=1 + foreach my $snkey (keys %$sn) + { + my $reqcopy = {%$req}; + $reqcopy->{node} = $sn->{$snkey}; $reqcopy->{'_xcatdest'} = $snkey; $reqcopy->{_xcatpreprocessed}->[0] = 1; - push @requests, $reqcopy; - } - } - else - { # non node options like -h - @ARGV=(); - my $args=$req->{arg}; - if ($args) { - @ARGV = @{$args}; - } else { - &usage($cb); - return(1); - } - # parse the options - Getopt::Long::Configure("posix_default"); - Getopt::Long::Configure("no_gnu_compat"); - Getopt::Long::Configure("bundling"); - if (!GetOptions( - 'h|help' => \$::HELP, - 'v|version' => \$::VERSION)) - { - &usage($cb); - return(1); - } - if ($::HELP) { - &usage($cb); - return(0); - } - if ($::VERSION) { - my $version = xCAT::Utils->Version(); - my $rsp={}; - $rsp->{data}->[0] = "$version"; - xCAT::MsgUtils->message("I", $rsp, $cb); - return(0); - } + + $reqcopy->{'update'}->[0]=$::UPDATE; + $reqcopy->{'quite'}->[0]=$::QUITE; + $reqcopy->{'useNmapfromMN'}->[0]=$usenmapfrommn; + $reqcopy->{'allapps'}=$all_apps; + + my %porthash=(); #('sshd,ll'=> { + # port=>'22,5001', + # node=>[node1,node2] + # } + #) + my %cmdhash=(); #('gpfs,myapp'=> { + # cmd=>'/tmp/mycmd1,/usr/bin/date', + # node=>[node1,node2] + # } + #) + my %dcmdhash=(); #('myapp2,myapp3'=> { + # dcmd=>'/tmp/mycmd1,/usr/bin/date', + # node=>[node1,node2] + # } + #) + my @nodes_for_sn=@{$sn->{$snkey}}; + foreach my $node (@nodes_for_sn) { + my @ports; + my @portapps; + my @cmds; + my @cmdapps; + my @dcmds; + my @dcmdapps; + foreach my $app (keys %apps) { + if ($app eq 'APPS') { next; } + my $group=$apps{$app}->{'group'}; + if (($group eq "ALL") || ($groups{$group}->{$node})) { + #print "app=$app\n"; + if (exists($apps{$app}->{'port'})) { + push @ports, $apps{$app}->{'port'}; + push @portapps, $app; + } + elsif (exists($apps{$app}->{'cmd'})) { + push @cmds, $apps{$app}->{'cmd'}; + push @cmdapps, $app; + } + elsif (exists($apps{$app}->{'dcmd'})) { + push @dcmds, $apps{$app}->{'dcmd'}; + push @dcmdapps, $app; + } + } + } + #print "ports=@ports\n"; + #print "portapps=@portapps\n"; + #print "cmds=@cmds\n"; + #print "cmdapps=@cmdapps\n"; + #print "dcmds=@dcmds\n"; + #print "dcmdapps=@dcmdapps\n"; + if (@portapps>0) { + my $tmpapps=join(',', @portapps); + if (($usenmapfrommn==1) && (@cmdapps==0) && (@dcmdapps==0)) { + #this is the case where mn handles ports for all nodes using nmap + #The current limitation is that when there are cmd or dcmd specified for the node + # nmap has to be done on the service node because if both mn and sn update the appstatus + # one will overwites the other. + if (exists($all_porthash{$tmpapps})) { + my $pa=$all_porthash{$tmpapps}->{'node'}; + push @$pa, $node; + } else { + $all_porthash{$tmpapps}->{'node'}=[$node]; + $all_porthash{$tmpapps}->{'port'}=join(',', @ports); + } + } else { + if (exists($porthash{$tmpapps})) { + my $pa=$porthash{$tmpapps}->{'node'}; + push @$pa, $node; + } else { + $porthash{$tmpapps}->{'node'}=[$node]; + $porthash{$tmpapps}->{'port'}=join(',', @ports); + } + } + } + if (@cmdapps>0) { + my $tmpapps=join(',', @cmdapps); + if (exists($cmdhash{$tmpapps})) { + my $pa=$cmdhash{$tmpapps}->{'node'}; + push @$pa, $node; + } else { + $cmdhash{$tmpapps}->{'node'}=[$node]; + $cmdhash{$tmpapps}->{'cmd'}=join(',', @cmds); + } + } + if (@dcmdapps>0) { + my $tmpapps=join(',', @dcmdapps); + if (exists($dcmdhash{$tmpapps})) { + my $pa=$dcmdhash{$tmpapps}->{'node'}; + push @$pa, $node; + } else { + $dcmdhash{$tmpapps}->{'node'}=[$node]; + $dcmdhash{$tmpapps}->{'dcmd'}=join(',', @dcmds); + } + } + } #end foreach (@nodes_for_sn) + + #print Dumper(%porthash); + #now push the settings into the requests + my $i=1; + if ((keys(%porthash) == 0) && (keys(%cmdhash) == 0) && (keys(%dcmdhash) == 0)) { next; } + foreach my $tmpapps (keys %porthash) { + $reqcopy->{'portapps'}->[0]= scalar keys %porthash; + $reqcopy->{"portapps$i"}->[0]= $tmpapps; + $reqcopy->{"portapps$i" . "port"}->[0]= $porthash{$tmpapps}->{'port'}; + $reqcopy->{"portapps$i" . "node"} = $porthash{$tmpapps}->{'node'};; + $i++; + } + $i=1; + foreach my $tmpapps (keys %cmdhash) { + $reqcopy->{'cmdapps'}->[0]= scalar keys %cmdhash; + $reqcopy->{"cmdapps$i"}->[0]= $tmpapps; + $reqcopy->{"cmdapps$i" . "cmd"}->[0]= $cmdhash{$tmpapps}->{'cmd'}; + $reqcopy->{"cmdapps$i" . "node"} = $cmdhash{$tmpapps}->{'node'};; + $i++; + } + $i=1; + foreach my $tmpapps (keys %dcmdhash) { + $reqcopy->{'dcmdapps'}->[0]= scalar keys %dcmdhash; + $reqcopy->{"dcmdapps$i"}->[0]= $tmpapps; + $reqcopy->{"dcmdapps$i" . "dcmd"}->[0]= $dcmdhash{$tmpapps}->{'dcmd'}; + $reqcopy->{"dcmdapps$i" . "node"} = $dcmdhash{$tmpapps}->{'node'}; + $i++; + } + + + #done + push @requests, $reqcopy; + } + + #mn handles all nmap when useNmapfromMN=1 on the site table + if (($usenmapfrommn == 1) && (keys(%all_porthash) > 0)) { + my @hostinfo=xCAT::Utils->determinehostname(); + my %iphash=(); + foreach(@hostinfo) {$iphash{$_}=1;} + my $handled=0; + foreach my $req (@requests) { + my $currsn=$req->{'_xcatdest'}; + if (exists($iphash{$currsn})) { + my $i=1; + foreach my $tmpapps (keys %all_porthash) { + $req->{'portapps'}->[0]= scalar keys %all_porthash; + $req->{"portapps$i"}->[0]= $tmpapps; + $req->{"portapps$i" . "port"}->[0]= $all_porthash{$tmpapps}->{'port'}; + $req->{"portapps$i" . "node"} = $all_porthash{$tmpapps}->{'node'};; + $i++; + } + $handled=1; + last; + } + } + + if (!$handled) { + my $reqcopy = {%$req}; + $reqcopy->{_xcatpreprocessed}->[0] = 1; + $reqcopy->{'update'}->[0]=$::UPDATE; + $reqcopy->{'quite'}->[0]=$::QUITE; + $reqcopy->{'useNmapfromMN'}->[0]=$usenmapfrommn; + $reqcopy->{'allapps'}=$all_apps; + my $i=1; + foreach my $tmpapps (keys %all_porthash) { + $reqcopy->{'portapps'}->[0]= scalar keys %all_porthash; + $reqcopy->{"portapps$i"}->[0]= $tmpapps; + $reqcopy->{"portapps$i" . "port"}->[0]= $all_porthash{$tmpapps}->{'port'}; + $reqcopy->{"portapps$i" . "node"} = $all_porthash{$tmpapps}->{'node'};; + $i++; + } + push @requests, $reqcopy; + } + } + + + #if ($usenmapfrommn) { + # my $reqcopy = {%$req}; + # $reqcopy->{'update'}->[0]=$::UPDATE; + # $reqcopy->{'useNmapfromMN'}->[0]=1; + # if (!$::UPDATE) { + # push @requests, $reqcopy; + # return \@requests; #do not distribute, nodestat seems to lose accuracy and slow down distributed, if using nmap + # } + #} + } + return \@requests; } sub interrogate_node { #Meant to run against confirmed up nodes my $node=shift; my $doreq=shift; + my $p_tmp=shift; + my %portservices = %$p_tmp; + my $status = ""; - if (nodesockopen($node,15002)) { - $status.="pbs," - } - if (nodesockopen($node,8002)) { - $status.="xend," - } - if (nodesockopen($node,22)) { - $status.="sshd," + my $appsd=""; #detailed status + my $ret={}; + $ret->{'status'}=$::STATUS_ACTIVE; + + foreach my $port (keys(%portservices)) { + if (nodesockopen($node,$port)) { + $status.=$portservices{$port} . ","; + $appsd.=$portservices{$port} . "=up,"; + } else { + $appsd.=$portservices{$port} . "=down,"; + } } + $status =~ s/,$//; + $appsd =~ s/,$//; + $ret->{'appsd'}=$appsd; if ($status) { - return $status; + $ret->{'appstatus'}=$status; + return $ret; } if ($status = installer_query($node)) { - return $status; + $ret->{'status'}=$status; + return $ret; } else { #pingable, but no *clue* as to what the state may be $doreq->({command=>['nodeset'], node=>[$node], arg=>['stat']}, \&getstat); - return 'ping '.$nodesetstats{$node}; + $ret->{'status'} = 'ping '.$nodesetstats{$node}; + return $ret; } } @@ -181,14 +448,16 @@ sub process_request_nmap { my $request = shift; my $callback = shift; my $doreq = shift; + my $nodelist=shift; + my $p_tmp=shift; + my %portservices = %$p_tmp; + my @nodes =(); + if ($nodelist) { @nodes=@$nodelist;} + + my $update=$request->{'update'}->[0]; + my %nodebyip; - my %portservices = ( - '22' => 'sshd', - '15002' => 'pbs', - '8002' => 'xend', - ); my @livenodes; - my @nodes = @{$request->{node}}; my %unknownnodes; foreach (@nodes) { $unknownnodes{$_}=1; @@ -204,6 +473,7 @@ sub process_request_nmap { } } + my $ret={}; my $node; my $fping; my $ports = join ',',keys %portservices; @@ -211,6 +481,7 @@ sub process_request_nmap { foreach (@nodes) { $deadnodes{$_}=1; } + #print "nmap -PE --send-ip -p $ports,3001 ".join(' ',@nodes) . "\n"; open($fping,"nmap -PE --send-ip -p $ports,3001 ".join(' ',@nodes). " 2> /dev/null|") or die("Can't start nmap: $!"); my $currnode=''; my $port; @@ -248,13 +519,21 @@ sub process_request_nmap { } elsif ($currnode) { if (/^MAC/) { my $status = join ',',sort keys %states ; + my $appsd=""; + foreach my $portnum(keys %portservices) { + my $app_t=$portservices{$portnum}; + if ($states{$app_t}) {$appsd .= $app_t . "=up,";} + else {$appsd .= $app_t . "=down,";} + } + $appsd =~ s/,$//; + unless ($status or ($installquerypossible and $status = installer_query($currnode))) { #pingable, but no *clue* as to what the state may be push @nodesetnodes,$currnode; #Aggregate call to nodeset next; } - $rsp{name}=[$currnode]; - $rsp{data} = [ $status ]; - $callback->({node=>[\%rsp]}); + $ret->{$currnode}->{'status'}=$::STATUS_ACTIVE; + $ret->{$currnode}->{'appstatus'}=$status; + $ret->{$currnode}->{'appsd'}=$appsd; $currnode=""; %states=(); next; @@ -276,28 +555,27 @@ sub process_request_nmap { arg=>['stat']}, \&getstat); foreach (@nodesetnodes) { - $rsp{name}=[$_]; - $rsp{data} = [ "ping ".$nodesetstats{$_} ]; - $callback->({node=>[\%rsp]}); + $ret->{$_}->{'status'}=$nodesetstats{$_}; } } foreach $currnode (sort keys %deadnodes) { - $rsp{name}=[$currnode]; - $rsp{data} = [ 'noping' ]; - $callback->({node=>[\%rsp]}); + $ret->{$currnode}->{'status'}="noping"; } + + return $ret; } -sub process_request { - %nodesetstats=(); - if ( -x '/usr/bin/nmap' ) { - return process_request_nmap(@_); - } + +sub process_request_port { my $request = shift; my $callback = shift; my $doreq = shift; + my $nodelist=shift; + my $p_tmp=shift; + my %portservices = %$p_tmp; + my @nodes = (); + if ($nodelist) { @nodes=@$nodelist;} - my @nodes = @{$request->{node}}; my %unknownnodes; foreach (@nodes) { $unknownnodes{$_}=1; @@ -311,28 +589,275 @@ sub process_request { } } - my $node; - my $fping; - open($fping,"fping ".join(' ',@nodes). " 2> /dev/null|") or die("Can't start fping: $!"); - while (<$fping>) { - my %rsp; - my $node=$_; - $node =~ s/ .*//; - chomp $node; - if (/ is alive/) { - $rsp{name}=[$node]; - $rsp{data} = [ interrogate_node($node,$doreq) ]; - $callback->({node=>[\%rsp]}); - } elsif (/is unreachable/) { - $rsp{name}=[$node]; - $rsp{data} = [ 'noping' ]; - $callback->({node=>[\%rsp]}); - } elsif (/ address not found/) { - $rsp{name}=[$node]; - $rsp{data} = [ 'nosuchhost' ]; - $callback->({node=>[\%rsp]}); + my $status={}; + + if (@nodes>0) { + my $node; + my $fping; + open($fping,"fping ".join(' ',@nodes). " 2> /dev/null|") or die("Can't start fping: $!"); + while (<$fping>) { + my %rsp; + my $node=$_; + $node =~ s/ .*//; + chomp $node; + if (/ is alive/) { + $status->{$node} = interrogate_node($node,$doreq, $p_tmp); + } elsif (/is unreachable/) { + $status->{$node}->{'status'}="noping"; + } elsif (/ address not found/) { + $status->{$node}->{'status'}="nosuchhost"; + } } - } + } + + return $status; +} + + +sub process_request_local_command { + my $request = shift; + my $callback = shift; + my $doreq = shift; + my $nodelist=shift; + my $p_tmp=shift; + my %cmdhash = %$p_tmp; + + my @nodes = (); + if ($nodelist) { @nodes=@$nodelist;} + + my $status={}; + + if (@nodes>0) { + foreach my $tmp_cmds (keys %cmdhash) { + my @cmds=split(',', $tmp_cmds); + my @apps=split(',', $cmdhash{$tmp_cmds}); + my $index=0; + foreach my $cmd (@cmds) { + my $nodes_string=join(',', @nodes); + my $ret=`$cmd $nodes_string`; + if (($? ==0) && ($ret)) { + my @ret_array=split('\n', $ret); + foreach(@ret_array) { + my @a=split(':', $_); + chomp($a[1]); + if (exists($status->{$a[0]})) { + $status->{$a[0]} .= "," . $apps[$index] . "=" . $a[1]; + } else { + $status->{$a[0]} = $apps[$index] . "=" . $a[1];; + } + } + } + $index++; + } + } + } + + return $status; +} + + +sub process_request_remote_command { + my $request = shift; + my $callback = shift; + my $doreq = shift; + my $nodelist=shift; + my $p_tmp=shift; + my %cmdhash = %$p_tmp; + my @nodes = (); + if ($nodelist) { @nodes=@$nodelist;} + + my $status={}; + + if (@nodes>0) { + foreach my $tmp_cmds (keys %cmdhash) { + my @cmds=split(',', $tmp_cmds); + my @apps=split(',', $cmdhash{$tmp_cmds}); + my $index=0; + foreach my $cmd (@cmds) { + my $nodes_string=join(',', @nodes); + #print "XCATBYPASS=Y xdsh $nodes_string $cmd\n"; + my $ret=`XCATBYPASS=Y xdsh $nodes_string $cmd`; + if ($ret) { + my @ret_array=split('\n', $ret); + foreach(@ret_array) { + my @a=split(':', $_, 2); + chomp($a[1]); #remove newline + $a[1] =~ s/^\s+//; #remove leading white spaces + $a[1] =~ s/\s+$//; #remove tailing white spaces + + if (exists($status->{$a[0]})) { + $status->{$a[0]} .= "," . $apps[$index] . "=" . $a[1]; + } else { + $status->{$a[0]} = $apps[$index] . "=" . $a[1];; + } + } + } + $index++; + } + } + } + + return $status; +} + +sub process_request { + my $request = shift; + my $callback = shift; + my $doreq = shift; + %nodesetstats=(); + + #print Dumper($request); + my $update=$request->{'update'}->[0]; + my $quite=$request->{'quite'}->[0]; + my $all_apps=$request->{'allapps'}; + + #if ( -x '/usr/bin/nmap' ) { + # my %portservices = ( + # '22' => 'sshd', + # '15002' => 'pbs', + # '8002' => 'xend', + # ); + # + # return process_request_nmap($request, $callback, $doreq, $request->{node}, \%portservices); + # } + + #handle ports and nodelist.status + my $status={}; + if (exists($request->{'portapps'})) { + for (my $i=1; $i<=$request->{'portapps'}->[0]; $i++) { + my %portservices=(); + my @apps=split(',', $request->{"portapps$i"}->[0]); + my @ports=split(',', $request->{"portapps$i" . "port"}->[0]); + my $nodes=$request->{"portapps$i" . "node"}; + for (my $j=0; $j <@ports; $j++) { + $portservices{$ports[$j]}=$apps[$j]; + } + + my $ret={}; + if ( -x '/usr/bin/nmap' ) { + $ret=process_request_nmap($request, $callback, $doreq, $nodes, \%portservices); + } else { + $ret=process_request_port($request, $callback, $doreq, $nodes, \%portservices); + } + %$status=(%$status, %$ret); + } + } + + #print Dumper($status); + + #handle local commands + if (exists($request->{'cmdapps'})) { + for (my $i=1; $i<=$request->{'cmdapps'}->[0]; $i++) { + my %cmdhash=(); + my @apps=split(',', $request->{"cmdapps$i"}->[0]); + my @cmds=split(',', $request->{"cmdapps$i" . "cmd"}->[0]); + my $nodes=$request->{"cmdapps$i" . "node"}; + for (my $j=0; $j <@cmds; $j++) { + $cmdhash{$cmds[$j]}=$apps[$j]; + } + + my $ret = process_request_local_command($request, $callback, $doreq, $nodes, \%cmdhash); + foreach my $node1 (keys(%$ret)) { + if (exists($status->{$node1})) { + my $appstatus=$status->{$node1}->{'appstatus'}; + if ($appstatus) { $status->{$node1}->{'appstatus'} .= "," . $ret->{$node1}; } + else { $status->{$node1}->{'appstatus'} = $ret->{$node1}; } + my $appsd=$status->{$node1}->{'appsd'}; + if ($appsd) { $status->{$node1}->{'appsd'} .= "," . $ret->{$node1}; } + else { $status->{$node1}->{'appsd'} = $ret->{$node1}; } + } else { + $status->{$node1}->{'appstatus'} = $ret->{$node1}; + $status->{$node1}->{'appsd'} = $ret->{$node1}; + } + + } + } + } + + #handle remote commands + if (exists($request->{'dcmdapps'})) { + for (my $i=1; $i<=$request->{'dcmdapps'}->[0]; $i++) { + my %dcmdhash=(); + my @apps=split(',', $request->{"dcmdapps$i"}->[0]); + my @dcmds=split(',', $request->{"dcmdapps$i" . "dcmd"}->[0]); + my $nodes=$request->{"dcmdapps$i" . "node"}; + for (my $j=0; $j <@dcmds; $j++) { + $dcmdhash{$dcmds[$j]}=$apps[$j]; + } + + my $ret = process_request_remote_command($request, $callback, $doreq, $nodes, \%dcmdhash); + foreach my $node1 (keys(%$ret)) { + if (exists($status->{$node1})) { + my $appstatus=$status->{$node1}->{'appstatus'}; + if ($appstatus) { $status->{$node1}->{'appstatus'} .= "," . $ret->{$node1}; } + else { $status->{$node1}->{'appstatus'} = $ret->{$node1}; } + my $appsd=$status->{$node1}->{'appsd'}; + if ($appsd) { $status->{$node1}->{'appsd'} .= "," . $ret->{$node1}; } + else { $status->{$node1}->{'appsd'} = $ret->{$node1}; } + } else { + $status->{$node1}->{'appstatus'} = $ret->{$node1}; + $status->{$node1}->{'appsd'} = $ret->{$node1}; + } + } + } + } + + #show the output + if (!$quite) { + foreach my $node1 (sort keys(%$status)) { + my %rsp; + $rsp{name}=[$node1]; + my $st=$status->{$node1}->{'status'}; + my $ast= $status->{$node1}->{'appstatus'}; + if ($st) { + $st = $ast ? "$st,$ast" : "$st"; + } else { + $st=$ast; + } + $rsp{data}->[0] = $st; + $callback->({node=>[\%rsp]}); + } + } + + #update the nodelist table + if ($update) { + my $nodetab=xCAT::Table->new('nodelist', -create=>1); + if ($nodetab) { + my $status1={}; + #get current values and compare with the new value to decide if update of db is necessary + my @nodes1=keys(%$status); + my $stuff = $nodetab->getNodesAttribs(\@nodes1, ['node', 'status', 'appstatus']); + foreach my $node1 (@nodes1) { + my $oldstatus=$stuff->{$node1}->[0]->{status}; + my $newstatus=$status->{$node1}->{status}; + if ($newstatus) { + if ((!$oldstatus) || ($newstatus ne $oldstatus)) { + $status1->{$node1}->{status}= $newstatus; + } + } + else { + if ($oldstatus) { + $status1->{$node1}->{status}= ""; + } + } + + my $oldappstatus=$stuff->{$node1}->[0]->{appstatus}; + my $newappstatus=$status->{$node1}->{appsd}; + if ($newappstatus) { + if ((!$oldappstatus) || ($newappstatus ne $oldappstatus)) { + $status1->{$node1}->{appstatus}= $newappstatus; + } + } + else { + if ($oldappstatus) { + $status1->{$node1}->{appstatus}= ""; + } + } + } + #print Dumper($status1); + $nodetab->setNodesAttribs($status1); + } + } } sub usage @@ -340,9 +865,106 @@ sub usage my $cb=shift; my $rsp={}; $rsp->{data}->[0]= "Usage:"; - $rsp->{data}->[1]= " nodestat [noderange]"; + $rsp->{data}->[1]= " nodestat [noderange] [-m|--usemon] [-u|update]"; $rsp->{data}->[2]= " nodestat [-h|--help|-v|--version]"; xCAT::MsgUtils->message("I", $rsp, $cb); } +#-------------------------------------------------------------------------------- +=head3 getStatusMonsettings + This function goes to the monsetting table to retrieve the settings related to + the node status and app status monitoring. + Arguments: + none. + Returns: + a hash that has settings from the monsetting table for node status and + app status monitoring. For example: + ( 'APPS'=>[ll,gpfs], + 'll' => + { + 'group' => 'service,compute', + 'port' => '5001' + }, + 'gpfs' => + { + 'group' => 'service', + 'cmd' => '/tmp/gpfscmd' + }; + ) +=cut +#-------------------------------------------------------------------------------- +sub getStatusMonsettings { + my %apps=(); + my $tab=xCAT::Table->new('monsetting'); + if ( defined($tab)) { + my ($ent) = $tab->getAttribs({name => 'xcatmon', key => 'apps' }, 'value'); + if ( defined($ent) ) { + my $tmp_list=$ent->{value}; + if ($tmp_list) { + my @applist=split(',', $tmp_list); + foreach my $app (@applist) { + $apps{$app}={}; + } + $apps{'APPS'}=\@applist; + my @results = $tab->getAttribs({name => 'xcatmon'}, 'key','value'); + if (@results) { + foreach(@results) { + my $key=$_->{key}; + my $value=$_->{value}; + if (exists($apps{$key})) { + my @tem_value=split(',',$value); + foreach my $pair (@tem_value) { + my @tmp_action=split('=', $pair); + if (exists($apps{$key}->{$tmp_action[0]})) { + $apps{$key}->{$tmp_action[0]} = $apps{$key}->{$tmp_action[0]} . "," . $tmp_action[1]; + } else { + $apps{$key}->{$tmp_action[0]}=$tmp_action[1]; + } + } + } + } + } + } + } + } + return %apps; + +} + +#-------------------------------------------------------------------------------- +=head3 getNodeStatusAndAppstatus + This function goes to the xCAT nodelist table to retrieve the saved node status and appstatus + for all the node that are managed by local nodes. + Arguments: + nodelist--- an array of nodes + Returns: + a hash pointer that has the node status and appstatus. The format is: + { node1=> { + status=>'active',appstatus=>'sshd=up,ll=up,gpfs=down' + } , + node2=> { + status=>'active',appstatus=>'sshd=up,ll=down,gpfs=down' + } + } + +=cut +#-------------------------------------------------------------------------------- +sub getMonNodesStatusAndAppStatus { + my @nodes=@_; + + my %status=(); + my $table=xCAT::Table->new("nodelist", -create =>1); + my $tabdata=$table->getNodesAttribs(\@nodes,['node', 'status', 'appstatus']); + foreach my $node (@nodes) { + my $tmp1=$tabdata->{$node}->[0]; + if ($tmp1) { + $status{$node}->{status}=$tmp1->{status}; + $status{$node}->{appstatus}=$tmp1->{appstatus}; + } + } + return %status; +} + + + 1;