xcat-core/xCAT-server/lib/xcat/plugins/nodestat.pm

package xCAT_plugin::nodestat;
BEGIN
{
  $::XCATROOT = $ENV{'XCATROOT'} ? $ENV{'XCATROOT'} : '/opt/xcat';
}
use lib "$::XCATROOT/lib/perl";


use strict;
use warnings;



use Socket;
use IO::Handle;
use Getopt::Long;
use Data::Dumper;
use xCAT::GlobalDef;


my %nodesetstats;
my %default_ports = (
    'ftp' => '21',
    'ssh' => '22',
    'sshd' => '22',
    'pbs' => '15002',
    'pbs_mom' => '15002',
    'xend' => '8002',
    'll' => '9616',
    'loadl' => '9616',
    'loadl_master' => '9616',
    'loadleveler' => '9616',
    'gpfs' => '1191'
    );

sub handled_commands {
   return { 
      nodestat => 'nodestat',
   };
}

sub pinghost {
   my $node = shift;
   my $rc = system("ping -q -n -c 1 -w 1 $node > /dev/null");
   if ($rc == 0) {
      return 1;
   } else {
      return 0;
   }
}

sub nodesockopen {
   my $node = shift;
   my $port = shift;
   my $socket;
   my $addr = gethostbyname($node);
   my $sin = sockaddr_in($port,$addr);
   my $proto = getprotobyname('tcp');
   socket($socket,PF_INET,SOCK_STREAM,$proto) || return 0;
   connect($socket,$sin) || return 0;
   return 1;
}

sub installer_query {
   my $node = shift;
   my $destport = 3001;
   my $socket;
   my $text = "";
   my $proto = getprotobyname('tcp');
   socket($socket,PF_INET,SOCK_STREAM,$proto) || return 0;
   my $addr = gethostbyname($node);
   my $sin = sockaddr_in($destport,$addr);
   connect($socket,$sin) || return 0;
   print $socket "stat \n";
   $socket->flush;
   while (<$socket>) { 
      $text.=$_;
   }
   $text =~ s/\n.*//;
   return $text;
   close($socket);
}


sub getstat {
   my $response = shift;
   foreach (@{$response->{node}}) {
        $nodesetstats{$_->{name}->[0]} = $_->{data}->[0];
   }
}

#-------------------------------------------------------

=head3  preprocess_request

  Check and setup for hierarchy 

=cut

#-------------------------------------------------------
sub preprocess_request
{
    my $req = shift;
    my $cb  = shift;
    my %sn;
    if ((defined($req->{_xcatpreprocessed})) &&
      ($req->{_xcatpreprocessed}->[0] == 1)) {
      return [$req];
    }

    @ARGV=();
    my $args=$req->{arg};  
    if ($args) {
	@ARGV = @{$args};
    } 

    # parse the options
    $::UPDATE=0;
    $::QUITE=0;
    $::MON=0;
    #Getopt::Long::Configure("posix_default");
    #Getopt::Long::Configure("no_gnu_compat");
    Getopt::Long::Configure("bundling");
    $Getopt::Long::ignorecase=0;
    if (!GetOptions(
             'm|usemon' => \$::MON,
	     'q|quite'   => \$::QUITE, #this is a internal flag used by monitoring
	     'u|updatedb'   => \$::UPDATE,
	     'h|help'     => \$::HELP,
	     'v|version'  => \$::VERSION))
    {
	&usage($cb);
	return(1);
    }
    if ($::HELP) {
	&usage($cb);
	return(0);
    } 
    if ($::VERSION) {
	my $version = xCAT::Utils->Version();
	my $rsp={};
	$rsp->{data}->[0] = "$version";
	xCAT::MsgUtils->message("I", $rsp, $cb);
	return(0);
    } 


    my $nodes    = $req->{node};
    my $service  = "xcat";
    my @requests;
    if ($nodes) { 
	my $usenmapfrommn=0;
	if (-x '/usr/bin/nmap' or -x '/usr/local/bin/nmap') {
	    my $sitetab = xCAT::Table->new('site');
	    if ($sitetab) {
		(my $ref) = $sitetab->getAttribs({key => 'useNmapfromMN'}, 'value');
		if ($ref) {
		    if ($ref->{value} =~ /1|yes|YES|Y|y/) { $usenmapfrommn=1; }
		}
	    }
	}

	#get monsettings
	my %apps = ();
        if ($::MON == 1) { %apps=getStatusMonsettings(); }
   
        #if no apps specified in the monsetting table, add sshd, pbs and xend
	if (keys(%apps) == 0) {
	    $apps{'sshd'}->{'group'} = "ALL";   #ALL means anything on the nodelist table, it is different from all
	    $apps{'sshd'}->{'port'} = "22"; 
 	    $apps{'pbs'}->{'group'} = "ALL"; 
	    $apps{'pbs'}->{'port'} = "15002"; 
	    $apps{'xend'}->{'group'} = "ALL"; 
	    $apps{'xend'}->{'port'} = "8002"; 
            $apps{'APPS'}=['sshd', 'pbs', 'xend'];
        } else {
	    #go thorugh the settings and put defaults in
	    foreach my $app (keys(%apps)) {
                if ($app eq 'APPS') { next; }
		if (!exists($apps{$app}->{'group'})) { $apps{$app}->{'group'} = "ALL"; }
		if (exists($apps{$app}->{'cmd'}) || exists($apps{$app}->{'dcmd'})) { next; }
		if (exists($apps{$app}->{'port'})) { next; }
		#add port number in if nothing is specified
		if (exists($default_ports{$app})) { $apps{$app}->{'port'} = $default_ports{$app}; }
		else {
		    my $p=`grep "^$app" /etc/services`;
		    if ($? == 0) {
			my @a_list=sort(split('\n', $p));
			my @a_temp=split('/',$a_list[0]);
			my @a=split(' ', $a_temp[0]);
			$apps{$app}->{'port'}=$a[1];
		    } else {
			my $rsp={};
			$rsp->{data}->[0]= "Cannot find port number for application $app. Please either specify a port number or a command in monsetting table for $app.";;
			xCAT::MsgUtils->message("I", $rsp, $cb);
			return (0);
		    }
		}
	    }

	    #always add sshd
	    if (!exists($apps{'ssh'}) || !exists($apps{'sshd'}) ) { 
		$apps{'sshd'}->{'group'} = "ALL"; 
		$apps{'sshd'}->{'port'} = "22"; 
                my $pa=$apps{'APPS'};
                push @$pa, 'sshd';
	    }
	}

	#print Dumper(%apps);
	# find service nodes for requested nodes
	# build an individual request for each service node
	my $sn = xCAT::Utils->get_ServiceNode($nodes, $service, "MN");

        #get the member for each group
        my %groups=();
	foreach my $app (keys %apps) {
	    if ($app eq 'APPS') { next; }
	    my $group=$apps{$app}->{'group'};
	    if (($group) && ($group ne "ALL") && (! exists($groups{$group}))) {
		my @tmp_nodes = xCAT::NodeRange::noderange($group);
		foreach (@tmp_nodes) {
		    $groups{$group}->{$_}=1;
		}
	    }
	}

	# build each request for each service node
        my $all_apps=$apps{'APPS'};
        my %all_porthash=(); #stores all the port apps if usenmapfrommn=1
	foreach my $snkey (keys %$sn)
	{
	    my $reqcopy = {%$req};
	    $reqcopy->{node} = $sn->{$snkey};
            $reqcopy->{'_xcatdest'} = $snkey;
            $reqcopy->{_xcatpreprocessed}->[0] = 1;


	    $reqcopy->{'update'}->[0]=$::UPDATE;
	    $reqcopy->{'quite'}->[0]=$::QUITE;
	    $reqcopy->{'useNmapfromMN'}->[0]=$usenmapfrommn;
            $reqcopy->{'allapps'}=$all_apps;

            my %porthash=(); #('sshd,ll'=> {
                             #       port=>'22,5001',
                             #       node=>[node1,node2]
                             #     }
                             #)
	    my %cmdhash=();  #('gpfs,myapp'=> {
                             #       cmd=>'/tmp/mycmd1,/usr/bin/date',
                             #       node=>[node1,node2]
                             #     }
                             #)
            my %dcmdhash=(); #('myapp2,myapp3'=> {
                             #       dcmd=>'/tmp/mycmd1,/usr/bin/date',
                             #       node=>[node1,node2]
                             #     }
                             #)
	    my @nodes_for_sn=@{$sn->{$snkey}};
            foreach my $node (@nodes_for_sn) {
		my @ports;
                my @portapps;
                my @cmds;
                my @cmdapps;
                my @dcmds;
                my @dcmdapps;
		foreach my $app (keys %apps) {
		    if ($app eq 'APPS') { next; }
		    my $group=$apps{$app}->{'group'};
		    if (($group eq "ALL") || ($groups{$group}->{$node})) {
                        #print "app=$app\n";
			if (exists($apps{$app}->{'port'})) {
			    push @ports, $apps{$app}->{'port'};
			    push @portapps, $app;
			}
			elsif (exists($apps{$app}->{'cmd'})) {
			    push @cmds, $apps{$app}->{'cmd'};
			    push @cmdapps, $app;
			}
			elsif (exists($apps{$app}->{'dcmd'})) {
			    push @dcmds, $apps{$app}->{'dcmd'};
			    push @dcmdapps, $app;
			}
		    }
		}
		#print "ports=@ports\n";
                #print "portapps=@portapps\n";
                #print "cmds=@cmds\n";
                #print "cmdapps=@cmdapps\n";
                #print "dcmds=@dcmds\n";
                #print "dcmdapps=@dcmdapps\n";
                if (@portapps>0) {
                    my $tmpapps=join(',', @portapps);
		    if (($usenmapfrommn==1) && (@cmdapps==0) && (@dcmdapps==0)) {
			#this is the case where mn handles ports for all nodes using nmap
                        #The current limitation is that when there are cmd or dcmd specified for the node
                        # nmap has to be done on the service node because if both mn and sn update the appstatus
                        # one will overwites the other. 
			if (exists($all_porthash{$tmpapps})) {
			    my $pa=$all_porthash{$tmpapps}->{'node'};
			    push @$pa, $node;
			} else {
			    $all_porthash{$tmpapps}->{'node'}=[$node];
			    $all_porthash{$tmpapps}->{'port'}=join(',', @ports);
			}
		    } else { 
			if (exists($porthash{$tmpapps})) {
			    my $pa=$porthash{$tmpapps}->{'node'};
			    push @$pa, $node;
			} else {
			    $porthash{$tmpapps}->{'node'}=[$node];
			    $porthash{$tmpapps}->{'port'}=join(',', @ports);
			}
		    }
		}
                if (@cmdapps>0) {
                    my $tmpapps=join(',', @cmdapps);
		    if (exists($cmdhash{$tmpapps})) {
			my $pa=$cmdhash{$tmpapps}->{'node'};
                        push @$pa, $node;
		    } else {
			$cmdhash{$tmpapps}->{'node'}=[$node];
                        $cmdhash{$tmpapps}->{'cmd'}=join(',', @cmds);
		    }
		}
                if (@dcmdapps>0) {
                    my $tmpapps=join(',', @dcmdapps);
		    if (exists($dcmdhash{$tmpapps})) {
			my $pa=$dcmdhash{$tmpapps}->{'node'};
                        push @$pa, $node;
		    } else {
			$dcmdhash{$tmpapps}->{'node'}=[$node];
                        $dcmdhash{$tmpapps}->{'dcmd'}=join(',', @dcmds);
		    }
		}
	    } #end foreach (@nodes_for_sn)

            #print Dumper(%porthash);
            #now push the settings into the requests
	    my $i=1;
            if ((keys(%porthash) == 0) && (keys(%cmdhash) == 0) && (keys(%dcmdhash) == 0)) { next; }
            foreach my $tmpapps (keys %porthash) {
		$reqcopy->{'portapps'}->[0]= scalar keys %porthash;
                $reqcopy->{"portapps$i"}->[0]= $tmpapps;
                $reqcopy->{"portapps$i" . "port"}->[0]= $porthash{$tmpapps}->{'port'};
                $reqcopy->{"portapps$i" . "node"} = $porthash{$tmpapps}->{'node'};;
                $i++;
	    }
            $i=1;
            foreach my $tmpapps (keys %cmdhash) {
		$reqcopy->{'cmdapps'}->[0]= scalar keys %cmdhash;
                $reqcopy->{"cmdapps$i"}->[0]= $tmpapps;
                $reqcopy->{"cmdapps$i" . "cmd"}->[0]= $cmdhash{$tmpapps}->{'cmd'};
                $reqcopy->{"cmdapps$i" . "node"} = $cmdhash{$tmpapps}->{'node'};;
                $i++;
	    }
            $i=1;
            foreach my $tmpapps (keys %dcmdhash) {
		$reqcopy->{'dcmdapps'}->[0]= scalar keys %dcmdhash;
                $reqcopy->{"dcmdapps$i"}->[0]= $tmpapps;
                $reqcopy->{"dcmdapps$i" . "dcmd"}->[0]= $dcmdhash{$tmpapps}->{'dcmd'};
                $reqcopy->{"dcmdapps$i" . "node"} = $dcmdhash{$tmpapps}->{'node'};
		$i++;
	    }


	    #done
	    push @requests, $reqcopy;
	}

        #mn handles all nmap when useNmapfromMN=1 on the site table
        if (($usenmapfrommn == 1) && (keys(%all_porthash) > 0)) {
	    my @hostinfo=xCAT::Utils->determinehostname();
	    my %iphash=();
	    foreach(@hostinfo) {$iphash{$_}=1;}
            my $handled=0;
            foreach my $req (@requests) {
		my $currsn=$req->{'_xcatdest'};
		if (exists($iphash{$currsn}))  {
		    my $i=1;
		    foreach my $tmpapps (keys %all_porthash) {
			$req->{'portapps'}->[0]= scalar keys %all_porthash;
			$req->{"portapps$i"}->[0]= $tmpapps;
			$req->{"portapps$i" . "port"}->[0]= $all_porthash{$tmpapps}->{'port'};
			$req->{"portapps$i" . "node"} = $all_porthash{$tmpapps}->{'node'};;
                        $i++;
		    }
                    $handled=1;
                    last;
		}
	    }

	    if (!$handled) {
		my $reqcopy = {%$req};
		$reqcopy->{_xcatpreprocessed}->[0] = 1;
		$reqcopy->{'update'}->[0]=$::UPDATE;
		$reqcopy->{'quite'}->[0]=$::QUITE;
		$reqcopy->{'useNmapfromMN'}->[0]=$usenmapfrommn;
		$reqcopy->{'allapps'}=$all_apps;
		my $i=1;
		foreach my $tmpapps (keys %all_porthash) {
		    $reqcopy->{'portapps'}->[0]= scalar keys %all_porthash;
		    $reqcopy->{"portapps$i"}->[0]= $tmpapps;
		    $reqcopy->{"portapps$i" . "port"}->[0]= $all_porthash{$tmpapps}->{'port'};
		    $reqcopy->{"portapps$i" . "node"} = $all_porthash{$tmpapps}->{'node'};;
                    $i++;
		}
		push @requests, $reqcopy;
	    }
	}

  
	#if ($usenmapfrommn) { 
	#    my $reqcopy = {%$req};
	#    $reqcopy->{'update'}->[0]=$::UPDATE;
	#    $reqcopy->{'useNmapfromMN'}->[0]=1;
	#    if (!$::UPDATE) {
	#	push @requests, $reqcopy;
	#	return \@requests; #do not distribute, nodestat seems to lose accuracy and slow down distributed, if using nmap
	#    }
	#}
	
    }

    return \@requests;
}

sub interrogate_node { #Meant to run against confirmed up nodes
    my $node=shift;
    my $doreq=shift;
    my $p_tmp=shift;
    my %portservices = %$p_tmp;

    my $status = "";
    my $appsd=""; #detailed status
    my $ret={};
    $ret->{'status'}="ping";

    foreach my $port (keys(%portservices)) {
	if (nodesockopen($node,$port)) {
	    $status.=$portservices{$port} . ",";
	    $appsd.=$portservices{$port} . "=up,";
	} else {
	    $appsd.=$portservices{$port} . "=down,";
	}
    }

    $status =~ s/,$//;
    $appsd =~ s/,$//;
    $ret->{'appsd'}=$appsd;
    if ($status) {
	$ret->{'appstatus'}=$status;
        return $ret;
    }
    if ($status = installer_query($node)) {
	$ret->{'status'}=$status;
        return  $ret;
    } else { #pingable, but no *clue* as to what the state may be
         $doreq->({command=>['nodeset'],
                  node=>[$node],
                  arg=>['stat']},
                  \&getstat);
         $ret->{'status'} =  'ping '.$nodesetstats{$node};
         return $ret;
     }
}

sub process_request_nmap {
   my $request = shift;
   my $callback = shift;
   my $doreq = shift;
   my $nodelist=shift;
   my $p_tmp=shift;
   my %portservices = %$p_tmp;
   my @nodes =();
   if ($nodelist) { @nodes=@$nodelist;}

   my $update=$request->{'update'}->[0];

   my %nodebyip;
   my @livenodes;
   my %unknownnodes;
   foreach (@nodes) {
	$unknownnodes{$_}=1;
	my $packed_ip = undef;
        $packed_ip = gethostbyname($_);
        if( !defined $packed_ip) {
                my %rsp;
                $rsp{name}=[$_];
                $rsp{data} = [ "Please make sure $_ exists in /etc/hosts or DNS" ];
                $callback->({node=>[\%rsp]});
        } else {
            $nodebyip{inet_ntoa($packed_ip)} = $_;
        }
   }

   my $ret={};
   my $node;
   my $fping;
   my $ports = join ',',keys %portservices;
   my %deadnodes;
   foreach (@nodes) {
       $deadnodes{$_}=1;
   }
   #print "nmap -PE --send-ip -p $ports,3001 ".join(' ',@nodes) . "\n";
   open($fping,"nmap -PE --send-ip -p $ports,3001 ".join(' ',@nodes). " 2> /dev/null|") or die("Can't start nmap: $!");
   my $currnode='';
   my $port;
   my $state;
   my %states;
   my %rsp;
   my $installquerypossible=0;
   my @nodesetnodes=();
   while (<$fping>) {
      if (/Interesting ports on ([^ ]*) /) {
          $currnode=$1;
          my $nip;
          if ($nip = gethostbyname($currnode)) {
              $nip = inet_ntoa($nip); #reverse lookup may not resemble the nodename, key by ip
              if ($nodebyip{$nip}) {
                 $currnode = $nodebyip{$nip};
              }
          }
          $installquerypossible=0; #reset possibility indicator
          %rsp=();
          unless ($deadnodes{$1}) {
              my $shortname;
              foreach (keys %deadnodes) {
                  if (/\./) {
                      $shortname = $_;
                      $shortname =~ s/\..*//;
                  }
                  if ($currnode =~ /^$_\./ or ($shortname and $shortname eq $currnode)) {
                      $currnode = $_;
                      last;
                  }
              }
          }
          delete $deadnodes{$currnode};
      } elsif ($currnode) {
          if (/^MAC/) {
              my $status = join ',',sort keys %states ;
              my $appsd="";
              foreach my $portnum(keys %portservices) {
                  my $app_t=$portservices{$portnum};
		  if ($states{$app_t}) {$appsd .= $app_t . "=up,";}
		  else {$appsd .= $app_t . "=down,";}
	      }
	      $appsd =~ s/,$//;

              unless ($status or ($installquerypossible and $status = installer_query($currnode))) { #pingable, but no *clue* as to what the state may be
                 push @nodesetnodes,$currnode; #Aggregate call to nodeset
                 next;
              }
              $ret->{$currnode}->{'status'}="ping";
              $ret->{$currnode}->{'appstatus'}=$status;
              $ret->{$currnode}->{'appsd'}=$appsd;
              $currnode="";
              %states=();
              next;
          }
          if (/^PORT/) { next; }
          ($port,$state) = split;
          if ($port =~ /^(\d*)\// and $state eq 'open') {
              if ($1 eq "3001") {
                $installquerypossible=1; #It is possible to actually query node
              } else {
                $states{$portservices{$1}}=1;
              }
          }
      } 
    }
    if (@nodesetnodes) {
        $doreq->({command=>['nodeset'],
                  node=>\@nodesetnodes,
                  arg=>['stat']},
                  \&getstat);
        foreach (@nodesetnodes) {
              $ret->{$_}->{'status'}=$nodesetstats{$_};
        }
    }
    foreach $currnode (sort keys %deadnodes) {
	$ret->{$currnode}->{'status'}="noping";
    }

   return $ret;
}


sub process_request_port {
   my $request = shift;
   my $callback = shift;
   my $doreq = shift;
   my $nodelist=shift;
   my $p_tmp=shift;
   my %portservices = %$p_tmp;
   my @nodes = ();
   if ($nodelist) { @nodes=@$nodelist;}

   my %unknownnodes;
   foreach (@nodes) {
	$unknownnodes{$_}=1;
	my $packed_ip = undef;
        $packed_ip = gethostbyname($_);
        if( !defined $packed_ip) {
                my %rsp;
                $rsp{name}=[$_];
                $rsp{data} = [ "Please make sure $_ exists in /etc/hosts" ];
                $callback->({node=>[\%rsp]});
        }
   }

   my $status={};

   if (@nodes>0) {
       my $node;
       my $fping;
       open($fping,"fping ".join(' ',@nodes). " 2> /dev/null|") or die("Can't start fping: $!");
       while (<$fping>) {
	   my %rsp;
	   my $node=$_;
	   $node =~ s/ .*//;
	   chomp $node;
	   if (/ is alive/) {
               $status->{$node} = interrogate_node($node,$doreq, $p_tmp);
	   } elsif (/is unreachable/) {
               $status->{$node}->{'status'}="noping";
	   } elsif (/ address not found/) {
	       $status->{$node}->{'status'}="nosuchhost";
	   }
       }
   }

   return $status;
}   


sub process_request_local_command {
   my $request = shift;
   my $callback = shift;
   my $doreq = shift;
   my $nodelist=shift;
   my $p_tmp=shift;
   my %cmdhash = %$p_tmp;

   my @nodes = ();
   if ($nodelist) { @nodes=@$nodelist;}

   my $status={};

   if (@nodes>0) {
       foreach my $tmp_cmds (keys %cmdhash) {
           my @cmds=split(',', $tmp_cmds);
           my @apps=split(',', $cmdhash{$tmp_cmds});
	   my $index=0;
           foreach my $cmd (@cmds) {
               my $nodes_string=join(',', @nodes);
	       my $ret=`$cmd $nodes_string`; 
	       if (($? ==0) && ($ret)) {
		   my @ret_array=split('\n', $ret);
                   foreach(@ret_array) {
                       my @a=split(':', $_);
                       chomp($a[1]);
		       if (exists($status->{$a[0]})) {
			   $status->{$a[0]} .= "," . $apps[$index] . "=" . $a[1];
		       } else {
			   $status->{$a[0]} = $apps[$index] . "=" . $a[1];;
		       }
		   }
	       }
               $index++;
	   }
       }
   }

   return $status;
}   


sub process_request_remote_command {
   my $request = shift;
   my $callback = shift;
   my $doreq = shift;
   my $nodelist=shift;
   my $p_tmp=shift;
   my %cmdhash = %$p_tmp;
   my @nodes = ();
   if ($nodelist) { @nodes=@$nodelist;}

   my $status={};

   if (@nodes>0) {
       foreach my $tmp_cmds (keys %cmdhash) {
           my @cmds=split(',', $tmp_cmds);
           my @apps=split(',', $cmdhash{$tmp_cmds});
	   my $index=0;
           foreach my $cmd (@cmds) {
               my $nodes_string=join(',', @nodes);
               #print "XCATBYPASS=Y xdsh $nodes_string $cmd\n";
	       my $ret=`XCATBYPASS=Y xdsh $nodes_string $cmd`; 
	       if ($ret) {
		   my @ret_array=split('\n', $ret);
                   foreach(@ret_array) {
                       my @a=split(':', $_, 2);
                       chomp($a[1]); #remove newline
		       $a[1] =~ s/^\s+//; #remove leading white spaces
		       $a[1] =~ s/\s+$//; #remove tailing white spaces

		       if (exists($status->{$a[0]})) {
			   $status->{$a[0]} .= "," . $apps[$index] . "=" . $a[1];
		       } else {
			   $status->{$a[0]} = $apps[$index] . "=" . $a[1];;
		       }
		   }
	       }
               $index++;
	   }
       }
   }

   return $status;
}   

sub process_request {
   my $request = shift;
   my $callback = shift;
   my $doreq = shift;
   %nodesetstats=();

   #print Dumper($request);
   my $update=$request->{'update'}->[0];
   my $quite=$request->{'quite'}->[0];
   my $all_apps=$request->{'allapps'};

   #if ( -x '/usr/bin/nmap' ) {
   #    my %portservices = (
   #	   '22' => 'sshd',
   #	   '15002' => 'pbs',
   #	   '8002' => 'xend',
   #	   );
   #
   #    return process_request_nmap($request, $callback, $doreq, $request->{node}, \%portservices);
   # }

   #handle ports and nodelist.status
   my $status={};
   if (exists($request->{'portapps'})) {
       for (my $i=1; $i<=$request->{'portapps'}->[0]; $i++) {
	   my %portservices=();
	   my @apps=split(',', $request->{"portapps$i"}->[0]);
           my @ports=split(',', $request->{"portapps$i" . "port"}->[0]);
           my $nodes=$request->{"portapps$i" . "node"};
	   for (my $j=0; $j <@ports; $j++) {
	       $portservices{$ports[$j]}=$apps[$j];
	   } 
           
           my $ret={};
	   if ( -x '/usr/bin/nmap' ) {
	       $ret=process_request_nmap($request, $callback, $doreq, $nodes, \%portservices);
	   }  else {
	       $ret=process_request_port($request, $callback, $doreq, $nodes, \%portservices);
           } 
           %$status=(%$status, %$ret);
       }
   }

   #print Dumper($status);

   #handle local commands
   if (exists($request->{'cmdapps'})) {
       for (my $i=1; $i<=$request->{'cmdapps'}->[0]; $i++) {
	   my %cmdhash=();
	   my @apps=split(',', $request->{"cmdapps$i"}->[0]);
           my @cmds=split(',', $request->{"cmdapps$i" . "cmd"}->[0]);
           my $nodes=$request->{"cmdapps$i" . "node"};
	   for (my $j=0; $j <@cmds; $j++) {
	       $cmdhash{$cmds[$j]}=$apps[$j];
	   } 
           
           my $ret = process_request_local_command($request, $callback, $doreq, $nodes, \%cmdhash);
           foreach my $node1 (keys(%$ret)) {
               if (exists($status->{$node1})) {
		   my $appstatus=$status->{$node1}->{'appstatus'};
                   if ($appstatus) { $status->{$node1}->{'appstatus'} .= "," . $ret->{$node1}; }
                   else { $status->{$node1}->{'appstatus'} = $ret->{$node1}; }
		   my $appsd=$status->{$node1}->{'appsd'};
                   if ($appsd) { $status->{$node1}->{'appsd'} .= "," . $ret->{$node1}; }
                   else { $status->{$node1}->{'appsd'} = $ret->{$node1}; }
	       } else {
		   $status->{$node1}->{'appstatus'} = $ret->{$node1};
		   $status->{$node1}->{'appsd'} = $ret->{$node1};
	       }

	   }    
       }
   }

   #handle remote commands
   if (exists($request->{'dcmdapps'})) {
       for (my $i=1; $i<=$request->{'dcmdapps'}->[0]; $i++) {
	   my %dcmdhash=();
	   my @apps=split(',', $request->{"dcmdapps$i"}->[0]);
           my @dcmds=split(',', $request->{"dcmdapps$i" . "dcmd"}->[0]);
           my $nodes=$request->{"dcmdapps$i" . "node"};
	   for (my $j=0; $j <@dcmds; $j++) {
	       $dcmdhash{$dcmds[$j]}=$apps[$j];
	   } 
           
           my $ret = process_request_remote_command($request, $callback, $doreq, $nodes, \%dcmdhash);
           foreach my $node1 (keys(%$ret)) {
               if (exists($status->{$node1})) {
		   my $appstatus=$status->{$node1}->{'appstatus'};
                   if ($appstatus) { $status->{$node1}->{'appstatus'} .= "," . $ret->{$node1}; }
                   else { $status->{$node1}->{'appstatus'} = $ret->{$node1}; }
		   my $appsd=$status->{$node1}->{'appsd'};
                   if ($appsd) { $status->{$node1}->{'appsd'} .= "," . $ret->{$node1}; }
                   else { $status->{$node1}->{'appsd'} = $ret->{$node1}; }
	       } else {
		   $status->{$node1}->{'appstatus'} = $ret->{$node1};
		   $status->{$node1}->{'appsd'} = $ret->{$node1};
	       }
	   }    
       }
   }

   #show the output 
   if (!$quite) {
       foreach my $node1 (sort keys(%$status)) {
	   my %rsp;
	   $rsp{name}=[$node1];
	   my $st=$status->{$node1}->{'status'};
	   my $ast= $status->{$node1}->{'appstatus'};
	   if ($st) {
	       $st = $ast ? "$st,$ast" : "$st";
	   } else {
	       $st=$ast;
	   }
	   $rsp{data}->[0] = $st;
	   $callback->({node=>[\%rsp]});
       }  
   }

   #update the nodelist table
   if ($update) {
       my $nodetab=xCAT::Table->new('nodelist', -create=>1);
       if ($nodetab) {
           my $status1={};
           #get current values and compare with the new value to decide if update of db is necessary
           my @nodes1=keys(%$status); 
	   my $stuff = $nodetab->getNodesAttribs(\@nodes1, ['node', 'status', 'appstatus']);
	   foreach my $node1 (@nodes1) {
	       my $oldstatus=$stuff->{$node1}->[0]->{status};
               my $newstatus=$status->{$node1}->{status};
               if ($newstatus) {
		  if ((!$oldstatus) || ($newstatus ne $oldstatus)) { 
		      $status1->{$node1}->{status}= $newstatus; 
		  }   
	       } 
	       else {
		   if ($oldstatus) {
		       $status1->{$node1}->{status}= "";
		   }
	       }

	       my $oldappstatus=$stuff->{$node1}->[0]->{appstatus};
               my $newappstatus=$status->{$node1}->{appsd};
               if ($newappstatus) {
		  if ((!$oldappstatus) || ($newappstatus ne $oldappstatus)) { 
		      $status1->{$node1}->{appstatus}= $newappstatus; 
		  }  
	       } 
	       else {
		   if ($oldappstatus) {
		       $status1->{$node1}->{appstatus}= "";
		   }
	       }
	   }  
           #print Dumper($status1);    
       	   $nodetab->setNodesAttribs($status1);
       }
   }
}

sub usage
{
    my $cb=shift;
    my $rsp={};
    $rsp->{data}->[0]= "Usage:";
    $rsp->{data}->[1]= "  nodestat [noderange] [-m|--usemon] [-u|--updatedb]";
    $rsp->{data}->[2]= "  nodestat [-h|--help|-v|--version]";
    xCAT::MsgUtils->message("I", $rsp, $cb);
}

#--------------------------------------------------------------------------------
=head3    getStatusMonsettings
      This function goes to the monsetting table to retrieve the settings related to
      the node status and app status monitoring.
    Arguments:
       none.
    Returns:
       a hash that has settings from the monsetting table for node status and 
       app status monitoring. For example:
       (  'APPS'=>[ll,gpfs],
          'll' =>
           {
              'group' => 'service,compute',
              'port' => '5001'
           },
        'gpfs' =>
           {
               'group' => 'service',
               'cmd' => '/tmp/gpfscmd'
           };
       )
=cut
#--------------------------------------------------------------------------------
sub getStatusMonsettings {
    my %apps=();
    my $tab=xCAT::Table->new('monsetting');
    if ( defined($tab)) {
	my ($ent) = $tab->getAttribs({name => 'xcatmon', key => 'apps' }, 'value');
	if ( defined($ent) ) {
	    my $tmp_list=$ent->{value};
	    if ($tmp_list) {
		my @applist=split(',', $tmp_list); 
		foreach my $app (@applist) {
		    $apps{$app}={};
		}
                $apps{'APPS'}=\@applist;
		my @results = $tab->getAttribs({name => 'xcatmon'}, 'key','value');
		if (@results) {
		    foreach(@results) {
			my $key=$_->{key};
			my $value=$_->{value};
			if (exists($apps{$key})) {
			    my @tem_value=split(',',$value);
			    foreach my $pair (@tem_value) {
				my @tmp_action=split('=', $pair);
				if (exists($apps{$key}->{$tmp_action[0]})) {
				    $apps{$key}->{$tmp_action[0]} = $apps{$key}->{$tmp_action[0]} . "," . $tmp_action[1];
				} else {
				    $apps{$key}->{$tmp_action[0]}=$tmp_action[1];
				}
			    }
			}
		    }
		}
	    }
	}
    }
    return %apps;

}

#--------------------------------------------------------------------------------
=head3    getNodeStatusAndAppstatus
      This function goes to the xCAT nodelist table to retrieve the saved node status and appstatus
      for all the node that are managed by local nodes.
    Arguments:
       nodelist--- an array of nodes
    Returns:
       a hash pointer that has the node status and appstatus. The format is: 
          { node1=> {
                     status=>'active',appstatus=>'sshd=up,ll=up,gpfs=down'
                   } , 
            node2=> {
                     status=>'active',appstatus=>'sshd=up,ll=down,gpfs=down'
                   } 
           }
           
=cut
#--------------------------------------------------------------------------------
sub getMonNodesStatusAndAppStatus {
    my @nodes=@_;

    my %status=();
    my $table=xCAT::Table->new("nodelist", -create =>1);
    my $tabdata=$table->getNodesAttribs(\@nodes,['node', 'status', 'appstatus']);
    foreach my $node (@nodes) {
	my $tmp1=$tabdata->{$node}->[0];
	if ($tmp1) {
	    $status{$node}->{status}=$tmp1->{status};
	    $status{$node}->{appstatus}=$tmp1->{appstatus};
	}
    }
    return %status;
}



1;