package xCAT_plugin::nodestat;

BEGIN
{
    $::XCATROOT = $ENV{'XCATROOT'} ? $ENV{'XCATROOT'} : '/opt/xcat';
}
use lib "$::XCATROOT/lib/perl";


use strict;
use warnings;



use Socket;
my $inet6support = eval {
    require Socket6;
    require IO::Socket::INET6;
    1;
};
use IO::Handle;
use Getopt::Long;
use Data::Dumper;
use xCAT::GlobalDef;
use xCAT::NetworkUtils;
require xCAT::Utils;
require xCAT::TableUtils;
require xCAT::ServiceNodeUtils;

my %nodesetstats;
my %chainhash;
my %default_ports = (
    'ftp'          => '21',
    'ssh'          => '22',
    'sshd'         => '22',
    'pbs'          => '15002',
    'pbs_mom'      => '15002',
    'xend'         => '8002',
    'll'           => '9616',
    'loadl'        => '9616',
    'loadl_master' => '9616',
    'loadleveler'  => '9616',
    'gpfs'         => '1191',
    'rdp'          => '3389',
    'msrpc'        => '135',
);

sub handled_commands {
    return {
        nodestat          => 'nodestat',
        nodestat_internal => 'nodestat',
    };
}

sub pinghost {
    my $node = shift;
    my $rc   = system("ping -q -n -c 1 -w 1 $node > /dev/null");
    if ($rc == 0) {
        return 1;
    } else {
        return 0;
    }
}

sub nodesockopen {
    my $node = shift;
    my $port = shift;
    my $socket;
    my $addr  = gethostbyname($node);
    my $sin   = sockaddr_in($port, $addr);
    my $proto = getprotobyname('tcp');
    socket($socket, PF_INET, SOCK_STREAM, $proto) || return 0;
    connect($socket, $sin) || return 0;
    return 1;
}

sub installer_query {
    my $node     = shift;
    my $destport = 3001;
    my $socket;
    my $text = "";
    if ($inet6support) {
        $socket = IO::Socket::INET6->new(PeerAddr => $node, PeerPort => 3001);
        unless ($socket) { return 0; }
    } else {
        my $proto = getprotobyname('tcp');
        socket($socket, PF_INET, SOCK_STREAM, $proto) || return 0;
        my $addr = gethostbyname($node);
        my $sin = sockaddr_in($destport, $addr);
        connect($socket, $sin) || return 0;
    }
    print $socket "stat \n";
    $socket->flush;
    while (<$socket>) {
        $text .= $_;
    }
    $text =~ s/\n.*//;
    return $text;
    close($socket);
}


sub getstat {
    my $response = shift;
    foreach (@{ $response->{node} }) {
        $nodesetstats{ $_->{name}->[0] } = $_->{data}->[0];
    }
}

#-------------------------------------------------------

=head3  preprocess_request

  Check and setup for hierarchy 

=cut

#-------------------------------------------------------
sub preprocess_request
{
    my $req = shift;
    my $cb  = shift;
    my %sn;
    if (defined $req->{_xcatpreprocessed}->[0] && $req->{_xcatpreprocessed}->[0] == 1) { return [$req]; }

    #exit if preprocessed

    my $command = $req->{command}->[0];
    if ($command eq "nodestat") {

        @ARGV = ();
        my $args = $req->{arg};
        if ($args) {
            @ARGV = @{$args};
        }

        # parse the options
        $::UPDATE = 0;
        $::QUITE  = 0;
        $::MON    = 0;
        $::POWER  = 0;

        #Getopt::Long::Configure("posix_default");
        #Getopt::Long::Configure("no_gnu_compat");
        Getopt::Long::Configure("bundling");
        $Getopt::Long::ignorecase = 0;
        if (!GetOptions(
                'm|usemon' => \$::MON,
                'q|quite' => \$::QUITE, #this is a internal flag used by monitoring
                'u|updatedb'  => \$::UPDATE,
                'p|powerstat' => \$::POWER,
                'h|help'      => \$::HELP,
                'v|version'   => \$::VERSION))
        {
            &usage($cb,1);
            return (1);
        }
        if ($::HELP) {
            &usage($cb);
            return (0);
        }
        if ($::VERSION) {
            my $version = xCAT::Utils->Version();
            my $rsp     = {};
            $rsp->{data}->[0] = "$version";
            xCAT::MsgUtils->message("I", $rsp, $cb);
            return (0);
        }
        my $nodes = $req->{node};
        if (!$nodes)
        {
            &usage($cb,1);
            return (1);
        }

        $req->{'update'}->[0] = $::UPDATE;
        $req->{'quite'}->[0]  = $::QUITE;
        $req->{'mon'}->[0]    = $::MON;
        $req->{'power'}->[0]  = $::POWER;
        return [$req];
    }

    #the following is for nodestat_internal command
    my $nodes   = $req->{node};
    my $service = "xcat";
    my @requests;
    if ($nodes) {
        my $usenmapfrommn = 0;
        if (-x '/usr/bin/nmap' or -x '/usr/local/bin/nmap') {

            #my $sitetab = xCAT::Table->new('site');
            #if ($sitetab) {
            #(my $ref) = $sitetab->getAttribs({key => 'useNmapfromMN'}, 'value');
            my @entries = xCAT::TableUtils->get_site_attribute("useNmapfromMN");
            my $t_entry = $entries[0];
            if (defined($t_entry)) {
                if ($t_entry =~ /1|yes|YES|Y|y/) { $usenmapfrommn = 1; }
            }

            #}
        }

        #get monsettings
        my %apps = ();
        my $mon  = $req->{'mon'}->[0];
        if ($mon == 1) { %apps = getStatusMonsettings(); }

        #if no apps specified in the monsetting table, add sshd, pbs and xend
        if (keys(%apps) == 0) {
            $apps{'sshd'}->{'group'} = "ALL"; #ALL means anything on the nodelist table, it is different from all
            $apps{'sshd'}->{'port'}  = "22";
            $apps{'https'}->{'group'} = "ALL"; #ALL means anything on the nodelist table, it is different from all
            $apps{'https'}->{'port'}  = "443";
            $apps{'pbs'}->{'group'}   = "ALL";
            $apps{'pbs'}->{'port'}    = "15002";
            $apps{'xend'}->{'group'}  = "ALL";
            $apps{'xend'}->{'port'}   = "8002";
            $apps{'rdp'}->{'group'}   = "ALL";
            $apps{'rdp'}->{'port'}    = "3389";
            $apps{'msrpc'}->{'group'} = "ALL";
            $apps{'msrpc'}->{'port'}  = "135";
            $apps{'APPS'} = [ 'sshd', 'https', 'pbs', 'xend' ];
        } else {

            #go thorugh the settings and put defaults in
            foreach my $app (keys(%apps)) {
                if ($app eq 'APPS') { next; }
                if (!exists($apps{$app}->{'group'})) { $apps{$app}->{'group'} = "ALL"; }
                if (exists($apps{$app}->{'cmd'}) || exists($apps{$app}->{'dcmd'}) || exists($apps{$app}->{'lcmd'})) { next; }
                if (exists($apps{$app}->{'port'})) { next; }

                #add port number in if nothing is specified
                if (exists($default_ports{$app})) { $apps{$app}->{'port'} = $default_ports{$app}; }
                else {
                    my $p = `grep "^$app" /etc/services`;
                    if ($? == 0) {
                        my @a_list = sort(split('\n', $p));
                        my @a_temp = split('/', $a_list[0]);
                        my @a      = split(' ', $a_temp[0]);
                        $apps{$app}->{'port'} = $a[1];
                    } else {
                        my $rsp = {};
                        $rsp->{data}->[0] = "Cannot find port number for application $app. Please either specify a port number or a command in monsetting table for $app.";
                        xCAT::MsgUtils->message("I", $rsp, $cb);
                        return (0);
                    }
                }
            }

            #always add sshd
            if (!exists($apps{'ssh'}) || !exists($apps{'sshd'})) {
                $apps{'sshd'}->{'group'} = "ALL";
                $apps{'sshd'}->{'port'}  = "22";
                my $pa = $apps{'APPS'};
                push @$pa, 'sshd';
            }
        }

        #print Dumper(%apps);
        # find service nodes for requested nodes
        # build an individual request for each service node
        my $sn = xCAT::ServiceNodeUtils->get_ServiceNode($nodes, $service, "MN");

        #get the member for each group
        my %groups = ();
        foreach my $app (keys %apps) {
            if ($app eq 'APPS') { next; }
            my $group = $apps{$app}->{'group'};
            if (($group) && ($group ne "ALL") && (!exists($groups{$group}))) {
                my @tmp_nodes = xCAT::NodeRange::noderange($group);
                foreach (@tmp_nodes) {
                    $groups{$group}->{$_} = 1;
                }
            }
        }

        # build each request for each service node
        my $all_apps = $apps{'APPS'};
        my %all_porthash = ();    #stores all the port apps if usenmapfrommn=1
        my %lcmdhash     = ();    #('myapp2,myapp3'=> {
                                  #       lcmd=>'/tmp/mycmd1,/usr/bin/date',
                                  #       node=>[node1,node2]
                                  #     }
                                  #)
        foreach my $snkey (keys %$sn)
        {
            my $reqcopy = {%$req};
            $reqcopy->{node}                   = $sn->{$snkey};
            $reqcopy->{'_xcatdest'}            = $snkey;
            $reqcopy->{_xcatpreprocessed}->[0] = 1;

            $reqcopy->{'useNmapfromMN'}->[0] = $usenmapfrommn;
            $reqcopy->{'allapps'} = $all_apps;

            my %porthash = ();    #('sshd,ll'=> {
                                  #       port=>'22,5001',
                                  #       node=>[node1,node2]
                                  #     }
                                  #)
            my %cmdhash  = ();    #('gpfs,myapp'=> {
                                  #       cmd=>'/tmp/mycmd1,/usr/bin/date',
                                  #       node=>[node1,node2]
                                  #     }
                                  #)
            my %dcmdhash = ();    #('myapp2,myapp3'=> {
                                  #       dcmd=>'/tmp/mycmd1,/usr/bin/date',
                                  #       node=>[node1,node2]
                                  #     }
                                  #)
            my @nodes_for_sn = @{ $sn->{$snkey} };

            foreach my $node (@nodes_for_sn) {
                my @ports;
                my @portapps;
                my @cmds;
                my @cmdapps;
                my @dcmds;
                my @dcmdapps;
                my @lcmdapps;
                my @lcmds;

                foreach my $app (keys %apps) {
                    if ($app eq 'APPS') { next; }
                    my $group = $apps{$app}->{'group'};
                    if (($group eq "ALL") || ($groups{$group}->{$node})) {

                        #print "app=$app\n";
                        if (exists($apps{$app}->{'port'})) {
                            push @ports,    $apps{$app}->{'port'};
                            push @portapps, $app;
                        }
                        elsif (exists($apps{$app}->{'cmd'})) {
                            push @cmds,    $apps{$app}->{'cmd'};
                            push @cmdapps, $app;
                        }
                        elsif (exists($apps{$app}->{'dcmd'})) {
                            push @dcmds,    $apps{$app}->{'dcmd'};
                            push @dcmdapps, $app;
                        }
                        elsif (exists($apps{$app}->{'lcmd'})) {
                            push @lcmds,    $apps{$app}->{'lcmd'};
                            push @lcmdapps, $app;
                        }
                    }
                }

                #print "ports=@ports\n";
                #print "portapps=@portapps\n";
                #print "cmds=@cmds\n";
                #print "cmdapps=@cmdapps\n";
                #print "dcmds=@dcmds\n";
                #print "dcmdapps=@dcmdapps\n";
                if (@portapps > 0) {
                    my $tmpapps = join(',', @portapps);
                    if (($usenmapfrommn == 1) && (@cmdapps == 0) && (@dcmdapps == 0) && (@lcmdapps == 0)) {

                        #this is the case where mn handles ports for all nodes using nmap
                        #The current limitation is that when there are cmd or dcmd specified for the node
                        # nmap has to be done on the service node because if both mn and sn update the appstatus
                        # one will overwites the other.
                        if (exists($all_porthash{$tmpapps})) {
                            my $pa = $all_porthash{$tmpapps}->{'node'};
                            push @$pa, $node;
                        } else {
                            $all_porthash{$tmpapps}->{'node'} = [$node];
                            $all_porthash{$tmpapps}->{'port'} = join(',', @ports);
                        }
                    } else {
                        if (exists($porthash{$tmpapps})) {
                            my $pa = $porthash{$tmpapps}->{'node'};
                            push @$pa, $node;
                        } else {
                            $porthash{$tmpapps}->{'node'} = [$node];
                            $porthash{$tmpapps}->{'port'} = join(',', @ports);
                        }
                    }
                }
                if (@cmdapps > 0) {
                    my $tmpapps = join(',', @cmdapps);
                    if (exists($cmdhash{$tmpapps})) {
                        my $pa = $cmdhash{$tmpapps}->{'node'};
                        push @$pa, $node;
                    } else {
                        $cmdhash{$tmpapps}->{'node'} = [$node];
                        $cmdhash{$tmpapps}->{'cmd'} = join(',', @cmds);
                    }
                }
                if (@dcmdapps > 0) {
                    my $tmpapps = join(',', @dcmdapps);
                    if (exists($dcmdhash{$tmpapps})) {
                        my $pa = $dcmdhash{$tmpapps}->{'node'};
                        push @$pa, $node;
                    } else {
                        $dcmdhash{$tmpapps}->{'node'} = [$node];
                        $dcmdhash{$tmpapps}->{'dcmd'} = join(',', @dcmds);
                    }
                }
                if (@lcmdapps > 0) {
                    my $i = 0;
                    foreach my $lapp (@lcmdapps) {
                        if (exists($lcmdhash{$lapp})) {
                            my $pa = $lcmdhash{$lapp}->{'node'};
                            push @$pa, $node;
                        } else {
                            $lcmdhash{$lapp}->{'node'} = [$node];
                            $lcmdhash{$lapp}->{'lcmd'} = $lcmds[$i];
                        }
                        $i++;
                    }
                }
            }    #end foreach (@nodes_for_sn)

            #print Dumper(%porthash);
            #print "cmdhash=" . Dumper(%cmdhash);
            #now push the settings into the requests
            my $i = 1;
            if ((keys(%porthash) == 0) && (keys(%cmdhash) == 0) && (keys(%dcmdhash) == 0) && (keys(%lcmdhash) == 0)) { next; }
            foreach my $tmpapps (keys %porthash) {
                $reqcopy->{'portapps'}->[0]   = scalar keys %porthash;
                $reqcopy->{"portapps$i"}->[0] = $tmpapps;
                $reqcopy->{ "portapps$i" . "port" }->[0] = $porthash{$tmpapps}->{'port'};
                $reqcopy->{ "portapps$i" . "node" } = $porthash{$tmpapps}->{'node'};
                $i++;
            }
            $i = 1;
            foreach my $tmpapps (keys %cmdhash) {
                $reqcopy->{'cmdapps'}->[0]   = scalar keys %cmdhash;
                $reqcopy->{"cmdapps$i"}->[0] = $tmpapps;
                $reqcopy->{ "cmdapps$i" . "cmd" }->[0] = $cmdhash{$tmpapps}->{'cmd'};
                $reqcopy->{ "cmdapps$i" . "node" } = $cmdhash{$tmpapps}->{'node'};
                $i++;
            }
            $i = 1;
            foreach my $tmpapps (keys %dcmdhash) {
                $reqcopy->{'dcmdapps'}->[0]   = scalar keys %dcmdhash;
                $reqcopy->{"dcmdapps$i"}->[0] = $tmpapps;
                $reqcopy->{ "dcmdapps$i" . "dcmd" }->[0] = $dcmdhash{$tmpapps}->{'dcmd'};
                $reqcopy->{ "dcmdapps$i" . "node" } = $dcmdhash{$tmpapps}->{'node'};
                $i++;
            }


            #done
            push @requests, $reqcopy;
        }    #enf sn_key


        #print "apps=" . Dumper(%apps);

        #mn handles all nmap when useNmapfromMN=1 on the site table
        if (($usenmapfrommn == 1) && (keys(%all_porthash) > 0)) {
            my @hostinfo = xCAT::NetworkUtils->determinehostname();
            my %iphash   = ();
            foreach (@hostinfo) { $iphash{$_} = 1; }
            my $handled = 0;
            foreach my $req (@requests) {
                my $currsn = $req->{'_xcatdest'};
                if (exists($iphash{$currsn})) {
                    my $i = 1;
                    foreach my $tmpapps (keys %all_porthash) {
                        $req->{'portapps'}->[0]   = scalar keys %all_porthash;
                        $req->{"portapps$i"}->[0] = $tmpapps;
                        $req->{ "portapps$i" . "port" }->[0] = $all_porthash{$tmpapps}->{'port'};
                        $req->{ "portapps$i" . "node" } = $all_porthash{$tmpapps}->{'node'};
                        $i++;
                    }
                    $handled = 1;
                    last;
                }
            }

            if (!$handled) {
                my $reqcopy = {%$req};
                $reqcopy->{_xcatpreprocessed}->[0] = 1;
                $reqcopy->{'useNmapfromMN'}->[0]   = $usenmapfrommn;
                $reqcopy->{'allapps'}              = $all_apps;
                my $i = 1;
                foreach my $tmpapps (keys %all_porthash) {
                    $reqcopy->{'portapps'}->[0]   = scalar keys %all_porthash;
                    $reqcopy->{"portapps$i"}->[0] = $tmpapps;
                    $reqcopy->{ "portapps$i" . "port" }->[0] = $all_porthash{$tmpapps}->{'port'};
                    $reqcopy->{ "portapps$i" . "node" } = $all_porthash{$tmpapps}->{'node'};
                    $i++;
                }
                push @requests, $reqcopy;
            }
        }


        #if ($usenmapfrommn) {
        #    my $reqcopy = {%$req};
        #    $reqcopy->{'update'}->[0]=$::UPDATE;
        #    $reqcopy->{'useNmapfromMN'}->[0]=1;
        #    if (!$::UPDATE) {
        #	push @requests, $reqcopy;
        #	return \@requests; #do not distribute, nodestat seems to lose accuracy and slow down distributed, if using nmap
        #    }
        #}

        #now handle local commands
        #print "lcmdhash=" . Dumper(%lcmdhash);
        if (keys(%lcmdhash) > 0) {
            my @hostinfo = xCAT::NetworkUtils->determinehostname();
            my %iphash   = ();
            foreach (@hostinfo) { $iphash{$_} = 1; }
            my $handled = 0;
            foreach my $req (@requests) {
                my $currsn = $req->{'_xcatdest'};
                if (exists($iphash{$currsn})) {
                    my $i = 1;
                    foreach my $lapp (keys %lcmdhash) {
                        $req->{'lcmdapps'}->[0]   = scalar keys %lcmdhash;
                        $req->{"lcmdapps$i"}->[0] = $lapp;
                        $req->{ "lcmdapps$i" . "cmd" }->[0] = $lcmdhash{$lapp}->{'lcmd'};
                        $req->{ "lcmdapps$i" . "node" } = $lcmdhash{$lapp}->{'node'};
                        $i++;
                    }
                    $handled = 1;
                    last;
                }
            }

            if (!$handled) {
                my $reqcopy = {%$req};
                $reqcopy->{_xcatpreprocessed}->[0] = 1;
                $reqcopy->{'allapps'} = $all_apps;
                my $i = 1;
                foreach my $lapp (keys %lcmdhash) {
                    $reqcopy->{'lcmdapps'}->[0]   = scalar keys %lcmdhash;
                    $reqcopy->{"lcmdapps$i"}->[0] = $lapp;
                    $reqcopy->{ "lcmdapps$i" . "cmd" }->[0] = $lcmdhash{$lapp}->{'lcmd'};
                    $reqcopy->{ "lcmdapps$i" . "node" } = $lcmdhash{$lapp}->{'node'};
                    $i++;
                }
                push @requests, $reqcopy;
            }
        }
    }

    return \@requests;
}

sub interrogate_node {    #Meant to run against confirmed up nodes
    my $node         = shift;
    my $doreq        = shift;
    my $p_tmp        = shift;
    my %portservices = %$p_tmp;

    my $status = "";
    my $appsd  = "";      #detailed status
    my $ret    = {};
    $ret->{'status'} = "ping";

    foreach my $port (keys(%portservices)) {
        if (nodesockopen($node, $port)) {
            $status .= $portservices{$port} . ",";
            $appsd  .= $portservices{$port} . "=up,";
        } else {
            $appsd .= $portservices{$port} . "=down,";
        }
    }

    $status =~ s/,$//;
    $appsd =~ s/,$//;
    $ret->{'appsd'} = $appsd;
    if ($status) {
        $ret->{'appstatus'} = $status;
        return $ret;
    }
    if ($status = installer_query($node)) {
        $ret->{'status'} = $status;
        return $ret;
    } else {    #pingable, but no *clue* as to what the state may be
        $doreq->({ command => ['nodeset'],
                node => [$node],
                arg  => ['stat'] },
            \&getstat);
        $ret->{'status'} = 'ping ' . $nodesetstats{$node};
        return $ret;
    }
}

sub process_request_nmap {
    my $request      = shift;
    my $callback     = shift;
    my $doreq        = shift;
    my $nodelist     = shift;
    my $p_tmp        = shift;
    my %portservices = %$p_tmp;
    my @nodes        = ();
    if ($nodelist) { @nodes = @$nodelist; }

    my %nodebyip;
    my @livenodes;
    my %unknownnodes;
    my $chaintab = xCAT::Table->new('chain', -create => 0);
    if ($chaintab) {
        %chainhash = %{ $chaintab->getNodesAttribs(\@nodes, ['currstate']) };
    }
    my $hoststab = xCAT::Table->new('hosts', -create => 0);
    my %hostsents;
    if ($hoststab) { %hostsents = %{ $hoststab->getNodesAttribs(\@nodes, ['ip']) }; }
    my @ips;
    my @ip6s;
    foreach (@nodes) {
        $unknownnodes{$_} = 1;
        my $ip = undef;
        if (($hostsents{$_}) && ($hostsents{$_}->[0]->{ip})) {
            $ip            = $hostsents{$_}->[0]->{ip};
            $nodebyip{$ip} = $_;
            $ip            = xCAT::NetworkUtils->getipaddr($ip);
            $nodebyip{$ip} = $_;
        } else {
            $ip = xCAT::NetworkUtils->getipaddr($_);
        }
        if (!defined $ip) {
            my %rsp;
            $rsp{name} = [$_];
            $rsp{data} = ["Please make sure $_ exists in /etc/hosts or DNS or hosts table"];
            $callback->({ node => [ \%rsp ] });
        } else {
            if ($ip =~ /:/) {
                push @ip6s, $ip;
            } else {
                push @ips, $ip;
            }
            $nodebyip{$ip} = $_;
        }
    }

    my $ret = {};
    my $node;
    my $fping;
    my $ports = join ',', keys %portservices;
    my %deadnodes;
    foreach (@nodes) {
        $deadnodes{$_} = 1;
    }

    #print "nmap -PE --send-ip -p $ports,3001 ".join(' ',@nodes) . "\n";
    # open($fping,"nmap -PE --send-ip -p $ports,3001 ".join(' ',@nodes). " 2> /dev/null|") or die("Can't start nmap: $!");
    my $currnode = '';
    my $port;
    my $state;
    my %states;
    my %rsp;
    my $installquerypossible = 0;
    my @nodesetnodes         = ();

    # get additional options from site table
    my @nmap_options = xCAT::TableUtils->get_site_attribute("nmapoptions");
    my $more_options = $nmap_options[0];

    foreach my $ip6 (0, 1) {    #first pass, ipv4, second pass ipv6
        if ($ip6 and scalar(@ip6s)) {
            open($fping, "nmap --unprivileged -6 -PS$ports,3001 -n --send-ip -p $ports,3001 $more_options " . join(' ', @ip6s) . " 2> /dev/null|") or die("Can't start nmap: $!");
        } elsif (not $ip6 and scalar(@ips)) {
            open($fping, "nmap --unprivileged -PE -n --send-ip -p $ports,3001 $more_options " . join(' ', @ips) . " 2> /dev/null|") or die("Can't start nmap: $!");
        } else { next; }
        while (<$fping>) {
            if (/Interesting ports on ([^ ]*)[: ]/ or /Nmap scan report for ([^ ]*)/) {
                my $tmpnode = $1;
                if ($currnode) { #if still thinking about last node, flush him out
                    my $status = join ',', sort keys %states;
                    my $appsd = "";
                    foreach my $portnum (keys %portservices) {
                        my $app_t = $portservices{$portnum};
                        if   ($states{$app_t}) { $appsd .= $app_t . "=up,"; }
                        else                   { $appsd .= $app_t . "=down,"; }
                    }
                    $appsd =~ s/,$//;
                    my $target = $currnode;
                    if ($hostsents{$target} and $hostsents{$target}->[0]->{ip}) { $target = $hostsents{$target}->[0]->{ip}; }

                    if ($status or ($installquerypossible and $status = installer_query($target))) { #pingable, but no *clue* as to what the state may be
                        $ret->{$currnode}->{'status'}    = "ping";
                        $ret->{$currnode}->{'appstatus'} = $status;
                        $ret->{$currnode}->{'appsd'}     = $appsd;
                        $currnode                        = "";
                        %states                          = ();
                    } else {
                        push @nodesetnodes, $currnode; #Aggregate call to nodeset
                    }
                }
                $currnode = $tmpnode;
                $currnode =~ s/:$//;
                $currnode =~ s/\n$//;


                my $nip;
                if ($nip = xCAT::NetworkUtils->getipaddr($currnode)) { #reverse lookup may not resemble the nodename, key by ip
                    if ($nodebyip{$nip}) {
                        $currnode = $nodebyip{$nip};
                    }
                }
                $installquerypossible = 0;    #reset possibility indicator
                %rsp                  = ();
                unless ($deadnodes{$currnode}) {
                    my $shortname;
                    foreach (keys %deadnodes) {
                        if (/\./) {
                            $shortname = $_;
                            $shortname =~ s/\..*//;
                        }
                        if ($currnode =~ /^$_\./ or ($shortname and $shortname eq $currnode)) {
                            $currnode = $_;
                            last;
                        }
                    }
                }
                delete $deadnodes{$currnode};
            } elsif ($currnode) {

                #if (/^MAC/) {  #oops not all nmap records end with MAC
                if (/^PORT/) { next; }
                ($port, $state) = split;
                if ($port and $port =~ /^(\d*)\// and $state eq 'open') {
                    if ($1 eq "3001" and defined($chainhash{$currnode}->[0]->{currstate}) and $chainhash{$currnode}->[0]->{currstate} =~ /^install/) {
                        $installquerypossible = 1; #It is possible to actually query node
                    } elsif ($1 ne "3001") {
                        $states{ $portservices{$1} } = 1;
                    }
                }
            }
        }
    }

    if ($currnode) {
        my $status = join ',', sort keys %states;
        my $appsd = "";
        foreach my $portnum (keys %portservices) {
            my $app_t = $portservices{$portnum};
            if   ($states{$app_t}) { $appsd .= $app_t . "=up,"; }
            else                   { $appsd .= $app_t . "=down,"; }
        }
        $appsd =~ s/,$//;

        my $target = $currnode;
        if ($hostsents{$target} and $hostsents{$target}->[0]->{ip}) { $target = $hostsents{$target}->[0]->{ip}; }
        if ($status or ($installquerypossible and $status = installer_query($target))) { #pingable, but no *clue* as to what the state may be
            $ret->{$currnode}->{'status'}    = "ping";
            $ret->{$currnode}->{'appstatus'} = $status;
            $ret->{$currnode}->{'appsd'}     = $appsd;
            $currnode                        = "";
            %states                          = ();
        } else {
            push @nodesetnodes, $currnode;    #Aggregate call to nodeset
        }
    }

    if (@nodesetnodes) {
        $doreq->({ command => ['nodeset'],
                node => \@nodesetnodes,
                arg  => ['stat'] },
            \&getstat);
        foreach (@nodesetnodes) {
            $ret->{$_}->{'status'} = $nodesetstats{$_};
        }
    }
    foreach $currnode (sort keys %deadnodes) {
        $ret->{$currnode}->{'status'} = "noping";
    }

    return $ret;
}


sub process_request_port {
    my $request      = shift;
    my $callback     = shift;
    my $doreq        = shift;
    my $nodelist     = shift;
    my $p_tmp        = shift;
    my %portservices = %$p_tmp;
    my @nodes        = ();
    if ($nodelist) { @nodes = @$nodelist; }

    my %unknownnodes;
    foreach (@nodes) {
        $unknownnodes{$_} = 1;
        my $packed_ip = undef;
        $packed_ip = xCAT::NetworkUtils->getipaddr($_);
        if (!defined $packed_ip) {
            my %rsp;
            $rsp{name} = [$_];
            $rsp{data} = ["Please make sure $_ exists in /etc/hosts"];
            $callback->({ node => [ \%rsp ] });
        }
    }

    my $status = {};

    if (@nodes > 0) {
        my $node;
        my $fping;
        open($fping, "fping " . join(' ', @nodes) . " 2> /dev/null|") or die("Can't start fping: $!");
        while (<$fping>) {
            my %rsp;
            my $node = $_;
            $node =~ s/ .*//;
            chomp $node;
            if (/ is alive/) {
                $status->{$node} = interrogate_node($node, $doreq, $p_tmp);
            } elsif (/is unreachable/) {
                $status->{$node}->{'status'} = "noping";
            } elsif (/ address not found/) {
                $status->{$node}->{'status'} = "nosuchhost";
            }
        }
    }

    return $status;
}


sub process_request_local_command {
    my $request  = shift;
    my $callback = shift;
    my $doreq    = shift;
    my $nodelist = shift;
    my $p_tmp    = shift;
    my %cmdhash  = %$p_tmp;

    my @nodes = ();
    if ($nodelist) { @nodes = @$nodelist; }

    my $status = {};

    if (@nodes > 0) {
        foreach my $tmp_cmds (keys %cmdhash) {
            my @cmds = split(',', $tmp_cmds);
            my @apps = split(',', $cmdhash{$tmp_cmds});
            my $index = 0;
            foreach my $cmd (@cmds) {
                my $nodes_string = join(',', @nodes);
                my $ret = `$cmd $nodes_string`;

                #print "ret=$ret\n";
                if (($? == 0) && ($ret)) {
                    my @ret_array = split('\n', $ret);
                    foreach (@ret_array) {
                        my @a = split(':', $_);
                        chomp($a[1]);
                        if (exists($status->{ $a[0] })) {
                            $status->{ $a[0] } .= "," . $apps[$index] . "=" . $a[1];
                        } else {
                            $status->{ $a[0] } = $apps[$index] . "=" . $a[1];
                        }
                    }
                }
                $index++;
            }
        }
    }

    return $status;
}


sub process_request_remote_command {
    my $request  = shift;
    my $callback = shift;
    my $doreq    = shift;
    my $nodelist = shift;
    my $p_tmp    = shift;
    my %cmdhash  = %$p_tmp;
    my @nodes    = ();
    if ($nodelist) { @nodes = @$nodelist; }

    my $status = {};

    if (@nodes > 0) {
        foreach my $tmp_cmds (keys %cmdhash) {
            my @cmds = split(',', $tmp_cmds);
            my @apps = split(',', $cmdhash{$tmp_cmds});
            my $index = 0;
            foreach my $cmd (@cmds) {
                my @ret = xCAT::InstUtils->xcmd($callback, $doreq, "xdsh", \@nodes, $cmd, 1);
                if (@ret) {
                    foreach (@ret) {
                        my @a = split(':', $_, 2);
                        chomp($a[1]);         #remove newline
                        $a[1] =~ s/^\s+//;    #remove leading white spaces
                        $a[1] =~ s/\s+$//;    #remove tailing white spaces

                        if (exists($status->{ $a[0] })) {
                            $status->{ $a[0] } .= "," . $apps[$index] . "=" . $a[1];
                        } else {
                            $status->{ $a[0] } = $apps[$index] . "=" . $a[1];
                        }
                    }
                }
                $index++;
            }
        }
    }

    return $status;
}

sub process_request {
    my $request  = shift;
    my $callback = shift;
    my $doreq    = shift;
    %nodesetstats = ();
    my $command   = $request->{command}->[0];
    my $separator = "XXXXXYYYYYZZZZZ";
    my $usefping;
    if (ref $request->{arg}) {
        @ARGV = @{ $request->{arg} };
        GetOptions(
            'f' => \$usefping
        );
    }



    if ($command eq "nodestat_internal") {

        #if ( -x '/usr/bin/nmap' ) {
        #    my %portservices = (
        #	   '22' => 'sshd',
        #	   '15002' => 'pbs',
        #	   '8002' => 'xend',
        #	   );
        #
        #    return process_request_nmap($request, $callback, $doreq, $request->{node}, \%portservices);
        # }

        #handle ports and nodelist.status
        my $status = {};
        if (exists($request->{'portapps'})) {
            for (my $i = 1 ; $i <= $request->{'portapps'}->[0] ; $i++) {
                my %portservices = ();
                my @apps = split(',', $request->{"portapps$i"}->[0]);
                my @ports = split(',', $request->{ "portapps$i" . "port" }->[0]);
                my $nodes = $request->{ "portapps$i" . "node" };
                for (my $j = 0 ; $j < @ports ; $j++) {
                    $portservices{ $ports[$j] } = $apps[$j];
                }

                my $ret = {};
                if (not $usefping and -x '/usr/bin/nmap') {
                    $ret = process_request_nmap($request, $callback, $doreq, $nodes, \%portservices);
                } else {
                    $ret = process_request_port($request, $callback, $doreq, $nodes, \%portservices);
                }
                %$status = (%$status, %$ret);
            }
        }


        #handle local commands
        if (exists($request->{'cmdapps'})) {
            for (my $i = 1 ; $i <= $request->{'cmdapps'}->[0] ; $i++) {
                my %cmdhash = ();
                my @apps = split(',', $request->{"cmdapps$i"}->[0]);
                my @cmds  = split(',', $request->{ "cmdapps$i" . "cmd" }->[0]);
                my $nodes = $request->{ "cmdapps$i" . "node" };
                for (my $j = 0 ; $j < @cmds ; $j++) {
                    $cmdhash{ $cmds[$j] } = $apps[$j];
                }

                my $ret = process_request_local_command($request, $callback, $doreq, $nodes, \%cmdhash);

                #print Dumper($ret);

                foreach my $node1 (keys(%$ret)) {
                    if (exists($status->{$node1})) {
                        my $appstatus = $status->{$node1}->{'appstatus'};
                        if ($appstatus) { $status->{$node1}->{'appstatus'} .= "," . $ret->{$node1}; }
                        else { $status->{$node1}->{'appstatus'} = $ret->{$node1}; }
                        my $appsd = $status->{$node1}->{'appsd'};
                        if ($appsd) { $status->{$node1}->{'appsd'} .= "," . $ret->{$node1}; }
                        else { $status->{$node1}->{'appsd'} = $ret->{$node1}; }
                    } else {
                        $status->{$node1}->{'appstatus'} = $ret->{$node1};
                        $status->{$node1}->{'appsd'}     = $ret->{$node1};
                    }
                }
            }
        }

        #handle local l commands
        if (exists($request->{'lcmdapps'})) {
            for (my $i = 1 ; $i <= $request->{'lcmdapps'}->[0] ; $i++) {
                my %cmdhash = ();
                my @apps = split(',', $request->{"lcmdapps$i"}->[0]);
                my @cmds  = split(',', $request->{ "lcmdapps$i" . "cmd" }->[0]);
                my $nodes = $request->{ "lcmdapps$i" . "node" };
                for (my $j = 0 ; $j < @cmds ; $j++) {
                    $cmdhash{ $cmds[$j] } = $apps[$j];
                }

                my $ret = process_request_local_command($request, $callback, $doreq, $nodes, \%cmdhash);

                foreach my $node1 (keys(%$ret)) {
                    if (exists($status->{$node1})) {
                        my $appstatus = $status->{$node1}->{'appstatus'};
                        if ($appstatus) { $status->{$node1}->{'appstatus'} .= "," . $ret->{$node1}; }
                        else { $status->{$node1}->{'appstatus'} = $ret->{$node1}; }
                        my $appsd = $status->{$node1}->{'appsd'};
                        if ($appsd) { $status->{$node1}->{'appsd'} .= "," . $ret->{$node1}; }
                        else { $status->{$node1}->{'appsd'} = $ret->{$node1}; }
                    } else {
                        $status->{$node1}->{'appstatus'} = $ret->{$node1};
                        $status->{$node1}->{'appsd'}     = $ret->{$node1};
                    }

                }
            }
        }


        #handle remote commands
        if (exists($request->{'dcmdapps'})) {
            for (my $i = 1 ; $i <= $request->{'dcmdapps'}->[0] ; $i++) {
                my %dcmdhash = ();
                my @apps = split(',', $request->{"dcmdapps$i"}->[0]);
                my @dcmds = split(',', $request->{ "dcmdapps$i" . "dcmd" }->[0]);
                my $nodes = $request->{ "dcmdapps$i" . "node" };
                for (my $j = 0 ; $j < @dcmds ; $j++) {
                    $dcmdhash{ $dcmds[$j] } = $apps[$j];
                }

                my $ret = process_request_remote_command($request, $callback, $doreq, $nodes, \%dcmdhash);
                foreach my $node1 (keys(%$ret)) {
                    if (exists($status->{$node1})) {
                        my $appstatus = $status->{$node1}->{'appstatus'};
                        if ($appstatus) { $status->{$node1}->{'appstatus'} .= "," . $ret->{$node1}; }
                        else { $status->{$node1}->{'appstatus'} = $ret->{$node1}; }
                        my $appsd = $status->{$node1}->{'appsd'};
                        if ($appsd) { $status->{$node1}->{'appsd'} .= "," . $ret->{$node1}; }
                        else { $status->{$node1}->{'appsd'} = $ret->{$node1}; }
                    } else {
                        $status->{$node1}->{'appstatus'} = $ret->{$node1};
                        $status->{$node1}->{'appsd'}     = $ret->{$node1};
                    }
                }
            }
        }


        #nodestat_internal command the output, nodestat command will collect it
        foreach my $node1 (sort keys(%$status)) {
            my %rsp;
            $rsp{name} = [$node1];
            my $st    = $status->{$node1}->{'status'};
            my $ast   = $status->{$node1}->{'appstatus'};
            my $appsd = $status->{$node1}->{'appsd'};
            $st    = $st    ? $st    : '';
            $ast   = $ast   ? $ast   : '';
            $appsd = $appsd ? $appsd : '';

            $rsp{data}->[0] = "$st$separator$ast$separator$appsd";
            $callback->({ node => [ \%rsp ] });
        }
    } else {    #nodestat command
                #first collect the status from the nodes
        my $reqcopy = {%$request};
        $reqcopy->{command}->[0] = 'nodestat_internal';
        my $ret = xCAT::Utils->runxcmd($reqcopy, $doreq, 0, 1);

        #print Dumper($ret);
        my $status       = {};
        my @noping_nodes = ();
        my $power        = $request->{'power'}->[0];
        foreach my $tmpdata (@$ret) {
            if ($tmpdata =~ /([^:]+): (.*)$separator(.*)$separator(.*)/) {

                #print "node=$1, status=$2, appstatus=$3, appsd=$4\n";
                if ($status->{$1}->{'status'}) {
                    $status->{$1}->{'status'} = $status->{$1}->{'status'} . ",$2";
                } else {
                    $status->{$1}->{'status'} = $2;
                }
                if ($status->{$1}->{'appstatus'}) {
                    $status->{$1}->{'appstatus'} = $status->{$1}->{'appstatus'} . ",$3";
                } else {
                    $status->{$1}->{'appstatus'} = $3;
                }
                if ($status->{$1}->{'appsd'}) {
                    $status->{$1}->{'appsd'} = $status->{$1}->{'appsd'} . ",$4";
                } else {
                    $status->{$1}->{'appsd'} = $4;
                }
                if (($power) && ($2 eq "noping")) {
                    push(@noping_nodes, $1);
                }
            } else {
                my $rsp;
                $rsp->{data}->[0] = "$tmpdata";
                xCAT::MsgUtils->message("I", $rsp, $callback);
            }
        }

        #print Dumper($status);
        #get power status for noping nodes
        if (($power) && (@noping_nodes > 0)) {

            #print "noping_nodes=@noping_nodes\n";
            my $ret = xCAT::Utils->runxcmd(
                {
                    command => ['rpower'],
                    node    => \@noping_nodes,
                    arg     => ['stat']
                },
                $doreq, 0, 1);

            foreach my $tmpdata (@$ret) {
                if ($tmpdata =~ /([^:]+): (.*)/) {
                    $status->{$1}->{'status'} = "noping($2)";
                } else {
                    my $rsp;
                    $rsp->{data}->[0] = "$tmpdata";
                    xCAT::MsgUtils->message("I", $rsp, $callback);
                }
            }
        }

        #print Dumper($request);
        my $update = $request->{'update'}->[0];
        my $quite  = $request->{'quite'}->[0];


        #show the output
        if (!$quite) {
            foreach my $node1 (sort keys(%$status)) {
                my %rsp;
                $rsp{name} = [$node1];
                my $st  = $status->{$node1}->{'status'};
                my $ast = $status->{$node1}->{'appstatus'};
                if ($st) {
                    if   ($st eq 'ping') { $st = $ast ? "$ast"     : "$st"; }
                    else                 { $st = $ast ? "$st,$ast" : "$st"; }
                } else {
                    $st = $ast;
                }
                $rsp{data}->[0] = $st;
                $callback->({ node => [ \%rsp ] });
            }
        }

        #update the nodelist table
        if ($update) {
            my $nodetab = xCAT::Table->new('nodelist', -create => 1);
            if ($nodetab) {
                my $status1 = {};

                #get current values and compare with the new value to decide if update of db is necessary
                my @nodes1 = keys(%$status);
                my $stuff = $nodetab->getNodesAttribs(\@nodes1, [ 'node', 'status', 'appstatus' ]);

                #get current local time
                my (
                    $sec, $min, $hour, $mday, $mon,
                    $year, $wday, $yday, $isdst
                  )
                  = localtime(time);
                my $currtime = sprintf("%02d-%02d-%04d %02d:%02d:%02d",
                    $mon + 1, $mday, $year + 1900,
                    $hour, $min, $sec);

                foreach my $node1 (@nodes1) {
                    my $oldstatus = $stuff->{$node1}->[0]->{status};
                    my $newstatus = $status->{$node1}->{status};
                    if ($newstatus) {
                        if ((!$oldstatus) || ($newstatus ne $oldstatus)) {
                            $status1->{$node1}->{status}     = $newstatus;
                            $status1->{$node1}->{statustime} = $currtime;
                        }
                    }
                    else {
                        if ($oldstatus) {
                            $status1->{$node1}->{status}     = "";
                            $status1->{$node1}->{statustime} = "";
                        }
                    }

                    if ($newstatus =~ /noping/) {
                        $status1->{$node1}->{appstatus} = "";
                    } else {
                        my $oldappstatus = $stuff->{$node1}->[0]->{'appstatus'};
                        my $newappstatus = $status->{$node1}->{'appsd'};
                        while ($newappstatus =~ /(\w+)\=(\w+)/) {
                            my $tmp1 = $1;
                            my $tmp2 = $2;
                            if ($oldappstatus) {
                                if ($oldappstatus =~ /$tmp1\=/) {
                                    $oldappstatus =~ s/$tmp1\=\w+/$tmp1\=$tmp2/g;
                                } else {
                                    $oldappstatus = $oldappstatus . "\,$tmp1\=$tmp2";
                                }
                            } else {
                                $oldappstatus = "$tmp1\=$tmp2";
                            }
                            $newappstatus =~ s/(\w+)\=(\w+)//;
                        }
                        $status1->{$node1}->{appstatus} = $oldappstatus;
                    }
                    $status1->{$node1}->{appstatustime} = $currtime;
                }

                #print Dumper($status1);
                $nodetab->setNodesAttribs($status1);
            }
        }
    }
}

sub usage
{
    my $cb  = shift;
    my $retcode=shift;
    my $rsp = {};
    $rsp->{data}->[0] = "Usage:";
    $rsp->{data}->[1] = "  nodestat [noderange] [-m|--usemon] [-p|powerstat] [-u|--updatedb]";
    $rsp->{data}->[2] = "  nodestat [-h|--help|-v|--version]";
    if($retcode){
       $rsp->{errorcode}->[0]=$retcode;
    }
    xCAT::MsgUtils->message("I", $rsp, $cb);
}

#--------------------------------------------------------------------------------

=head3    getStatusMonsettings
      This function goes to the monsetting table to retrieve the settings related to
      the node status and app status monitoring.
    Arguments:
       none.
    Returns:
       a hash that has settings from the monsetting table for node status and 
       app status monitoring. For example:
       (  'APPS'=>[ll,gpfs],
          'll' =>
           {
              'group' => 'service,compute',
              'port' => '5001'
           },
        'gpfs' =>
           {
               'group' => 'service',
               'cmd' => '/tmp/gpfscmd'
           };
       )
=cut

#--------------------------------------------------------------------------------
sub getStatusMonsettings {
    my %apps = ();
    my $tab  = xCAT::Table->new('monsetting');
    if (defined($tab)) {
        my ($ent) = $tab->getAttribs({ name => 'xcatmon', key => 'apps' }, 'value');
        if (defined($ent)) {
            my $tmp_list = $ent->{value};
            if ($tmp_list) {
                my @applist = split(',', $tmp_list);
                foreach my $app (@applist) {
                    $apps{$app} = {};
                }
                $apps{'APPS'} = \@applist;
                my @results = $tab->getAttribs({ name => 'xcatmon' }, 'key', 'value');
                if (@results) {
                    foreach (@results) {
                        my $key   = $_->{key};
                        my $value = $_->{value};
                        if (exists($apps{$key})) {
                            my @tem_value = split(',', $value);
                            foreach my $pair (@tem_value) {
                                my @tmp_action = split('=', $pair);
                                if (exists($apps{$key}->{ $tmp_action[0] })) {
                                    $apps{$key}->{ $tmp_action[0] } = $apps{$key}->{ $tmp_action[0] } . "," . $tmp_action[1];
                                } else {
                                    $apps{$key}->{ $tmp_action[0] } = $tmp_action[1];
                                }
                            }
                        }
                    }
                }
            }
        }
    }
    return %apps;

}

#--------------------------------------------------------------------------------

=head3    getNodeStatusAndAppstatus
      This function goes to the xCAT nodelist table to retrieve the saved node status and appstatus
      for all the node that are managed by local nodes.
    Arguments:
       nodelist--- an array of nodes
    Returns:
       a hash pointer that has the node status and appstatus. The format is: 
          { node1=> {
                     status=>'active',appstatus=>'sshd=up,ll=up,gpfs=down'
                   } , 
            node2=> {
                     status=>'active',appstatus=>'sshd=up,ll=down,gpfs=down'
                   } 
           }
           
=cut

#--------------------------------------------------------------------------------
sub getMonNodesStatusAndAppStatus {
    my @nodes = @_;

    my %status = ();
    my $table = xCAT::Table->new("nodelist", -create => 1);
    my $tabdata = $table->getNodesAttribs(\@nodes, [ 'node', 'status', 'appstatus' ]);
    foreach my $node (@nodes) {
        my $tmp1 = $tabdata->{$node}->[0];
        if ($tmp1) {
            $status{$node}->{status}    = $tmp1->{status};
            $status{$node}->{appstatus} = $tmp1->{appstatus};
        }
    }
    return %status;
}



1;