xcat-core/perl-xCAT/xCAT/PPCcfg.pm

# IBM(c) 2007 EPL license http://www.eclipse.org/legal/epl-v10.html

package xCAT::PPCcfg;
use strict;
use Getopt::Long;
use xCAT::PPCcli qw(SUCCESS EXPECT_ERROR RC_ERROR NR_ERROR);
use xCAT::PPCfsp;
use xCAT::Usage;
use Storable qw(freeze thaw);
use POSIX "WNOHANG";
use xCAT::NetworkUtils;
use xCAT::MsgUtils qw(verbose_message);

use LWP;
use HTTP::Cookies;
##########################################
# Globals
##########################################
my %rspconfig = (
    sshcfg   => \&sshcfg,
    frame    => \&frame,
    hostname => \&hostname
);

my %rsp_result;
my $start;
##########################################################################
# Parse the command line for options and operands
##########################################################################
sub parse_args {

    my $request = shift;
    my $command = $request->{command};
    my $args    = $request->{arg};
    my %opt     = ();
    my %cmds    = ();
    my @fsp     = (
        "memdecfg",
        "decfg",
        "procdecfg",
        "iocap",
        "time",
        "date",
        "autopower",
        "sysdump",
        "spdump",
        "network",
        "HMC_passwd",
        "admin_passwd",
        "general_passwd",
        "*_passwd",
        "hostname",
        "resetnet"
    );
    my @bpa = (
        "frame",
        "password",
        "newpassword",
        "HMC_passwd",
        "admin_passwd",
        "general_passwd",
        "*_passwd",
        "hostname",
        "resetnet"
    );
    my @ppc = (
        "sshcfg"
    );
    my %rsp = (
        cec   => \@fsp,
        frame => \@bpa,
        fsp   => \@fsp,
        bpa   => \@bpa,
        ivm   => \@ppc,
        hmc   => \@ppc
    );
    #############################################
    # Get support command list
    #############################################
    #my $typetab  = xCAT::Table->new( 'nodetype' );
    #my $nodes = $request->{node};
    #foreach (@$nodes) {
    #    if ( defined( $typetab )) {
    #        my ($ent) = $typetab->getAttribs({ node=>$_},'nodetype');
    #        if ( defined($ent) ) {
    #               $request->{hwtype} = $ent->{nodetype};
    #               last;
    #        }
    #
    #    }
    #
    #}

    my $nodes    = $request->{node};
    my $typehash = xCAT::DBobjUtils->getnodetype($nodes);
    foreach my $nn (@$nodes) {
        $request->{hwtype} = $$typehash{$nn};
        last if ($request->{hwtype});
    }

    my $supported = $rsp{ $request->{hwtype} };

    #############################################
    # Responds with usage statement
    #############################################
    local *usage = sub {
        my $usage_string = xCAT::Usage->getUsage($command);
        return ([ $_[0], $usage_string ]);
    };
    #############################################
    # Process command-line arguments
    #############################################
    if (!defined($args)) {
        return (usage("No command specified"));
    }
    #############################################
    # Checks case in GetOptions, allows opts
    # to be grouped (e.g. -vx), and terminates
    # at the first unrecognized option.
    #############################################
    @ARGV                     = @$args;
    $Getopt::Long::ignorecase = 0;
    Getopt::Long::Configure("bundling");
    $request->{method} = undef;

    if (!GetOptions(\%opt, qw(V|verbose resetnet))) {
        return (usage());
    }
    ####################################
    # Check for "-" with no option
    ####################################
    if (grep(/^-$/, @ARGV)) {
        return (usage("Missing option: -"));
    }
    ####################################
    # Check for "=" with no argument
    ####################################
    if (my ($c) = grep(/=$/, @ARGV)) {
        return (usage("Missing argument: $c"));
    }
    ####################################
    # Check for unsupported commands
    ####################################
    foreach my $arg (@ARGV) {
        my ($command, $value) = split(/=/, $arg);
        if (!grep(/^$command$/, @$supported) and !$opt{resetnet}) {
            return (usage("Invalid command: $arg"));
        }
        if (exists($cmds{$command})) {
            return (usage("Command multiple times: $command"));
        }
        $cmds{$command} = $value;
    }

    ####################################
    # Check command arguments
    ####################################
    foreach (keys %cmds) {
        if ($cmds{$_}) {
            my $result = parse_option($request, $_, $cmds{$_});
            if ($result) {
                return (usage($result));
            }
        } elsif ($_ =~ /_passwd$/) {
            return (usage("No argument specified for '$_'"));
        }
    }
    ####################################
    # Return method to invoke
    ####################################
    if ($request->{hwtype} =~ /(^hmc|ivm)$/) {
        $request->{method} = "cfg";
        return (\%opt);
    }
    ####################################
    # Return method to invoke
    ####################################
    if (exists($cmds{frame}) or exists($cmds{hostname})) {
        $request->{hcp}    = "hmc";
        $request->{method} = "cfg";
        return (\%opt);
    }
    ####################################
    # Return method to invoke
    ####################################
    if ($opt{resetnet}) {
        $request->{hcp}    = "hmc";
        $request->{method} = "resetnet";
        return (\%opt);
    }

    ####################################
    # Return method to invoke
    ####################################
    if (exists($cmds{HMC_passwd}) or exists($cmds{general_passwd}) or exists($cmds{admin_passwd}) or exists($cmds{"*_passwd"})) {
        $request->{hcp}    = "hmc";
        $request->{method} = "passwd";
        return (\%opt);
    }

    $request->{method} = \%cmds;
    return (\%opt);
}

##########################################################################
# Parse the command line optional arguments
##########################################################################
sub parse_option {

    my $request = shift;
    my $command = shift;
    my $value   = shift;

    ####################################
    # Set/get time
    ####################################
    if ($command =~ /^time$/) {
        if ($value !~
/^([0-1]?[0-9]|2[0-3]):(0?[0-9]|[1-5][0-9]):(0?[0-9]|[1-5][0-9])$/) {
            return ("Invalid time format '$value'");
        }
    }
    ####################################
    # Set/get date
    ####################################
    if ($command =~ /^date$/) {
        if ($value !~
            /^(0?[1-9]|1[012])-(0?[1-9]|[12][0-9]|3[01])-(20[0-9]{2})$/) {
            return ("Invalid date format '$value'");
        }
    }
    ####################################
    # Set/get options
    ####################################
    if ($command =~ /^(autopower|iocap|sshcfg)$/) {
        if ($value !~ /^(enable|disable)$/i) {
            return ("Invalid argument '$value'");
        }
    }
    ####################################
    # Deconfiguration policy
    ####################################
    if ($command =~ /^decfg$/) {
        if ($value !~ /^(enable|disable):.*$/i) {
            return ("Invalid argument '$value'");
        }
    }
    ####################################
    # Processor deconfiguration
    ####################################
    if ($command =~ /^procdecfg$/) {
        if ($value !~ /^(configure|deconfigure):\d+:(all|[\d,]+)$/i) {
            return ("Invalid argument '$value'");
        }
    }
    ################################
    # Memory deconfiguration
    ################################
    elsif ($command =~ /^memdecfg$/) {
        if ($value !~ /^(configure|deconfigure):\d+:(unit|bank):(all|[\d,]+)$/i) {
            return ("Invalid argument '$value'");
        }
    }
    if ($command eq 'network') {
        my ($adapter_name, $ip, $host, $gateway, $netmask) =
          split /,/, $value;
        return ("Network interface name is required") if (!$adapter_name);
        return ("Invalide network interface name $adapter_name") if ($adapter_name !~ /^eth\d$/);
        return undef if ($ip eq '*');
        return ("Invalid IP address format") if ($ip and $ip !~ /\d+\.\d+\.\d+\.\d+/);
        return ("Invalid netmask format") if ($netmask and $netmask !~ /\d+\.\d+\.\d+\.\d+/);
    }

    if ($command eq 'frame') {
        if ($value !~ /^\d+$/i && $value ne '*') {
            return ("Invalid frame number '$value'");
        }
    }

    if ($command eq 'admin_passwd' or $command eq 'general_passwd' or $command eq '*_passwd') {
        my ($passwd, $newpasswd) = split /,/, $value;
        if (!$passwd or !$newpasswd) {
            return ("Current password and new password couldn't be empty");
        }
    }

    if ($command eq 'HMC_passwd') {
        my ($passwd, $newpasswd) = split /,/, $value;
        if (!$newpasswd) {
            return ("New password couldn't be empty for user 'HMC'");
        }
    }

    return undef;
}

##########################################################################
# Update passwords for different users on FSP/BPA
##########################################################################
sub passwd {

    my $request = shift;
    my $hash    = shift;
    my $exp     = shift;
    my $args    = $request->{arg};
    my $result;
    my $users;

    foreach my $arg (@$args) {
        my ($user,   $value)     = split /=/, $arg;
        my ($passwd, $newpasswd) = split /,/, $value;
        $user =~ s/_passwd$//;
        $user =~ s/^HMC$/access/g;

        if ($user eq "*") {
            push @$users, "access";
            push @$users, "admin";
            push @$users, "general";
        } else {
            push @$users, $user;
        }

        foreach my $usr (@$users) {
            while (my ($cec, $h) = each(%$hash)) {
                while (my ($node, $d) = each(%$h)) {
                    my $type = @$d[4];
                    xCAT::MsgUtils->verbose_message($request, "rspconfig :modify password of $usr for node:$node.");
                    my $data = xCAT::PPCcli::chsyspwd($exp, $usr, $type, $cec, $passwd, $newpasswd);
                    my $Rc       = shift(@$data);
                    my $usr_back = $usr;
                    $usr_back =~ s/^access$/HMC/g;
                    push @$result, [ $node, "$usr_back: @$data[0]", $Rc ];

                    ##################################
                    # Write the new password to table
                    ##################################
                    if ($Rc == SUCCESS) {
                        xCAT::MsgUtils->verbose_message($request, "rspconfig :update xCATdb for node:$node,ID:$usr_back.");
                        xCAT::PPCdb::update_credentials($node, $type, $usr_back, $newpasswd);
                    }
                }
            }
        }
    }

    return ([@$result]);
}

##########################################################################
# Handles all PPC rspconfig commands
##########################################################################
sub cfg {

    my $request = shift;
    my $hash    = shift;
    my $exp     = shift;
    my $args    = $request->{arg};
    my $result;

    foreach (@$args) {
        ##################################
        # Ignore switches in command-line
        ##################################
        unless (/^-/) {
            my ($cmd, $value) = split /=/;

            no strict 'refs';
            $result = $rspconfig{$cmd}($request, $exp, $value, $hash);
            use strict;
        }
    }
    return ($result);
}

##########################################################################
# Enables/disables/displays SSH access to HMC/IVM
##########################################################################
sub sshcfg {
    my $request = shift;
    my $exp     = shift;
    my $mode    = shift;
    my $server  = @$exp[3];
    my $userid  = @$exp[4];
    my $fname = ((xCAT::Utils::isAIX()) ? "/.ssh/" : "/root/.ssh/") . "id_rsa.pub";
    my $auth = "/home/$userid/.ssh/authorized_keys2";

    #####################################
    # Get SSH key on Management Node
    #####################################
    unless (open(RSAKEY, "<$fname")) {
        return ([ [ $server, "Error opening '$fname'", RC_ERROR ] ]);
    }
    my ($sshkey) = <RSAKEY>;
    close(RSAKEY);

    #####################################
    # userid@host not found in key file
    #####################################
    #if ( $sshkey !~ /\s+(\S+\@\S+$)/ ) {
    #    return( [[$server,"Cannot find userid\@host in '$fname'",RC_ERROR]] );
    #}
    my $logon = $1;

    #####################################
    # Determine if SSH is enabled
    #####################################
    if (!defined($mode)) {
        my ($keytype, $key_string) = split /\ /, $sshkey;
        chomp($key_string);
        xCAT::MsgUtils->verbose_message($request, "rspconfig :check sshcfg for user:$logon on node:$server.");
        my $result = xCAT::PPCcli::send_cmd($exp, "cat $auth");
        my $Rc = shift(@$result);

        #################################
        # Return error
        #################################
        if ($Rc != SUCCESS) {
            return ([ [ $server, @$result[0], $Rc ] ]);
        }
        #################################
        # Find logon in key file
        #################################
        foreach (@$result) {
            my ($tmp1, $tmp2) = split /\ /, $_;
            chomp($tmp2);
            if ("$tmp2" eq "$key_string") {
                return ([ [ $server, "enabled", SUCCESS ] ]);
            }
        }
        return ([ [ $server, "disabled", SUCCESS ] ]);
    }
    #####################################
    # Enable/disable SSH
    #####################################
    xCAT::MsgUtils->verbose_message($request, "rspconfig :sshcfg $mode for user:$logon on node:$server.");
    my $result = xCAT::PPCcli::mkauthkeys($exp, $mode, $logon, $sshkey);
    my $Rc = shift(@$result);

    #################################
    # Return error
    #################################
    if ($Rc != SUCCESS) {
        return ([ [ $server, @$result[0], $Rc ] ]);
    }
    return ([ [ $server, lc($mode . "d"), SUCCESS ] ]);
}

sub frame {
    my $request = shift;
    my $exp     = shift;
    my $value   = shift;
    my $hash    = shift;
    my $arg     = $request->{arg};

    foreach (@$arg) {
        my $result;
        my $Rc;
        my $data;

        my ($cmd, $value) = split /=/, $_;
        if ($cmd ne "frame") {
            return ([ [ @$exp[2], "Multiple option $cmd and frame is not accepted", SUCCESS ] ]);
        }

        #################################
        # Open xCAT database to sync with
        # the frame number between hcp
        # and database
        #################################
        my $tab = xCAT::Table->new("ppc");

        while (my ($cec, $h) = each(%$hash)) {
            while (my ($node, $d) = each(%$h)) {
                if (!defined($value)) {

                    #################################
                    # Get frame number
                    #################################
                    xCAT::MsgUtils->verbose_message($request, "rspconfig :get frame_num for node:$node.");
                    $data = xCAT::PPCcli::lssyscfg($exp, @$d[4], @$d[2], 'frame_num');
                    $Rc = shift(@$data);

                    #################################
                    # Return error
                    #################################
                    if ($Rc != SUCCESS) {
                        return ([ [ $node, @$data[0], $Rc ] ]);
                    }

                    push @$result, [ $node, @$data[0], SUCCESS ];

                    #################################
                    # Set frame number to database
                    #################################
                    $tab->setNodeAttribs($node, { id => @$data[0] });

                } elsif ($value eq '*') {
                    #################################
                    # Set frame number
                    # Read the settings from database
                    #################################
                    my $ent = $tab->getNodeAttribs($node, ['id']);

                    #################################
                    # Return error
                    #################################
                    if (!defined($ent) or !defined($ent->{id})) {
                        return ([ [ $node, "Cannot find frame num in database", RC_ERROR ] ]);
                    }
                    xCAT::MsgUtils->verbose_message($request, "rspconfig :set frame_num=" . $ent->{id} . " for node:$node.");
                    $data = xCAT::PPCcli::chsyscfg($exp, "bpa", $d, "frame_num=" . $ent->{id});
                    $Rc = shift(@$data);

                    #################################
                    # Return error
                    #################################
                    if ($Rc != SUCCESS) {
                        return ([ [ $node, @$data[0], $Rc ] ]);
                    }

                    push @$result, [ $node, @$data[0], SUCCESS ];

                } else {
                    #################################
                    # Set frame number
                    # Read the frame number from opt
                    #################################
                    xCAT::MsgUtils->verbose_message($request, "rspconfig :set frame_num=$value for node:$node.");
                    $data = xCAT::PPCcli::chsyscfg($exp, "bpa", $d, "frame_num=$value");
                    $Rc = shift(@$data);

                    #################################
                    # Return error
                    #################################
                    if ($Rc != SUCCESS) {
                        return ([ [ $node, @$data[0], $Rc ] ]);
                    }

                    push @$result, [ $node, @$data[0], SUCCESS ];

                    #################################
                    # Set frame number to database
                    #################################
                    xCAT::MsgUtils->verbose_message($request, "rspconfig : set frame_num, update node:$node attr id=$value.");
                    $tab->setNodeAttribs($node, { id => $value });
                }
            }

            return ([@$result]);
        }
    }
}

sub hostname {
    my $request = shift;
    my $exp     = shift;
    my $value   = shift;
    my $hash    = shift;
    my $arg     = $request->{arg};
    my $result;

    foreach (@$arg) {
        my $data;
        my $Rc;

        my ($cmd, $value) = split /=/, $_;
        if ($cmd ne "hostname") {
            return ([ [ @$exp[2], "Multiple option $cmd and hostname is not accepted", SUCCESS ] ]);
        }

        while (my ($cec, $h) = each(%$hash)) {
            while (my ($node, $d) = each(%$h)) {
                if (!defined($value)) {
                    #################################
                    # Get system name
                    #################################
                    xCAT::MsgUtils->verbose_message($request, "rspconfig :get system name for node:$node.");
                    $data = xCAT::PPCcli::lssyscfg($exp, @$d[4], @$d[2], 'name');
                    $Rc = shift(@$data);

                    #################################
                    # Return error
                    #################################
                    if ($Rc != SUCCESS) {
                        push @$result, [ $node, @$data[0], $Rc ];
                    }

                    push @$result, [ $node, @$data[0], SUCCESS ];
                } elsif ($value eq '*') {
                    xCAT::MsgUtils->verbose_message($request, "rspconfig :set system name:$node for node:$node.");
                    $data = xCAT::PPCcli::chsyscfg($exp, @$d[4], $d, "new_name=$node");
                    $Rc = shift(@$data);

                    #################################
                    # Return error
                    #################################
                    if ($Rc != SUCCESS) {
                        push @$result, [ $node, @$data[0], $Rc ];
                    }

                    push @$result, [ $node, @$data[0], SUCCESS ];
                } else {
                    xCAT::MsgUtils->verbose_message($request, "rspconfig :set system name:$value for node:$node.");
                    $data = xCAT::PPCcli::chsyscfg($exp, @$d[4], $d, "new_name=$value");
                    $Rc = shift(@$data);

                    #################################
                    # Return error
                    #################################
                    if ($Rc != SUCCESS) {
                        push @$result, [ $node, @$data[0], $Rc ];
                    }

                    push @$result, [ $node, @$data[0], SUCCESS ];
                }
            }
        }
    }

    return ([@$result]);
}
##########################################################################
# Do resetnet public entry
##########################################################################
sub resetnet {
    my $request = shift;
    doresetnet($request);
    return 0;
}
##########################################################################
# Reset the network interfraces if necessary
##########################################################################
sub doresetnet {

    my $req = shift;
    my %iphash;
    my $targets;
    my $result;
    my %grouphash;
    my %oihash;
    my %machash;
    my %vpdhash;

    unless ($req) {
        send_msg($req, 1, "request is empty, return");
        return;
    }
    ###########################################
    # prepare to reset network
    ###########################################
    xCAT::MsgUtils->verbose_message($req, "rspconfig :do resetnet begin to phase nodes");
    my $hoststab = xCAT::Table->new('hosts');
    if (!$hoststab) {
        send_msg($req, 1, "Error open hosts table");
        return;
    } else {
        my @hostslist = $hoststab->getAllNodeAttribs([ 'node', 'otherinterfaces' ]);
        foreach my $otherentry (@hostslist) {
            $oihash{ $otherentry->{node} } = $otherentry->{otherinterfaces};
        }
    }

    my $mactab = xCAT::Table->new('mac');
    if (!$mactab) {
        send_msg($req, 1, "Error open mac table");
        return;
    } else {
        my @maclist = $mactab->getAllNodeAttribs([ 'node', 'mac' ]);
        foreach my $macentry (@maclist) {
            $machash{ $macentry->{node} } = $macentry->{mac};
        }
    }
    $mactab = ();

    my $vpdtab = xCAT::Table->new('vpd');
    if (!$vpdtab) {
        send_msg($req, 1, "Error open vpd table");
        return;
    } else {
        my @vpdlist = $vpdtab->getAllNodeAttribs([ 'node', 'mtm', 'serial', 'side' ]);
        foreach my $vpdentry (@vpdlist) {
            if ($vpdentry->{side} =~ /(\w)\-\w/) {
                my $side = $1;
                $vpdhash{ $vpdentry->{node} } = $vpdentry->{mtm} . "*" . $vpdentry->{serial} . "*" . $side;
            }
        }
    }
    $vpdtab = ();

    unless ($req->{node}) {
        send_msg($req, 0, "no node specified");
        return;
    }
    ###########################################
    # Process nodes and get network information
    ###########################################
    my $nodetype = $req->{hwtype};
    if ($nodetype =~ /^(cec|frame)$/) {

        # this brunch is just for the xcat 2.6(or 2.6+) database
        foreach my $nn (@{ $req->{node} }) {
            my $cnodep = xCAT::DBobjUtils->getchildren($nn);
            $nodetype = ($nodetype =~ /^frame$/i) ? "bpa" : "fsp";
            if ($cnodep) {
                foreach my $cnode (@$cnodep) {
                    my $ip = xCAT::NetworkUtils::getNodeIPaddress($cnode);
                    my $oi = $oihash{$cnode};
                    if (!defined $ip) {
                        send_msg($req, "doresetnet: can't get $cnode ip");
                        next;
                    }
                    if (!defined $oi) {
                        send_msg($req, "doresetnet: can't get $cnode hosts.otherinterfaces");
                        next;
                    }
                    if (exists($oihash{$cnode}) and $ip eq $oihash{$cnode}) {
                        send_msg($req, 0, "$cnode: same ip address, skipping $nn network reset");
                    } elsif (!exists $machash{$cnode}) {
                        send_msg($req, 0, "$cnode: no mac defined, skipping $nn network reset");
                    } else {
                        $iphash{$cnode}{sip} = $ip;
                        $iphash{$cnode}{tip} = $oihash{$cnode};
                        if (exists $grouphash{ $vpdhash{$cnode} }) {
                            $grouphash{ $vpdhash{$cnode} } .= ",$cnode";
                        } else {
                            $grouphash{ $vpdhash{$cnode} } = "$cnode";
                        }
                        $targets->{$nodetype}->{$ip}->{'args'} = "0.0.0.0,$cnode";
                        $targets->{$nodetype}->{$ip}->{'mac'} = $machash{$cnode};
                        $targets->{$nodetype}->{$ip}->{'name'} = $cnode;
                        $targets->{$nodetype}->{$ip}->{'ip'}   = $ip;
                        $targets->{$nodetype}->{$ip}->{'type'} = $nodetype;
                        my %netinfo = xCAT::DBobjUtils->getNetwkInfo([$ip]);
                        $targets->{$nodetype}->{$ip}->{'args'} .= ",$netinfo{$ip}{'gateway'},$netinfo{$ip}{'mask'}";

                        #xCAT::MsgUtils->verbose_message($req, "doresetnet: get node $cnode info $targets->{$nodetype}->{$ip}->{'args'}, ip is $ip");
                        $targets->{$nodetype}->{$oi}->{'args'} = "0.0.0.0,$cnode";
                        $targets->{$nodetype}->{$oi}->{'mac'} = $machash{$cnode};
                        $targets->{$nodetype}->{$oi}->{'name'} = $cnode;
                        $targets->{$nodetype}->{$oi}->{'ip'}   = $oi;
                        $targets->{$nodetype}->{$oi}->{'type'} = $nodetype;
                        %netinfo = xCAT::DBobjUtils->getNetwkInfo([$oi]);
                        $targets->{$nodetype}->{$oi}->{'args'} .= ",$netinfo{$oi}{'gateway'},$netinfo{$oi}{'mask'}";

                        #xCAT::MsgUtils->verbose_message($req, "doresetnet: get node $cnode info $targets->{$nodetype}->{$oi}->{'args'}, oi is $oi");

                    }
                }
            } else {
                send_msg($req, 1, "Can't get the fsp/bpa nodes for the $nn");
                return;
            }
        }

        # this brunch is just for the xcat 2.5(or 2.5-) databse
    } elsif ($nodetype =~ /^(fsp|bpa)$/) {
        foreach my $nn (@{ $req->{node} }) {
            my $ip = xCAT::NetworkUtils::getNodeIPaddress($nn);
            if (!defined $ip) {
                send_msg($req, "doresetnet: can't get $nn ip");
                next;
            }
            if (!exists $oihash{$nn}) {
                send_msg($req, "doresetnet: can't get $nn hosts.otherinterfaces");
                next;
            }
            my $oi = $oihash{$nn};
            if (exists($oihash{$nn}) and $ip eq $oihash{$nn}) {
                send_msg($req, 0, "$nn: same ip address, skipping network reset");
            } elsif (!exists $machash{$nn}) {
                send_msg($req, 0, "$nn: no mac defined, skipping network reset");
            } else {
                $iphash{$nn}{sip} = $ip;
                $iphash{$nn}{tip} = $oihash{$nn};
                if (exists $grouphash{ $vpdhash{$nn} }) {
                    $grouphash{ $vpdhash{$nn} } .= ",$nn";
                } else {
                    $grouphash{ $vpdhash{$nn} } = "$nn";
                }
                $targets->{$nodetype}->{$ip}->{'args'} = "0.0.0.0,$nn";
                $targets->{$nodetype}->{$ip}->{'mac'}  = $machash{$nn};
                $targets->{$nodetype}->{$ip}->{'name'} = $nn;
                $targets->{$nodetype}->{$ip}->{'ip'}   = $ip;
                $targets->{$nodetype}->{$ip}->{'type'} = $nodetype;
                my %netinfo = xCAT::DBobjUtils->getNetwkInfo([$ip]);
                $targets->{$nodetype}->{$ip}->{'args'} .= ",$netinfo{$ip}{'gateway'},$netinfo{$ip}{'mask'}";

                #xCAT::MsgUtils->verbose_message($req, "doresetnet: get node $nn info $targets->{$nodetype}->{$ip}->{'args'},ip is $ip");
                $targets->{$nodetype}->{$oi}->{'args'} = "0.0.0.0,$nn";
                $targets->{$nodetype}->{$oi}->{'mac'}  = $machash{$nn};
                $targets->{$nodetype}->{$oi}->{'name'} = $nn;
                $targets->{$nodetype}->{$oi}->{'ip'}   = $oi;
                $targets->{$nodetype}->{$oi}->{'type'} = $nodetype;
                %netinfo = xCAT::DBobjUtils->getNetwkInfo([$oi]);
                $targets->{$nodetype}->{$oi}->{'args'} .= ",$netinfo{$oi}{'gateway'},$netinfo{$oi}{'mask'}";

                #xCAT::MsgUtils->verbose_message($req, "doresetnet: get node $nn info $targets->{$nodetype}->{$oi}->{'args'}, oi is $oi");
            }
        }
    } elsif (!$nodetype) {
        send_msg($req, 0, "no nodetype defined, skipping network reset");
        return;
    } else {
        send_msg($req, 0, "$nodetype not supported, skipping network reset");
        return;
    }

    unless (%grouphash) {
        send_msg($req, 0, "Failed to group the nodes, skipping network reset");
        return;
    }
    ###########################################
    # Update target hardware w/discovery info
    ###########################################
    my %rsp_dev = get_rsp_dev($req, $targets);

    ######################################################
    # Start to reset network. Fork one process per BPA/FSP
    ######################################################
    %oihash  = ();
    %machash = ();
    %vpdhash = ();
    $start   = Time::HiRes::gettimeofday();
    my $children = 0;
    $SIG{CHLD} = sub { while (waitpid(-1, WNOHANG) > 0) { $children--; } };
    my $fds      = new IO::Select;
    my $callback = $req->{callback};
    my $ij       = 0;

    foreach my $node (keys %grouphash) {
        my %iphashfornode;
        my $gc = $grouphash{$node};
        my %rsp_devfornode;

        foreach my $tn (split /,/, $gc) {
            $iphashfornode{$tn} = $iphash{$tn};
            for my $ti (keys %{ $iphash{$tn} }) {
                my $tip = $iphash{$tn}{$ti};
                $rsp_devfornode{$tip} = $rsp_dev{$tip};
            }
        }

        xCAT::MsgUtils->verbose_message($req, "========> begin to fork process for node $node");
        ######################################################
        # Begin fork
        ######################################################
        my $pipe;
        my $rspdevref    = \%rsp_devfornode;
        my $grouphashref = $gc;
        my $iphashref    = \%iphashfornode;
        my $result;
        my @data = ("RSPCONFIG6sK4ci");

        #######################################
        # Pipe childs output back to parent
        #######################################
        my $parent;
        my $child;
        pipe $parent, $child;
        $ij++;
        $ij = int($ij % 60);
        my $pid = xCAT::Utils->xfork();

        if (!defined($pid)) {
            ###################################
            # Fork error
            ###################################
            send_msg($req, 1, "Fork error: $!");
            return undef;
        }
        elsif ($pid == 0) {
            sleep $ij;
            ###################################
            # Child process, clear memory first
            ###################################
            %rsp_dev   = ();
            %grouphash = ();
            %iphash    = ();
            close($parent);
            $req->{pipe} = $child;
            my $msgs;
            my $report;

            #try and try to avoid the fail that caused by refreshing IP when doing resetnet
            my $time = 0;
            while (1) {
                my $erflag = 0;
                $msgs = child_process($grouphashref, $iphashref, $rspdevref, $req, $node);
                foreach my $port (keys %$msgs) {
                    unless ($msgs->{$port} =~ /successful/) {
                        $erflag = 1;
                        last;
                    }
                }
                if ($erflag) {
                    $report = ();
                    foreach my $port1 (keys %$msgs) {
                        $report .= $port1 . ":" . $msgs->{$port1} . ";";
                    }
                    xCAT::MsgUtils->verbose_message($req, "========> try again, $report");

                    #send_msg( $req, 0, "========> try again, $report");
                    sleep 3;
                    $time++;
                } else {
                    last;
                }
                last if ($time > 10);
            }
            $report = ();
            foreach my $port (keys %$msgs) {
                $report .= $port . ":" . $msgs->{$port} . ";";
            }
            send_msg($req, 0, "Resetnet result for $node is : $report");
            ####################################
            # Pass result array back to parent
            ####################################
            my %data;
            $data{errorcode} = 0;
            my $out = $req->{pipe};
            print $out freeze([ \%data ]);
            print $out "\nENDOFFREEZE6sK4ci\n";
            exit(0);
        } else {
            ###################################
            # Parent process
            ###################################
            close($child);
            $pipe = $parent;
        }


        if ($pipe) {
            $fds->add($pipe);
            $children++;
        }
    }
    #############################################
    # Process responses from children
    #############################################
    while ($children > 0) {
        child_response($callback, $fds);
    }
    while (child_response($callback, $fds)) { }

    my $elapsed = Time::HiRes::gettimeofday() - $start;
    my $msg = sprintf("Total rspconfig Time: %.3f sec\n", $elapsed);
    xCAT::MsgUtils->verbose_message($req, $msg);

    return undef;
}
##########################################################################
# child process
##########################################################################
sub child_process {
    my $grouphashref = shift;
    my $iphashref    = shift;
    my $rspdevref    = shift;
    my $req          = shift;
    my $node         = shift;
    my %msginfo;
    my @ns = split /,/, $grouphashref;
    my @valid_ips;
    my @portneedreset;
    my @portsuccess;

    ##########################################################
    # ping static ip firstly, if succesufully, skip resetnet
    ##########################################################
    foreach my $fspport (@ns) {
        my $ip = ${ $iphashref->{$fspport} }{sip};
        my $rc = system("ping -q -n -c 1 -w 1 $ip > /dev/null");
        if ($rc == 0) {
            xCAT::MsgUtils->verbose_message($req, "ping static $ip successfully");
            push @valid_ips,   $ip;        # static ip should be used first
            push @portsuccess, $fspport;
            $msginfo{$fspport} = "successful";
        } else {
            xCAT::MsgUtils->verbose_message($req, "ping static $ip failed, need to do resetnet for $fspport");
            push @portneedreset, $fspport;
        }
    }
    if (scalar(@portneedreset) == 0) {
        return \%msginfo;
    }
    ###########################################
    # ping temp ip secondary
    ###########################################
    foreach my $fspport (@ns) {
        my $ip = ${ $iphashref->{$fspport} }{tip};
        my $rc = system("ping -q -n -c 1 -w 1 $ip > /dev/null");
        if ($rc == 0) {
            push @valid_ips, $ip;
            xCAT::MsgUtils->verbose_message($req, "ping temp $ip successfully");
        } else {
            xCAT::MsgUtils->verbose_message($req, "ping temp $ip failed");
        }
    }
    if (scalar(@valid_ips) == 0) {
        foreach my $fspport (@ns) {
            $msginfo{$fspport} = "failed to find valid ip to log on";
        }
        return \%msginfo;
    }
    #########################################
    # log on, no retry here
    #########################################
    my @exp;
    my $goodip;
    my $retry = 2;
    foreach my $ip (@valid_ips) {
        @exp = xCAT::PPCcfg::connect(${ $rspdevref->{$ip} }{username}, ${ $rspdevref->{$ip} }{password}, $ip);
        ####################################
        # Successfully connected
        ####################################
        if (ref($exp[0]) eq "LWP::UserAgent") {
            $goodip = $ip;
            xCAT::MsgUtils->verbose_message($req, "log in successfully with $ip");
            last;
        }
    }
    my $msg = "login result is :" . join(',', @exp);
    xCAT::MsgUtils->verbose_message($req, $msg);

    ####################################
    # do resetnet
    ####################################
    unless ($goodip) {
        foreach my $fspport (@ns) {
            $msginfo{$fspport} = "failed to log on with $exp[0]";
        }
        return \%msginfo;
    }
    my %handled;
    my $port;
    if (scalar(@portneedreset) == 2) {   ## do resetnet for the other port first
        $port = $portneedreset[0];
        my $ip = ${ $iphashref->{$port} }{sip};
        if ($goodip eq $ip) {
            $port = $portneedreset[1];
        }
        xCAT::MsgUtils->verbose_message($req, "begin to reset for port $port.. good ip is $goodip, ip is $ip....................................");
        my $rc = system("ping -q -n -c 1 -w 1 $ip > /dev/null");
        unless ($rc == 0) {
            $ip = ${ $iphashref->{$port} }{tip};
            $handled{network} = $ip . "," . ${ $rspdevref->{$ip} }{args};
            my @cmds    = ("network=$ip,${$rspdevref->{$ip}}{args}");
            my %request = (
                ppcretry   => 1,
                verbose    => 0,
                ppcmaxp    => 64,
                ppctimeout => 0,
                fsptimeout => 0,
                ppcretry   => 3,
                maxssh     => 8,
                arg        => \@cmds,
                method     => \%handled,
                command    => 'rspconfig',
                hwtype     => ${ $rspdevref->{$ip} }{type},
            );
            xCAT::MsgUtils->verbose_message($req, "Begin to do reset for $port, nic is $ip");
            my $result = xCAT::PPCfsp::handler($ip, \%request, \@exp, 1);
            if ($result) {
                my $errcode = ${ @$result[0] }{errorcode};
                if ($errcode == 0) {
                    $msginfo{$port} = "successful";
                } else {
                    my $node = ${ @$result[0] }{node};
                    $msginfo{$port} = @{ ${ @{ ${ @$node[0] }{data} }[0] }{contents} }[0];
                }
            } else {
                $msginfo{$port} = "failed with unknown reason";
            }
        } else {
            $msginfo{$port} = "successful";
        }
    }
    if ($port) {
        if ($port eq $portneedreset[0]) {
            $port = $portneedreset[1];
        } else {
            $port = $portneedreset[0];
        }
    } else {
        $port = $portneedreset[0];
    }
    xCAT::MsgUtils->verbose_message($req, "begin to reset for port $port......................................");
    my $ip = ${ $iphashref->{$port} }{sip};
    my $rc = system("ping -q -n -c 1 -w 1 $ip > /dev/null");
    unless ($rc == 0) {    #should be unless!!!!!!!!!!!!!
        $ip = ${ $iphashref->{$port} }{tip};
        $handled{network} = $ip . "," . ${ $rspdevref->{$ip} }{args};
        my @cmds    = ("network=$ip,${$rspdevref->{$ip}}{args}");
        my %request = (
            ppcretry   => 1,
            verbose    => 0,
            ppcmaxp    => 64,
            ppctimeout => 0,
            fsptimeout => 0,
            ppcretry   => 3,
            maxssh     => 8,
            arg        => \@cmds,
            method     => \%handled,
            command    => 'rspconfig',
            hwtype     => ${ $rspdevref->{$ip} }{type},
        );
        xCAT::MsgUtils->verbose_message($req, "Begin to do reset for $port, nic is $ip");
        my $result = xCAT::PPCfsp::handler($ip, \%request, \@exp);
        if ($result) {
            my $errcode = ${ @$result[0] }{errorcode};
            if ($errcode == 0) {
                $msginfo{$port} = "successful";
            } else {
                my $node = ${ @$result[0] }{node};
                $msginfo{$port} = @{ ${ @{ ${ @$node[0] }{data} }[0] }{contents} }[0];
            }
        } else {
            $msginfo{$port} = "failed with unknown reason";
        }
    } else {
        xCAT::PPCfsp::disconnect(\@exp);
        $msginfo{$port} = "successful";
    }
    return \%msginfo;
}

#############################################
# Get rsp devices and their logon info
#############################################
sub get_rsp_dev
{
    my $request = shift;
    my $targets = shift;

    my $mm  = $targets->{'mm'}  ? $targets->{'mm'}  : {};
    my $hmc = $targets->{'hmc'} ? $targets->{'hmc'} : {};
    my $fsp = $targets->{'fsp'} ? $targets->{'fsp'} : {};
    my $bpa = $targets->{'bpa'} ? $targets->{'bpa'} : {};

    if (%$mm)
    {
        my $bladeuser = 'USERID';
        my $bladepass = 'PASSW0RD';

        #if ( $verbose ) {
        #    trace( $request, "telneting to management-modules....." );
        #}
        #############################################
        # Check passwd table for userid/password
        #############################################
        my $passtab = xCAT::Table->new('passwd');
        if ($passtab) {

            #my ($ent) = $passtab->getAttribs({key=>'blade'},'username','password');
            my $ent = $passtab->getNodeAttribs('blade', [ 'username', 'password' ]);
            if (defined($ent)) {
                $bladeuser = $ent->{username};
                $bladepass = $ent->{password};
            }
        }
        #############################################
        # Get userid/password
        #############################################
        my $mpatab = xCAT::Table->new('mpa');
        for my $nd (keys %$mm) {
            my $user = $bladeuser;
            my $pass = $bladepass;

            if (defined($mpatab)) {

                #my ($ent) = $mpatab->getAttribs({mpa=>$_},'username','password');
                my $ent = $mpatab->getNodeAttribs($nd, [ 'username', 'password' ]);
                if (defined($ent->{password})) { $pass = $ent->{password}; }
                if (defined($ent->{username})) { $user = $ent->{username}; }
            }
            $mm->{$nd}->{username} = $user;
            $mm->{$nd}->{password} = $pass;
        }
    }
    if (%$hmc)
    {
        #############################################
        # Get HMC userid/password
        #############################################
        foreach (keys %$hmc) {
            ($hmc->{$_}->{username}, $hmc->{$_}->{password}) = xCAT::PPCdb::credentials($hmc->{$_}->{name}, lc($hmc->{$_}->{'type'}), "hscroot");
            xCAT::MsgUtils->verbose_message($request, "user/passwd for $_ is $hmc->{$_}->{username} $hmc->{$_}->{password}");
        }
    }

    if (%$fsp)
    {
        #############################################
        # Get FSP userid/password
        #############################################
        foreach (keys %$fsp) {
            ($fsp->{$_}->{username}, $fsp->{$_}->{password}) = xCAT::PPCdb::credentials($fsp->{$_}->{name}, lc($fsp->{$_}->{'type'}), "admin");
            xCAT::MsgUtils->verbose_message($request, "user/passwd for $_ is $fsp->{$_}->{username} $fsp->{$_}->{password}");
        }
    }

    if (%$bpa)
    {
        #############################################
        # Get BPA userid/password
        #############################################
        foreach (keys %$bpa) {
            ($bpa->{$_}->{username}, $bpa->{$_}->{password}) = xCAT::PPCdb::credentials($bpa->{$_}->{name}, lc($bpa->{$_}->{'type'}), "admin");
            xCAT::MsgUtils->verbose_message($request, "user/passwd for $_ is $bpa->{$_}->{username} $bpa->{$_}->{password}");
        }
    }

    return (%$mm, %$hmc, %$fsp, %$bpa);
}
##########################################################################
# Invokes the callback with the specified message
##########################################################################
sub send_msg {

    my $request = shift;
    my $ecode   = shift;
    my %output;

    #################################################
    # Called from child process - send to parent
    #################################################
    if (exists($request->{pipe})) {
        my $out = $request->{pipe};

        $output{errorcode} = $ecode;
        $output{data}      = \@_;
        print $out freeze([ \%output ]);
        print $out "\nENDOFFREEZE6sK4ci\n";
    }
    #################################################
    # Called from parent - invoke callback directly
    #################################################
    elsif (exists($request->{callback})) {
        my $callback = $request->{callback};
        $output{errorcode} = $ecode;
        $output{data}      = \@_;
        $callback->(\%output);
    }
}
##########################################################################
# Collect output from the child processes
##########################################################################
sub child_response {

    my $callback  = shift;
    my $fds       = shift;
    my @ready_fds = $fds->can_read(1);

    foreach my $rfh (@ready_fds) {
        my $data = <$rfh>;

        #################################
        # Read from child process
        #################################
        if (defined($data)) {
            while ($data !~ /ENDOFFREEZE6sK4ci/) {
                $data .= <$rfh>;
            }
            my $responses = thaw($data);

            #############################
            # rspconfig results
            #############################
            if (@$responses[0] =~ /^RSPCONFIG6sK4ci$/) {

                #shift @$responses;
                #my $ip = @$responses[0];
                #my @rsp1 = (@$responses[1]);
                #$rsp_result{$ip} = \@rsp1;
                #$ip = @$responses[2];
                #if ($ip) {
                #    my @rsp2 = (@$responses[3]);
                #    $rsp_result{$ip} = \@rsp2;
                #}
                next;
            }
            #############################
            # Message or verbose trace
            #############################
            foreach (@$responses) {
                $callback->($_);
            }
            next;
        }
        #################################
        # Done - close handle
        #################################
        $fds->remove($rfh);
        close($rfh);
    }
}
##########################################################################
# Logon through remote FSP HTTP-interface
##########################################################################
sub connect {

    my $username = shift;
    my $passwd   = shift;
    my $server   = shift;
    my $verbose  = shift;
    my $lwp_log;

    ##################################
    # Use timeout
    ##################################
    my $timeout = 10;

    ##################################
    # Redirect STDERR to variable
    ##################################
    if ($verbose) {
        close STDERR;
        if (!open(STDERR, '>', \$lwp_log)) {
            return ("Unable to redirect STDERR: $!");
        }
    }
    $IO::Socket::SSL::VERSION = undef;
    eval { require Net::SSL };

    ##################################
    # Turn on tracing
    ##################################
    if ($verbose) {
        LWP::Debug::level('+');
    }
    ##################################
    # Create cookie
    ##################################
    my $cookie = HTTP::Cookies->new();
    $cookie->set_cookie(0, 'asm_session', '0', 'cgi-bin', '', '443', 0, 0, 3600, 0);

    ##################################
    # Create UserAgent
    ##################################
    my $ua = LWP::UserAgent->new();

    ##################################
    # Set options
    ##################################
    my $url = "https://$server/cgi-bin/cgi?form=2";
    $ua->cookie_jar($cookie);
    $ua->timeout($timeout);

    ##################################
    # Submit logon
    ##################################
    my $res = $ua->post($url,
        [ user => $username,
            password => $passwd,
            lang     => "0",
            submit   => "Log in" ]
    );

    ##################################
    # Logon failed
    ##################################
    if (!$res->is_success()) {
        return ($lwp_log . $res->status_line);
    }
    ##################################
    # To minimize number of GET/POSTs,
    # if we successfully logon, we should
    # get back a valid cookie:
    #    Set-Cookie: asm_session=3038839768778613290
    #
    ##################################
    if ($res->as_string =~ /Set-Cookie: asm_session=(\d+)/) {
        ##############################
        # Successful logon....
        # Return:
        #    UserAgent
        #    Server hostname
        #    UserId
        #    Redirected STDERR/STDOUT
        ##############################
        return ($ua,
            $server,
            $username,
            \$lwp_log);
    }
    ##############################
    # Logon error
    ##############################
    $res = $ua->get($url);
    ##############################
    # Check for specific failures
    # $res->status_line is like "200 OK"
    # $res->content is like <!doctype html public "-//W3C//DTD HTML 4.01 Transitional//EN" .....Too many users......</html>
    # $res->base is like https://41.17.4.2/cgi-bin/cgi?form=2
    ##############################
    my $err;
    if ($res->content =~ /Too many users/i) {
        $err = "Too many users";
    } elsif ($res->content =~ /Invalid user ID or password/i) {
        $err = "Invalid user ID or password";
    } else {
        $err = "Logon failure with unknown reason";
    }

    return ($lwp_log . $err);
}



1;