Fix #3087: nodeset improvement - using multi-process to parallelize (#3118)

* Fix #3087: nodeset improvement - using multi-process to parallelize
 - before deliver the request to xcatd, we can split the request to multiple sub-requests which contains different noderange
 - in that, xcatd could hanle those sub-requests in parallel, just like some issue multiple commands.

* refine according to the review comments:
 - not using POSIX::ceil
 - Callback is reserved for filter method

* refine codes according review comments

* make the comments right and more readable.

perl-xCAT/xCAT/Scope.pm

@@ -1,8 +1,173 @@
 package xCAT::Scope;
+
 use xCAT::Utils;
 use xCAT::Table;
 use xCAT::ServiceNodeUtils qw(getSNList);
 
+
+#-----------------------------------------------------------------------------
+
+=head3 split_node_array
+
+    Split a node array into multiple subsets in case to handle them in parallel.
+
+    Arguments:
+       Reference of source array
+       Maximum subset number
+       Default element capacity in each subset
+    Returns: An array of all subsets
+    Error:
+        none
+    Example:
+        my $subsets = split_node_array(\@nodes, 5, 250);
+=cut
+
+#-----------------------------------------------------------------------------
+sub split_node_array {
+    my $source = shift;
+    if ($source =~ /xCAT::Scope/) {
+        $source = shift;
+    }
+    my $max_sub = shift;
+    my $capacity = shift;
+
+    if ($max_sub < 2) {return [$source];}
+ 
+    my @dest = ();
+    my $total = $#{$source} + 1;
+    my $n_sub = int ($total / $capacity);
+    unless ($n_sub * $capacity == $total) { $n_sub++;} #POSIX::ceil
+
+    if ( $n_sub <= 1 ) {
+        # Only 1 subset is enough
+        $dest[0] = $source;
+
+    } elsif ( $n_sub > $max_sub ) {
+        # Exceed, then to recaculate the capacity of each subset as we only allow max_sub
+        $capacity = int ($total / $max_sub);
+        if ( $total % $max_sub > 0 ) {
+            $capacity += 1;
+        }
+        my $start = $end = 0;
+        for (1..$max_sub) {
+            $end = $start + $capacity - 1;
+            if ( $end > $total - 1 ) {
+                $end = $total - 1
+            }
+
+            my @temp = @$source[$start..$end];
+            $dest[$_-1]=\@temp;
+            $start = $end + 1;
+        }
+
+    } else {
+        # Only n_sub subsets are required, split the noderange into each subset
+        my $start = $end = 0;
+        for (1..$n_sub) {
+            $end = $start + $capacity - 1;
+            if ( $end > $total - 1 ) {
+                $end = $total - 1
+            }
+            #print "subset #$_: $start to $end";
+            my @temp = @$source[$start..$end];
+            $dest[$_-1]=\@temp;
+            $start = $end + 1;
+        }
+    }
+
+    return \@dest;
+}
+
+#-----------------------------------------------------------------------------
+
+=head3 get_parallel_scope
+
+    Convert a request object to an array of multiple requests according to the 
+    splitted node range.
+
+    Arguments:
+       Reference of request
+       Maximum subset number: Optional, default is 5
+       Default element capacity in each subset: Optional, default is 250
+    Returns: An array of requests
+    Error:
+        none
+    Example:
+        my $reqs = xCAT::Scope->get_parallel_scope($request);
+=cut
+
+#-----------------------------------------------------------------------------
+sub get_parallel_scope {
+    my $req = shift;
+    if ($req =~ /xCAT::Scope/) {
+        $req = shift;
+    }
+    my ($max_sub, $capacity) = @_;
+    #TODO, make the value configurable
+    unless ($max_sub) { $max_sub = 5; }
+    unless ($capacity) { $capacity = 250; }
+
+    my $subsets = split_node_array(\@{$req->{node}}, $max_sub, $capacity);
+    # Just return the origin one if node range is not big enough.
+    if ($#{$subsets} < 1) { return [$req]; }
+
+    my @requests = (); 
+    foreach (@$subsets) {
+        my $reqcopy = {%$req};
+        $reqcopy->{node} = $_;
+        push @requests, $reqcopy;
+    }
+    return \@requests;
+}
+
+#-----------------------------------------------------------------------------
+
+=head3 get_broadcast_scope_with_parallel
+
+    Convert a request object to an array of multiple requests according to the 
+    splitted node range.
+
+    Arguments:
+       Reference of request
+       Callback: TODO, Optional, the Callback will be used to filter the nodes
+    Returns: An array of requests
+    Error:
+        none
+    Example:
+        my $reqs = xCAT::Scope->get_broadcast_scope($request);
+=cut
+
+#-----------------------------------------------------------------------------
+sub get_broadcast_scope_with_parallel {
+    my $req = shift;
+    if ($req =~ /xCAT::Scope/) {
+        $req = shift;
+    }
+    #Exit if the packet has been preprocessed in its history
+    if ($req->{_xcatpreprocessed}->[0] == 1) { return [$req]; }
+
+    #Handle the one for current management/service node
+    my $reqs = get_parallel_scope($req); 
+    my @requests = @$reqs;
+
+    #Broadcast the request to other management/service nodes
+    foreach (xCAT::ServiceNodeUtils->getSNList()) {
+        if (xCAT::NetworkUtils->thishostisnot($_)) {
+            my $xcatdest = $_;
+            my $reqcopy = {%$req};
+            $reqcopy->{'_xcatdest'} = $_;
+            $reqcopy->{_xcatpreprocessed}->[0] = 1;
+            #Apply callback to filter the node range in future.
+            $reqs = get_parallel_scope($reqcopy);
+            foreach (@$reqs) {
+                push @requests, {%$_};
+            }
+        }
+    }
+    return \@requests;
+}
+
+
 sub get_broadcast_scope {
     my $req = shift;
     if ($req =~ /xCAT::Scope/) {

xCAT-server/lib/xcat/plugins/destiny.pm

@@ -76,15 +76,24 @@ sub process_request {
 
 sub relay_response {
     my $resp = shift;
+    my $failure = 0;
     $callback->($resp);
     if ($resp and ($resp->{errorcode} and $resp->{errorcode}->[0]) or ($resp->{error} and $resp->{error}->[0])) {
-        $errored = 1;
+        $failure = 1;
     }
-    foreach (@{ $resp->{node} }) {
-        if ($_->{error} or $_->{errorcode}) {
-            $errored = 1;
+    # quick return when detect failure.
+    unless ( $failure ) {
+        foreach (@{ $resp->{node} }) {
+            if ($_->{error} or $_->{errorcode}) {
+                $failure = 1;
+                last;
+            }
         }
     }
+    if ( $failure ) {
+        $errored = $failure;
+    }
+
 }
 
 sub setdestiny {
@@ -408,6 +417,7 @@ sub setdestiny {
                     bootparams => \$bphash}, \&relay_response);
             if ($errored) {
                 # The error messeage for mkinstall/mknetboot/mkstatelite had been output within relay_response function above, don't need to output more
+                xCAT::MsgUtils->trace($verbose_on_off, "d", "destiny->process_request: Failed in processing mk$tempstate.  Processing will not continue.");
                 return;
             }
 

xCAT-server/lib/xcat/plugins/grub2.pm

@@ -468,10 +468,10 @@ sub preprocess_request {
             return [$req];
         }
         if (@CN > 0) {    # if compute nodes broadcast to all servicenodes
-            return xCAT::Scope->get_broadcast_scope($req, @_);
+            return xCAT::Scope->get_broadcast_scope_with_parallel($req);
         }
     }
-    return [$req];
+    return xCAT::Scope->get_parallel_scope($req);
 }
 
 sub process_request {
@@ -592,9 +592,10 @@ sub process_request {
     my $inittime = 0;
     if (exists($request->{inittime})) { $inittime = $request->{inittime}->[0]; }
     if (!$inittime) { $inittime = 0; }
-    $errored = 0;
+
     my %bphash;
     unless ($args[0] eq 'stat') {    # or $args[0] eq 'enact') {
+        $errored = 0;
         xCAT::MsgUtils->trace($verbose_on_off, "d", "grub2: issue setdestiny request");
         $sub_req->({ command => ['setdestiny'],
                 node     => \@nodes,
@@ -602,8 +603,11 @@ sub process_request {
                 arg      => \@args,
                 bootparams => \%bphash
                 }, \&pass_along);
+        if ($errored) { 
+            xCAT::MsgUtils->trace($verbose_on_off, "d", "petitboot: Failed in processing setdestiny.  Processing will not continue.");
+            return; 
+        }
     }
-    if ($errored) { return; }
 
     my $chaintab = xCAT::Table->new('chain', -create => 1);
     my $chainhash = $chaintab->getNodesAttribs(\@nodes, ['currstate']);

xCAT-server/lib/xcat/plugins/petitboot.pm

@@ -360,10 +360,10 @@ sub preprocess_request {
             return [$req];
         }
         if (@CN > 0) {    # if compute nodes broadcast to all servicenodes
-            return xCAT::Scope->get_broadcast_scope($req, @_);
+            return xCAT::Scope->get_broadcast_scope_with_parallel($req);
         }
     }
-    return [$req];
+    return xCAT::Scope->get_parallel_scope($req);
 }
 
 
@@ -487,9 +487,10 @@ sub process_request {
     my $inittime = 0;
     if (exists($request->{inittime})) { $inittime = $request->{inittime}->[0]; }
     if (!$inittime) { $inittime = 0; }
-    $errored = 0;
+
     my %bphash;
     unless ($args[0] eq 'stat') {    # or $args[0] eq 'enact') {
+        $errored = 0;
         xCAT::MsgUtils->trace($verbose_on_off, "d", "petitboot: issue setdestiny request");
         $sub_req->({ command => ['setdestiny'],
                 node     => \@nodes,
@@ -497,8 +498,11 @@ sub process_request {
                 arg      => \@args,
                 bootparams => \%bphash},
                 \&pass_along);
+        if ($errored) { 
+            xCAT::MsgUtils->trace($verbose_on_off, "d", "petitboot: Failed in processing setdestiny.  Processing will not continue.");
+            return; 
+        }
     }
-    if ($errored) { return; }
 
     # Fix the bug 4611: PowerNV stateful CN provision will hang at reboot stage#
     if ($args[0] eq 'next') {

xCAT-server/lib/xcat/plugins/xnba.pm

@@ -117,7 +117,7 @@ sub setstate {
             #Implement the kcmdline append here for
             #most generic, least code duplication
 
-###hack start
+        ###hack start
             # This is my comment. There are many others like it, but this one is mine.
             # My comment is my best friend. It is my life. I must master it as I must master my life.
             # Without me, my comment is useless. Without my comment, I am useless.
@@ -167,7 +167,7 @@ sub setstate {
 
             #$kern->{kcmdline} .= " ".$kern->{addkcmdline};
             $kern->{kcmdline} .= " " . $kcmdlinehack;
-###hack end
+        ###hack end
 
         }
     }
@@ -243,7 +243,7 @@ sub setstate {
                 if ($kern->{kernel} =~ /esxi[56]/) {  #Make uefi boot provisions
                     my $ucfg;
                     open($ucfg, '>', $tftpdir . "/xcat/xnba/nodes/" . $node . ".uefi");
-                    if ($kern->{kcmdline} =~ / xcat\/netboot/) {
+                    if ($kern->{kcmdline} =~ /xcat\/netboot/) {
                         $kern->{kcmdline} =~ s/xcat\/netboot/\/tftpboot\/xcat\/netboot/;
                     }
                     print $ucfg "#!gpxe\n";
@@ -408,10 +408,10 @@ sub preprocess_request {
             return [$req];
         }
         if (@CN > 0) {    # if compute nodes broadcast to all servicenodes
-            return xCAT::Scope->get_broadcast_scope($req, @_);
+            return xCAT::Scope->get_broadcast_scope_with_parallel($req);
         }
     }
-    return [$req];
+    return xCAT::Scope->get_parallel_scope($req);
 }
 
 sub process_request {
@@ -545,9 +545,10 @@ sub process_request {
     my $inittime = 0;
     if (exists($::XNBA_request->{inittime})) { $inittime = $::XNBA_request->{inittime}->[0]; }
     if (!$inittime) { $inittime = 0; }
-    $errored = 0;
+
     my %bphash;
     unless ($args[0] eq 'stat') {    # or $args[0] eq 'enact') {
+        $errored = 0;
         xCAT::MsgUtils->trace($verbose_on_off, "d", "xnba: issue setdestiny request");
         $sub_req->({ command => ['setdestiny'],
                 node     => \@nodes,
@@ -555,10 +556,12 @@ sub process_request {
                 arg      => \@args ,
                 bootparams => \%bphash},
                 \&pass_along);
+        if ($errored) { 
+            xCAT::MsgUtils->trace($verbose_on_off, "d", "xnba: Failed in processing setdestiny.  Processing will not continue.");
+            return; 
+        }
     }
 
-    if ($errored) { return; }
-
     #Time to actually configure the nodes, first extract database data with the scalable calls
     my $chaintab = xCAT::Table->new('chain');
     my $noderestab = xCAT::Table->new('noderes'); #in order to detect per-node tftp directories