From fe6b771addc5a5e5ceed871bf090d57650872111 Mon Sep 17 00:00:00 2001 From: Bin Xu Date: Thu, 1 Jun 2017 17:22:22 +0800 Subject: [PATCH] Fix #3087: nodeset improvement - using multi-process to parallelize (#3118) * Fix #3087: nodeset improvement - using multi-process to parallelize - before deliver the request to xcatd, we can split the request to multiple sub-requests which contains different noderange - in that, xcatd could hanle those sub-requests in parallel, just like some issue multiple commands. * refine according to the review comments: - not using POSIX::ceil - Callback is reserved for filter method * refine codes according review comments * make the comments right and more readable. --- perl-xCAT/xCAT/Scope.pm | 165 ++++++++++++++++++++++ xCAT-server/lib/xcat/plugins/destiny.pm | 18 ++- xCAT-server/lib/xcat/plugins/grub2.pm | 12 +- xCAT-server/lib/xcat/plugins/petitboot.pm | 12 +- xCAT-server/lib/xcat/plugins/xnba.pm | 19 +-- 5 files changed, 206 insertions(+), 20 deletions(-) diff --git a/perl-xCAT/xCAT/Scope.pm b/perl-xCAT/xCAT/Scope.pm index 654e6950a..a7aaf2588 100644 --- a/perl-xCAT/xCAT/Scope.pm +++ b/perl-xCAT/xCAT/Scope.pm @@ -1,8 +1,173 @@ package xCAT::Scope; + use xCAT::Utils; use xCAT::Table; use xCAT::ServiceNodeUtils qw(getSNList); + +#----------------------------------------------------------------------------- + +=head3 split_node_array + + Split a node array into multiple subsets in case to handle them in parallel. + + Arguments: + Reference of source array + Maximum subset number + Default element capacity in each subset + Returns: An array of all subsets + Error: + none + Example: + my $subsets = split_node_array(\@nodes, 5, 250); +=cut + +#----------------------------------------------------------------------------- +sub split_node_array { + my $source = shift; + if ($source =~ /xCAT::Scope/) { + $source = shift; + } + my $max_sub = shift; + my $capacity = shift; + + if ($max_sub < 2) {return [$source];} + + my @dest = (); + my $total = $#{$source} + 1; + my $n_sub = int ($total / $capacity); + unless ($n_sub * $capacity == $total) { $n_sub++;} #POSIX::ceil + + if ( $n_sub <= 1 ) { + # Only 1 subset is enough + $dest[0] = $source; + + } elsif ( $n_sub > $max_sub ) { + # Exceed, then to recaculate the capacity of each subset as we only allow max_sub + $capacity = int ($total / $max_sub); + if ( $total % $max_sub > 0 ) { + $capacity += 1; + } + my $start = $end = 0; + for (1..$max_sub) { + $end = $start + $capacity - 1; + if ( $end > $total - 1 ) { + $end = $total - 1 + } + + my @temp = @$source[$start..$end]; + $dest[$_-1]=\@temp; + $start = $end + 1; + } + + } else { + # Only n_sub subsets are required, split the noderange into each subset + my $start = $end = 0; + for (1..$n_sub) { + $end = $start + $capacity - 1; + if ( $end > $total - 1 ) { + $end = $total - 1 + } + #print "subset #$_: $start to $end"; + my @temp = @$source[$start..$end]; + $dest[$_-1]=\@temp; + $start = $end + 1; + } + } + + return \@dest; +} + +#----------------------------------------------------------------------------- + +=head3 get_parallel_scope + + Convert a request object to an array of multiple requests according to the + splitted node range. + + Arguments: + Reference of request + Maximum subset number: Optional, default is 5 + Default element capacity in each subset: Optional, default is 250 + Returns: An array of requests + Error: + none + Example: + my $reqs = xCAT::Scope->get_parallel_scope($request); +=cut + +#----------------------------------------------------------------------------- +sub get_parallel_scope { + my $req = shift; + if ($req =~ /xCAT::Scope/) { + $req = shift; + } + my ($max_sub, $capacity) = @_; + #TODO, make the value configurable + unless ($max_sub) { $max_sub = 5; } + unless ($capacity) { $capacity = 250; } + + my $subsets = split_node_array(\@{$req->{node}}, $max_sub, $capacity); + # Just return the origin one if node range is not big enough. + if ($#{$subsets} < 1) { return [$req]; } + + my @requests = (); + foreach (@$subsets) { + my $reqcopy = {%$req}; + $reqcopy->{node} = $_; + push @requests, $reqcopy; + } + return \@requests; +} + +#----------------------------------------------------------------------------- + +=head3 get_broadcast_scope_with_parallel + + Convert a request object to an array of multiple requests according to the + splitted node range. + + Arguments: + Reference of request + Callback: TODO, Optional, the Callback will be used to filter the nodes + Returns: An array of requests + Error: + none + Example: + my $reqs = xCAT::Scope->get_broadcast_scope($request); +=cut + +#----------------------------------------------------------------------------- +sub get_broadcast_scope_with_parallel { + my $req = shift; + if ($req =~ /xCAT::Scope/) { + $req = shift; + } + #Exit if the packet has been preprocessed in its history + if ($req->{_xcatpreprocessed}->[0] == 1) { return [$req]; } + + #Handle the one for current management/service node + my $reqs = get_parallel_scope($req); + my @requests = @$reqs; + + #Broadcast the request to other management/service nodes + foreach (xCAT::ServiceNodeUtils->getSNList()) { + if (xCAT::NetworkUtils->thishostisnot($_)) { + my $xcatdest = $_; + my $reqcopy = {%$req}; + $reqcopy->{'_xcatdest'} = $_; + $reqcopy->{_xcatpreprocessed}->[0] = 1; + #Apply callback to filter the node range in future. + $reqs = get_parallel_scope($reqcopy); + foreach (@$reqs) { + push @requests, {%$_}; + } + } + } + return \@requests; +} + + sub get_broadcast_scope { my $req = shift; if ($req =~ /xCAT::Scope/) { diff --git a/xCAT-server/lib/xcat/plugins/destiny.pm b/xCAT-server/lib/xcat/plugins/destiny.pm index 41d63fc7b..ba55424f8 100755 --- a/xCAT-server/lib/xcat/plugins/destiny.pm +++ b/xCAT-server/lib/xcat/plugins/destiny.pm @@ -76,15 +76,24 @@ sub process_request { sub relay_response { my $resp = shift; + my $failure = 0; $callback->($resp); if ($resp and ($resp->{errorcode} and $resp->{errorcode}->[0]) or ($resp->{error} and $resp->{error}->[0])) { - $errored = 1; + $failure = 1; } - foreach (@{ $resp->{node} }) { - if ($_->{error} or $_->{errorcode}) { - $errored = 1; + # quick return when detect failure. + unless ( $failure ) { + foreach (@{ $resp->{node} }) { + if ($_->{error} or $_->{errorcode}) { + $failure = 1; + last; + } } } + if ( $failure ) { + $errored = $failure; + } + } sub setdestiny { @@ -408,6 +417,7 @@ sub setdestiny { bootparams => \$bphash}, \&relay_response); if ($errored) { # The error messeage for mkinstall/mknetboot/mkstatelite had been output within relay_response function above, don't need to output more + xCAT::MsgUtils->trace($verbose_on_off, "d", "destiny->process_request: Failed in processing mk$tempstate. Processing will not continue."); return; } diff --git a/xCAT-server/lib/xcat/plugins/grub2.pm b/xCAT-server/lib/xcat/plugins/grub2.pm index 5b2bb8ef3..14fa9702a 100644 --- a/xCAT-server/lib/xcat/plugins/grub2.pm +++ b/xCAT-server/lib/xcat/plugins/grub2.pm @@ -468,10 +468,10 @@ sub preprocess_request { return [$req]; } if (@CN > 0) { # if compute nodes broadcast to all servicenodes - return xCAT::Scope->get_broadcast_scope($req, @_); + return xCAT::Scope->get_broadcast_scope_with_parallel($req); } } - return [$req]; + return xCAT::Scope->get_parallel_scope($req); } sub process_request { @@ -592,9 +592,10 @@ sub process_request { my $inittime = 0; if (exists($request->{inittime})) { $inittime = $request->{inittime}->[0]; } if (!$inittime) { $inittime = 0; } - $errored = 0; + my %bphash; unless ($args[0] eq 'stat') { # or $args[0] eq 'enact') { + $errored = 0; xCAT::MsgUtils->trace($verbose_on_off, "d", "grub2: issue setdestiny request"); $sub_req->({ command => ['setdestiny'], node => \@nodes, @@ -602,8 +603,11 @@ sub process_request { arg => \@args, bootparams => \%bphash }, \&pass_along); + if ($errored) { + xCAT::MsgUtils->trace($verbose_on_off, "d", "petitboot: Failed in processing setdestiny. Processing will not continue."); + return; + } } - if ($errored) { return; } my $chaintab = xCAT::Table->new('chain', -create => 1); my $chainhash = $chaintab->getNodesAttribs(\@nodes, ['currstate']); diff --git a/xCAT-server/lib/xcat/plugins/petitboot.pm b/xCAT-server/lib/xcat/plugins/petitboot.pm index 04430340b..04f7bcdf7 100644 --- a/xCAT-server/lib/xcat/plugins/petitboot.pm +++ b/xCAT-server/lib/xcat/plugins/petitboot.pm @@ -360,10 +360,10 @@ sub preprocess_request { return [$req]; } if (@CN > 0) { # if compute nodes broadcast to all servicenodes - return xCAT::Scope->get_broadcast_scope($req, @_); + return xCAT::Scope->get_broadcast_scope_with_parallel($req); } } - return [$req]; + return xCAT::Scope->get_parallel_scope($req); } @@ -487,9 +487,10 @@ sub process_request { my $inittime = 0; if (exists($request->{inittime})) { $inittime = $request->{inittime}->[0]; } if (!$inittime) { $inittime = 0; } - $errored = 0; + my %bphash; unless ($args[0] eq 'stat') { # or $args[0] eq 'enact') { + $errored = 0; xCAT::MsgUtils->trace($verbose_on_off, "d", "petitboot: issue setdestiny request"); $sub_req->({ command => ['setdestiny'], node => \@nodes, @@ -497,8 +498,11 @@ sub process_request { arg => \@args, bootparams => \%bphash}, \&pass_along); + if ($errored) { + xCAT::MsgUtils->trace($verbose_on_off, "d", "petitboot: Failed in processing setdestiny. Processing will not continue."); + return; + } } - if ($errored) { return; } # Fix the bug 4611: PowerNV stateful CN provision will hang at reboot stage# if ($args[0] eq 'next') { diff --git a/xCAT-server/lib/xcat/plugins/xnba.pm b/xCAT-server/lib/xcat/plugins/xnba.pm index e13c8dc79..dd8fe9d1c 100644 --- a/xCAT-server/lib/xcat/plugins/xnba.pm +++ b/xCAT-server/lib/xcat/plugins/xnba.pm @@ -117,7 +117,7 @@ sub setstate { #Implement the kcmdline append here for #most generic, least code duplication -###hack start + ###hack start # This is my comment. There are many others like it, but this one is mine. # My comment is my best friend. It is my life. I must master it as I must master my life. # Without me, my comment is useless. Without my comment, I am useless. @@ -167,7 +167,7 @@ sub setstate { #$kern->{kcmdline} .= " ".$kern->{addkcmdline}; $kern->{kcmdline} .= " " . $kcmdlinehack; -###hack end + ###hack end } } @@ -243,7 +243,7 @@ sub setstate { if ($kern->{kernel} =~ /esxi[56]/) { #Make uefi boot provisions my $ucfg; open($ucfg, '>', $tftpdir . "/xcat/xnba/nodes/" . $node . ".uefi"); - if ($kern->{kcmdline} =~ / xcat\/netboot/) { + if ($kern->{kcmdline} =~ /xcat\/netboot/) { $kern->{kcmdline} =~ s/xcat\/netboot/\/tftpboot\/xcat\/netboot/; } print $ucfg "#!gpxe\n"; @@ -408,10 +408,10 @@ sub preprocess_request { return [$req]; } if (@CN > 0) { # if compute nodes broadcast to all servicenodes - return xCAT::Scope->get_broadcast_scope($req, @_); + return xCAT::Scope->get_broadcast_scope_with_parallel($req); } } - return [$req]; + return xCAT::Scope->get_parallel_scope($req); } sub process_request { @@ -545,9 +545,10 @@ sub process_request { my $inittime = 0; if (exists($::XNBA_request->{inittime})) { $inittime = $::XNBA_request->{inittime}->[0]; } if (!$inittime) { $inittime = 0; } - $errored = 0; + my %bphash; unless ($args[0] eq 'stat') { # or $args[0] eq 'enact') { + $errored = 0; xCAT::MsgUtils->trace($verbose_on_off, "d", "xnba: issue setdestiny request"); $sub_req->({ command => ['setdestiny'], node => \@nodes, @@ -555,10 +556,12 @@ sub process_request { arg => \@args , bootparams => \%bphash}, \&pass_along); + if ($errored) { + xCAT::MsgUtils->trace($verbose_on_off, "d", "xnba: Failed in processing setdestiny. Processing will not continue."); + return; + } } - if ($errored) { return; } - #Time to actually configure the nodes, first extract database data with the scalable calls my $chaintab = xCAT::Table->new('chain'); my $noderestab = xCAT::Table->new('noderes'); #in order to detect per-node tftp directories