From e05432da84293a52ca3804d042ea6cedab36bb70 Mon Sep 17 00:00:00 2001 From: ligc Date: Fri, 3 Jul 2009 08:56:14 +0000 Subject: [PATCH] automatic retry: initial code drop git-svn-id: https://svn.code.sf.net/p/xcat/code/xcat-core/trunk@3694 8638fb3e-16cb-4fca-ae20-7b5d299a9bcd --- perl-xCAT/xCAT/PPCboot.pm | 48 ++++++- perl-xCAT/xCAT/PPCpower.pm | 113 ++++++++++++++- perl-xCAT/xCAT/Usage.pm | 4 +- perl-xCAT/xCAT/Utils.pm | 281 +++++++++++++++++++++++++++++++++++++ 4 files changed, 441 insertions(+), 5 deletions(-) diff --git a/perl-xCAT/xCAT/PPCboot.pm b/perl-xCAT/xCAT/PPCboot.pm index 39d6008aa..40526f9ed 100644 --- a/perl-xCAT/xCAT/PPCboot.pm +++ b/perl-xCAT/xCAT/PPCboot.pm @@ -5,6 +5,8 @@ use strict; use Getopt::Long; use xCAT::PPCcli qw(SUCCESS EXPECT_ERROR RC_ERROR NR_ERROR); use xCAT::Usage; +use xCAT::Utils; +use xCAT::MsgUtils; ########################################################################## @@ -67,6 +69,12 @@ sub parse_args { } } + if (exists( $opt{m} ) ){ + my $res = xCAT::Utils->check_deployment_monitoring_settings($request, \%opt); + if ($res != SUCCESS) { + return(usage()); + } + } #################################### # Check for "-" with no option #################################### @@ -259,7 +267,7 @@ sub rnetboot { my $hwtype = @$exp[2]; my $result; my $name; - + my $callback = $request->{callback}; ##################################### # Get node data ##################################### @@ -350,6 +358,44 @@ sub rnetboot { $result = do_rnetboot( $request, $d, $exp, $name, $node, \%opt ); } $sitetab->close; + + if (defined($request->{opt}->{m})) { + + my $retries = 0; + my @monnodes = ($name); + my $monsettings = xCAT::Utils->generate_monsettings($request, \@monnodes); + xCAT::Utils->monitor_installation($request, $monsettings);; + while ($retries++ < $monsettings->{'retrycount'} && scalar(keys %{$monsettings->{'nodes'}}) > 0) { + ####lparnetboot can not support multiple nodes in one invocation + ####for now, does not know how the $d and \%opt will be changed if + ####support mulitiple nodes in one invocation, + ####so just use the original node name and node attribute array and hash + + my $rsp={}; + $rsp->{data}->[0] = "$node: Reinitializing the installation: $retries retry"; + xCAT::MsgUtils->message("I", $rsp, $callback); + if ($vcon and $vcon->{"value"} and $vcon->{"value"} eq "yes" ) { + $result = xCAT::PPCcli::lpar_netboot( + $exp, + $request->{verbose}, + $name, + $d, + \%opt ); + } else { + $result = do_rnetboot( $request, $d, $exp, $name, $node, \%opt ); + } + xCAT::Utils->monitor_installation($request, $monsettings); + + } + #failed after retries + if (scalar(keys %{$monsettings->{'nodes'}}) > 0) { + foreach my $node (keys %{$monsettings->{'nodes'}}) { + my $rsp={}; + $rsp->{data}->[0] = "The node \"$node\" can not reach the expected status after $monsettings->{'retrycount'} retries, the installation for this done failed"; + xCAT::MsgUtils->message("E", $rsp, $callback); + } + } + } $Rc = shift(@$result); diff --git a/perl-xCAT/xCAT/PPCpower.pm b/perl-xCAT/xCAT/PPCpower.pm index a4fa9cb28..0a3086991 100644 --- a/perl-xCAT/xCAT/PPCpower.pm +++ b/perl-xCAT/xCAT/PPCpower.pm @@ -5,7 +5,7 @@ use strict; use Getopt::Long; use xCAT::PPCcli qw(SUCCESS EXPECT_ERROR RC_ERROR NR_ERROR); use xCAT::Usage; - +use xCAT::MsgUtils; ########################################################################## # Parse the command line for options and operands @@ -39,7 +39,7 @@ sub parse_args { $Getopt::Long::ignorecase = 0; Getopt::Long::Configure( "bundling" ); - if ( !GetOptions( \%opt, qw(V|Verbose) )) { + if ( !GetOptions( \%opt, qw(V|Verbose m:s@ t=s r=s) )) { return( usage() ); } #################################### @@ -75,6 +75,13 @@ sub parse_args { $cmd = ($cmd eq "boot") ? "powercmd_boot" : "powercmd"; } $request->{method} = $cmd; + + if (exists( $opt{m} ) ){ + my $res = xCAT::Utils->check_deployment_monitoring_settings($request, \%opt); + if ($res != SUCCESS) { + return(usage()); + } + } return( \%opt ); } @@ -136,6 +143,7 @@ sub powercmd_boot { my $hash = shift; my $exp = shift; my @output = (); + my $callback = $request->{'callback'}; ###################################### # Power commands are grouped by CEC @@ -191,6 +199,62 @@ sub powercmd_boot { $d ); push @output, [$name,@$result[1],@$result[0]]; } + if (defined($request->{opt}->{m})) { + + my $retries = 0; + my @monnodes = keys %$hash; + my $monsettings = xCAT::Utils->generate_monsettings($request, \@monnodes); + xCAT::Utils->monitor_installation($request, $monsettings); + while ($retries++ < $monsettings->{'retrycount'} && scalar(keys %{$monsettings->{nodes}}) > 0) { + + #The nodes that need to retry + my @nodesretry = keys %{$monsettings->{'nodes'}}; + my $nodes = join ',', @nodesretry; + my $rsp={}; + $rsp->{data}->[0] = "$nodes: Reinitializing the installation: $retries retry"; + xCAT::MsgUtils->message("I", $rsp, $callback); + + + foreach my $node (keys %$hash) + { + # The installation for this node has been finished + if(!grep(/^$node$/, @nodesretry)) { + delete($hash->{$node}); + } + } + while (my ($name,$d) = each(%$hash) ) { + my $type = @$d[4]; + my $id = ($type=~/^(fsp|bpa)$/) ? $type : @$d[0]; + + if ( $Rc != SUCCESS ) { + push @output, [$name,$data,$Rc]; + next; + } + if ( !exists( $data->{$id} )) { + push @output, [$name,"Node not found",1]; + next; + } + my $state = power_status($data->{$id}); + my $op = ($state =~ /^off$/) ? "on" : "reset"; + + my $result = xCAT::PPCcli::chsysstate( + $exp, + $op, + $d ); + push @output, [$name,@$result[1],@$result[0]]; + } + my @monnodes = keys %{$monsettings->{nodes}}; + xCAT::Utils->monitor_installation($request, $monsettings); + } + #failed after retries + if (scalar(keys %{$monsettings->{'nodes'}}) > 0) { + foreach my $node (keys %{$monsettings->{nodes}}) { + my $rsp={}; + $rsp->{data}->[0] = "The node \"$node\" can not reach the expected status after $monsettings->{'retrycount'} retries, the installation for this done failed"; + xCAT::MsgUtils->message("E", $rsp, $callback); + } + } + } return( \@output ); } @@ -204,6 +268,7 @@ sub powercmd { my $hash = shift; my $exp = shift; my @result = (); + my $callback = $request->{'callback'}; #################################### # Power commands are grouped by CEC @@ -225,6 +290,50 @@ sub powercmd { ################################ push @result, [$name,@$values[0],$Rc]; } + + if (defined($request->{opt}->{m})) { + + my $retries = 0; + my @monnodes = keys %$hash; + my $monsettings = xCAT::Utils->generate_monsettings($request, \@monnodes); + xCAT::Utils->monitor_installation($request, $monsettings); + while ($retries++ < $monsettings->{'retrycount'} && scalar(keys %{$monsettings->{nodes}}) > 0) { + #The nodes that need to retry + my @nodesretry = keys %{$monsettings->{'nodes'}}; + my $nodes = join ',', @nodesretry; + + my $rsp={}; + $rsp->{data}->[0] = "$nodes: Reinitializing the installation: $retries retry"; + xCAT::MsgUtils->message("I", $rsp, $callback); + + foreach my $node (keys %$hash) + { + # The installation for this node has been finished + if(!grep(/^$node$/, @nodesretry)) { + delete($hash->{$node}); + } + } + while (my ($name,$d) = each(%$hash) ) { + my $values = xCAT::PPCcli::chsysstate( + $exp, + $request->{op}, + $d ); + my $Rc = shift(@$values); + + push @result, [$name,@$values[0],$Rc]; + } + my @monnodes = keys %{$monsettings->{nodes}}; + xCAT::Utils->monitor_installation($request, $monsettings); + } + #failed after retries + if (scalar(keys %{$monsettings->{'nodes'}}) > 0) { + foreach my $node (keys %{$monsettings->{nodes}}) { + my $rsp={}; + $rsp->{data}->[0] = "The node \"$node\" can not reach the expected status after $monsettings->{'retrycount'} retries, the installation for this done failed"; + xCAT::MsgUtils->message("E", $rsp, $callback); + } + } + } return( \@result ); } diff --git a/perl-xCAT/xCAT/Usage.pm b/perl-xCAT/xCAT/Usage.pm index 637d174f2..0aa22b952 100644 --- a/perl-xCAT/xCAT/Usage.pm +++ b/perl-xCAT/xCAT/Usage.pm @@ -17,10 +17,10 @@ use xCAT::Utils; my %usage = ( "rnetboot" => -"Usage: rnetboot [-s net|hd] [-f] [-V|--verbose] +"Usage: rnetboot [-s net|hd] [-f] [-V|--verbose] [-m table.colum==expectedstatus] [-m table.colum==expectedstatus...] [-r ] [-t ] rnetboot [-h|--help|-v|--version]", "rpower" => -"Usage: rpower [--nodeps] [on|onstandby|off|reset|stat|state|boot] [-V|--verbose] +"Usage: rpower [--nodeps] [on|onstandby|off|reset|stat|state|boot] [-V|--verbose] [-m table.colum==expectedstatus][-m table.colum==expectedstatus...] [-r ] [-t ] rpower [-h|--help|-v|--version] KVM Virtualization specific: rpower [boot] [ -c ] diff --git a/perl-xCAT/xCAT/Utils.pm b/perl-xCAT/xCAT/Utils.pm index a8404ee01..06ce1605a 100644 --- a/perl-xCAT/xCAT/Utils.pm +++ b/perl-xCAT/xCAT/Utils.pm @@ -4325,4 +4325,285 @@ sub selection_string_match() } return $match; } +#------------------------------------------------------------------------------- + +=head3 check_deployment_monitoring_settings + Check the deployment retry monitoring settings. + Arguments: + $request: request hash + $mstring: The monitoring setting string + specified with the -m flag for rpower or rnetboot + Returns: + 0 - ok + 1 - failed + Globals: + none + Example: + my $rc=xCAT::Utils->check_deployment_monitoring_settings($opt(m)) + Comments: + none + +=cut + +#------------------------------------------------------------------------------- +sub check_deployment_monitoring_settings() +{ + my ($class, $request, $opt_ref) = @_; + + my $callback = $request->{callback}; + my @mstring = @{$opt_ref->{'m'}}; + + # -r flag is required with -m flag + if (!defined($opt_ref->{'t'})) { + my $rsp={}; + $rsp->{data}->[0] = "Flag missing, the -t flag is required"; + xCAT::MsgUtils->message("E", $rsp, $callback); + return 1; + } + + foreach my $m (@mstring) { + if ($m eq '') { + #No value specified with -m flag + next; + } + my $attr; + my $val; + if ($m =~ /[^=]*==/) { + ($attr, $val) = split /==/,$m,2; + } elsif ($m =~ /^[^=]*=~/) { + ($attr, $val) = split /=~/,$m,2; + $val =~ s/^\///; + $val =~ s/\/$//; + } else { + my $rsp={}; + $rsp->{data}->[0] = "Invalid string \"$m\" specified with -m flag"; + xCAT::MsgUtils->message("E", $rsp, $callback); + return 1; + } + + # The attr is table.column + if ($attr !~ /\..*$/) { + my $rsp={}; + $rsp->{data}->[0] = "Invalid attribute \"$attr\" specified with -m flag, should be table.column"; + xCAT::MsgUtils->message("E", $rsp, $callback); + return 1; + } + if($val eq '') { + my $rsp={}; + $rsp->{data}->[0] = "The value of attribute \"$attr\" can not be NULL"; + xCAT::MsgUtils->message("E", $rsp, $callback); + return 1; + } + } + return 0; +} + +#------------------------------------------------------------------------------- + +=head3 generate_monsettings() + Generate installation monitoring settings hash. + Arguments: + $request: request hash + \@monnodes: nodes to be monitored + Returns: + \%monsettings - the ref of %monsettings hash + Globals: + none + Example: + my $monsettings_ref = xCAT::Utils->generate_monsettings($request, \@monnodes) + Comments: + none + +=cut + +#------------------------------------------------------------------------------- +sub generate_monsettings() +{ + my ($class, $request, $monnodes_ref) = @_; + + my @monnodes = @$monnodes_ref; + my $callback = $request->{callback}; + my @mstring = @{$request->{opt}->{m}}; + my %monsettings = (); + + #set default value for each attribute, + #to avoid ugly perl syntax error + my %defaultattrs = ( + "timeout" => "10", + "retrycount" => "3" + ); + + #Monitoring settings check already done in parse_args, + #Assume it is correct. + foreach my $m (@mstring) { + if ($m eq '') { + # No value specified with -m flag + next; + } + my $attr; + my $val; + my $matchtype; + if ($m =~ /^[^=]*\==/) { + ($attr, $val) = split /==/,$m,2; + $matchtype='match'; + } elsif ($m =~ /^[^=]*=~/) { + ($attr, $val) = split /=~/,$m,2; + $val =~ s/^\///; + $val =~ s/\/$//; + $matchtype='regex'; + } + + #This is a table.column + my ($tab, $col) = split '\.', $attr; + $monsettings{'monattrs'}{$tab}{$col}{'val'} = $val; + $monsettings{'monattrs'}{$tab}{$col}{'matchtype'} = $matchtype; + } + + if (defined($request->{opt}->{r})) { + $monsettings{'retrycount'} = $request->{opt}->{r}; + } + if (defined($request->{opt}->{t})) { + $monsettings{'timeout'} = $request->{opt}->{t}; + } + + #Set the default values + foreach my $attr (keys %defaultattrs) { + if ((!defined($monsettings{$attr})) || ($monsettings{$attr} eq '')) { + $monsettings{$attr} = $defaultattrs{$attr}; + } + } + if(!defined($monsettings{'monattrs'}) || (scalar(keys %{$monsettings{'monattrs'}}) == 0)) { + $monsettings{'monattrs'}{'nodelist'}{'status'}{'val'} = "booted"; + $monsettings{'monattrs'}{'nodelist'}{'status'}{'matchtype'} = "match"; + } + + #Initialize the %{$monsettings{'nodes'}} hash + foreach my $node (@monnodes) { + foreach my $tab (keys %{$monsettings{'monattrs'}}) { + foreach my $col (keys %{$monsettings{'monattrs'}{$tab}}) { + $monsettings{'nodes'}{$node}{'status'}{$tab}{$col} = ''; + } + } + } + return \%monsettings; +} +#------------------------------------------------------------------------------- + +=head3 monitor_installation + Monitoring os installation progress. + Arguments: + $request: request hash + Returns: + 0 - ok + 1 - failed + Globals: + none + Example: + my $rc=xCAT::Utils->monitor_installation($opt(m)) + Comments: + none + +=cut + +#------------------------------------------------------------------------------- +sub monitor_installation() +{ + my ($class, $request, $monsettings) = @_; + my $callback = $request->{callback}; + + my $mstring = $request->{opt}->{m}; + #This is the first time the monitor_installation is called, + +# my $rsp={}; +# my $monnodes = join ',', @monitornodes; +# $rsp->{data}->[0] = "Start monitoring the installation progress with settings \"$mstring\" for nodes $monnodes"; +# xCAT::MsgUtils->message("I", $rsp, $callback); + + $monsettings->{'timeelapsed'} = 0; + while(($monsettings->{'timeelapsed'} < $monsettings->{'timeout'}) &&(scalar(keys %{$monsettings->{'nodes'}}))) { + #polling interval is 1 minute, + #do not do the first check until 1 minute after the os installation starts + sleep 5; #TODO, change it to 60 before checkin code + + + #update the timeelapsed + $monsettings->{'timeelapsed'}++; + + my @monitornodes = keys %{$monsettings->{'nodes'}}; + # Look up tables, do not look up the same table more than once + my %tabattrs = (); + foreach my $tab (keys %{$monsettings->{'monattrs'}}) { + foreach my $col (keys %{$monsettings->{'monattrs'}->{$tab}}) { + if (!grep(/^$col$/, @{$tabattrs{$tab}})) { + push @{$tabattrs{$tab}}, $col; + } + } + } + + foreach my $node (keys %{$monsettings->{'nodes'}}) { + foreach my $montable (keys %tabattrs) { + #Get the new status of the node + my $montab_ref = xCAT::Table->new($montable); + if ($montab_ref) { + my @attrs = @{$tabattrs{$montable}}; + my $tabdata = $montab_ref->getNodesAttribs(\@monitornodes, \@attrs); + foreach my $attr (@{$tabattrs{$montable}}) { + # nodestatus changed, print a message + if (($monsettings->{'nodes'}->{$node}->{'status'}->{$montable}->{$attr} ne '') + && ($monsettings->{'nodes'}->{$node}->{'status'}->{$montable}->{$attr} ne $tabdata->{$node}->[0]->{$attr})) { + my $rsp={}; + $rsp->{data}->[0] = "$node $montable.$attr: $monsettings->{'nodes'}->{$node}->{'status'}->{$montable}->{$attr} => $tabdata->{$node}->[0]->{$attr}"; + xCAT::MsgUtils->message("I", $rsp, $callback); + } + #set the new status + $monsettings->{'nodes'}->{$node}->{'status'}->{$montable}->{$attr} = $tabdata->{$node}->[0]->{$attr}; + } + $montab_ref->close(); + } else { #can not open the table + my $rsp={}; + $rsp->{data}->[0] = "Open table $montable failed"; + xCAT::MsgUtils->message("E", $rsp, $callback); + return (); + } + } + #expected status?? + my $statusmatch = 1; + foreach my $temptab (keys %{$monsettings->{'monattrs'}}) { + foreach my $tempcol (keys %{$monsettings->{'monattrs'}->{$temptab}}) { + my $currentstatus = $monsettings->{'nodes'}->{$node}->{'status'}->{$temptab}->{$tempcol}; + my $expectedstatus = $monsettings->{'monattrs'}->{$temptab}->{$tempcol}->{'val'}; + my $matchtype = $monsettings->{'monattrs'}->{$temptab}->{$tempcol}->{'matchtype'}; + #regular expression + if($matchtype eq 'match') { + if ($currentstatus ne $expectedstatus) { + $statusmatch = 0; + } + } elsif($matchtype eq 'regex') { + if ($currentstatus !~ /$expectedstatus/) { + $statusmatch = 0; + } + } + } #end foreach + } #end foreach + if ($statusmatch == 1) { + my $rsp={}; + $rsp->{data}->[0] = "$node: Reached the expected status"; + xCAT::MsgUtils->message("I", $rsp, $callback); + delete $monsettings->{'nodes'}->{$node}; + } + + + } #end foreach my $node + } #end while + + if(scalar(keys %{$monsettings->{'nodes'}}) > 0) + { + foreach my $n (keys %{$monsettings->{'nodes'}}) { + my $rsp={}; + $rsp->{data}->[0] = "$n: does not transit to the expected status"; + xCAT::MsgUtils->message("E",$rsp, $callback); + } + } + return $monsettings; +} 1;