xCAT/xcat-core
2
0
Fork 0
mirror of https://github.com/xcat2/xcat-core.git synced 2025-06-12 17:30:19 +00:00
Code Issues Releases Wiki Activity

Merge branch 'master' of github.com:xcat2/xcat-core into E2808

Browse Source
This commit is contained in:
XuWei
2017-04-12 23:28:45 -04:00
parent e5b3e4786e 80cc015923
commit df9e65c4b9
5 changed files with 104 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
docs/source/guides/admin-guides/references/man1/rflash.1.rst
View File

@ -50,7 +50,7 @@ OpenPOWER BMC specific:
=======================
\ **rflash**\ \ *noderange*\ \ *hpm_file_path*\ [\ **-c | -**\ **-check**\ ] [\ **-V**\ ]
\ **rflash**\ \ *noderange*\ \ *hpm_file_path*\ [\ **-c | -**\ **-check**\ ] [\ **-**\ **-retry=**\ \ *count*\ ] [\ **-V**\ ]
@ -174,6 +174,12 @@ The command will update firmware for OpenPOWER BMC when given an OpenPOWER node
\ **-**\ **-retry=**\ \ *count*\
Specify number of times to retry the update if failure is detected. Default value is 2. Value of 0 can be used to indicate no retries.
\ **-v|-**\ **-version**\
Displays the command's version.

6
xCAT-client/pods/man1/rflash.1.pod
View File

@ -24,7 +24,7 @@ B<rflash> I<noderange> I<http_directory>
=head2 OpenPOWER BMC specific:
B<rflash> I<noderange> I<hpm_file_path> [B<-c>|B<--check>] [B<-V>]
B<rflash> I<noderange> I<hpm_file_path> [B<-c>|B<--check>] [B<--retry=>I<count>] [B<-V>]
=head1 B<Description>
@ -114,6 +114,10 @@ Used to commit the flash image in the temporary side of the chip to the permanen
Used to recover the flash image in the permanent side of the chip to the temporary side for both managed systems and power subsystems.
=item B<--retry=>I<count>
Specify number of times to retry the update if failure is detected. Default value is 2. Value of 0 can be used to indicate no retries.
=item B<-v|--version>
Displays the command's version.

92
xCAT-server/lib/xcat/plugins/ipmi.pm
View File

@ -1766,6 +1766,7 @@ sub do_firmware_update {
my $ret;
my $ipmitool_ver;
my $verbose = 0;
my $retry = 2;
my $verbose_opt;
$ret = get_ipmitool_version(\$ipmitool_ver);
exit $ret if $ret < 0;
@ -1834,8 +1835,6 @@ sub do_firmware_update {
if ($bmc_password) {
$pre_cmd = $pre_cmd . " -P $bmc_password";
}
xCAT::SvrUtils::sendmsg("rflash started, please wait.......",
$callback, $sessdata->{node}, %allerrornodes);
# check for 8335-GTB Model Type to adjust buffer size
my $buffer_size = "30000";
@ -1849,7 +1848,7 @@ sub do_firmware_update {
$buffer_size = "15000";
}
# check verbose and buffersize options
# check verbose, buffersize, and retry options
for my $opt (@{$sessdata->{'extraargs'}}) {
if ($opt =~ /-V{1,4}/) {
$verbose_opt = lc($opt);
@ -1862,8 +1861,18 @@ sub do_firmware_update {
$buffer_size = $buffer_value;
}
}
if ($opt =~ /retry=/) {
my ($attribute, $retry_value) = split(/=/, $opt);
if ($retry_value) {
# retry option was passed in, reset the default
$retry = $retry_value;
}
}
}
xCAT::SvrUtils::sendmsg("rflash started, upgrade failure will be retried up to $retry times. Please wait...",
$callback, $sessdata->{node}, %allerrornodes);
# check for 8335-GTB Firmware above 1610A release. If below, exit
if ($output =~ /8335-GTB/) {
$cmd = $pre_cmd . " fru print 47";
@ -1883,6 +1892,9 @@ sub do_firmware_update {
}
}
RETRY_UPGRADE:
my $failed_upgrade = 0;
# step 1 power off
$cmd = $pre_cmd . " chassis power off";
xCAT::SvrUtils::sendmsg("Preparing to upgrade firmware, powering chassis off...", $callback, $sessdata->{node}, %allerrornodes);
@ -1925,9 +1937,21 @@ sub do_firmware_update {
$callback, $sessdata->{node});
$output = xCAT::Utils->runcmd($cmd, -1);
# if upgrade command failed and we exausted number of retries
# report an error, exit to the caller and leave node in powered off state
# otherwise report an error, power on the node and try upgrade again
if ($::RUNCMD_RC != 0) {
$exit_with_error_func->($sessdata->{node}, $callback,
"Running ipmitool command $cmd failed: $output");
if ($retry == 0) {
# No more retries left, report and error and exit
$exit_with_error_func->($sessdata->{node}, $callback,
"Running ipmitool command $cmd failed: $output");
}
else {
# Error upgrading, set a flag to attempt a retry
xCAT::SvrUtils::sendmsg("Running ipmitool command $cmd failed: $output", $callback, $sessdata->{node}, %allerrornodes);
$failed_upgrade = 1;
}
}
# step 5 power on
@ -1937,15 +1961,65 @@ sub do_firmware_update {
"Timeout to check the bmc status");
}
xCAT::SvrUtils::sendmsg("Firmware updated, powering chassis on to populate FRU information...", $callback, $sessdata->{node}, %allerrornodes);
if ($failed_upgrade) {
xCAT::SvrUtils::sendmsg("Firmware update failed, powering chassis on for a retry. This can take several minutes. $retry retries left ...", $callback, $sessdata->{node}, %allerrornodes);
}
else {
xCAT::SvrUtils::sendmsg("Firmware updated, powering chassis on to populate FRU information...", $callback, $sessdata->{node}, %allerrornodes);
}
$cmd = $pre_cmd . " chassis power on";
$output = xCAT::Utils->runcmd($cmd, -1);
if ($::RUNCMD_RC != 0) {
$exit_with_error_func->($sessdata->{node}, $callback,
"Running ipmitool command $cmd failed: $output");
}
$exit_with_success_func->($sessdata->{node}, $callback,
"Success updating firmware.");
my $node_ready_for_retry = 0;
if ($failed_upgrade) {
# Update has failed in step 4. Wait for node to reboot and try again
if ($verbose) {
xCAT::SvrUtils::sendmsg("Sleeping for a few min waiting for node to power on before attempting a retry", $callback, $sessdata->{node}, %allerrornodes);
}
sleep(300); # sleep for 5 min for node to reboot
# Start testing every 10 sec for node to be booted. Give up after 5 min.
foreach (1..30) {
# Test node is booted in to OS
$cmd = "nodestat $sessdata->{node} | grep sshd";
$output = xCAT::Utils->runcmd($cmd, -1);
if ($::RUNCMD_RC == 0) {
# Node is ready to retry an upgrage
if ($verbose) {
xCAT::SvrUtils::sendmsg("Detected node booted. Will retry upgrade", $callback, $sessdata->{node}, %allerrornodes);
}
$node_ready_for_retry = 1;
last;
}
else {
# Still not booted, wait for 10 sec and try again
if ($verbose) {
xCAT::SvrUtils::sendmsg("Node still not ready, Test again in 10 sec.", $callback, $sessdata->{node}, %allerrornodes);
}
sleep(10);
}
}
if ($node_ready_for_retry) {
$retry--; # decrement number of retries left
# Yes, it is a goto statement here. Ugly, but removes the need to restructure
# the above block of code.
goto RETRY_UPGRADE;
}
else {
# After 10 min of waiting node has not rebooted. Give up retrying.
$exit_with_error_func->($sessdata->{node}, $callback,
"Giving up waiting for the node to reboot. No further retries will be attempted.");
}
}
else {
$exit_with_success_func->($sessdata->{node}, $callback,
"Success updating firmware.");
}
}
sub rflash {
@ -1958,7 +2032,7 @@ sub rflash {
if ($opt =~ /^(-c|--check)$/i) {
$sessdata->{subcommand} = "check";
# support verbose options for ipmitool command
} elsif ($opt !~ /.*\.hpm$/i && $opt !~ /^-V{1,4}$/ && $opt !~ /^--buffersize=/) {
} elsif ($opt !~ /.*\.hpm$/i && $opt !~ /^-V{1,4}$/ && $opt !~ /^--buffersize=/ && $opt !~ /^--retry=/) {
$callback->({ error => "The option $opt is not supported",
errorcode => 1 });
return;

12
xCAT-server/lib/xcat/plugins/openbmc.pm
View File

@ -273,6 +273,8 @@ sub process_request {
$cookie_jar = HTTP::Cookies->new({});
$async = HTTP::Async->new(
cookie_jar => $cookie_jar,
timeout => 10,
max_request_time => 60,
ssl_options => {
SSL_verify_mode => 0,
},
@ -353,7 +355,7 @@ sub parse_args {
$check = unsupported($callback); if (ref($check) eq "ARRAY") { return $check; }
$subcommand = "all" if (!defined($ARGV[0]));
unless ($subcommand =~ /^cpu$|^dimm$|^bios$|^model$|^serial$|^all$/) {
unless ($subcommand =~ /^cpu$|^dimm$|^model$|^serial$|^firm$|^all$/) {
return ([ 1, "Unsupported command: $command $subcommand" ]);
}
} elsif ($command eq "getopenbmccons") {
@ -423,7 +425,7 @@ sub parse_command_status {
$subcommand = "all";
}
if ($subcommand eq "cpu" or $subcommand eq "dimm" or $subcommand eq "bios" or $subcommand eq "model" or $subcommand eq "serial" or $subcommand eq "all") {
if ($subcommand eq "cpu" or $subcommand eq "dimm" or $subcommand eq "firm" or $subcommand eq "model" or $subcommand eq "serial" or $subcommand eq "all") {
$next_status{LOGIN_RESPONSE} = "RINV_REQUEST";
$next_status{RINV_REQUEST} = "RINV_RESPONSE";
$status_info{RINV_RESPONSE}{argv} = "$subcommand";
@ -602,6 +604,8 @@ sub deal_with_response {
delete $handle_id_node{$handle_id};
print "$node: DEBUG " . lc ($node_info{$node}{cur_status}) . " " . $response->status_line . "\n";
if ($response->status_line ne "200 OK") {
my $error;
if ($response->status_line eq "503 Service Unavailable") {
@ -621,8 +625,6 @@ sub deal_with_response {
return;
}
print "$node: DEBUG " . lc ($node_info{$node}{cur_status}) . " " . $response->status_line . "\n";
$status_info{ $node_info{$node}{cur_status} }->{process}->($node, $response);
return;
@ -755,7 +757,7 @@ sub rinv_response {
xCAT::SvrUtils::sendmsg("$serialnumber", $callback, $node);
next;
}
} elsif ($grep_string eq "all" or $key_url =~ /\/$grep_string/) {
} elsif ($grep_string eq "all" or $key_url =~ /\/$grep_string/ or ($grep_string eq "firm" and defined($content{Name}) and $content{Name} eq "OpenPOWER Firmware")) {
if ($key_url =~ /\/(cpu\d*)\/(\w+)/) {
$src = "$1 $2";
} else {

3
xCAT-test/xcattest
View File

@ -907,7 +907,8 @@ sub runcase
#by
my $runstart = timelocal(localtime());
log_this("\nRUN:$cmd");
my $runstartstr = scalar(localtime());
log_this("\nRUN:$cmd [$runstartstr]");
push(@record, "\nRUN:$cmd");
@output = &runcmd($cmd);
$rc = $::RUNCMD_RC;