fe927896fc
git-svn-id: https://svn.code.sf.net/p/xcat/code/xcat-core/trunk@7470 8638fb3e-16cb-4fca-ae20-7b5d299a9bcd
2393 lines
78 KiB
Perl
2393 lines
78 KiB
Perl
#!/usr/bin/env perl -w
|
|
# IBM(c) 2007 EPL license http://www.eclipse.org/legal/epl-v10.html
|
|
#####################################################
|
|
#
|
|
# xCAT plugin package to handle rolling updates
|
|
#
|
|
#####################################################
|
|
|
|
package xCAT_plugin::rollupdate;
|
|
|
|
BEGIN
|
|
{
|
|
$::XCATROOT = $ENV{'XCATROOT'} ? $ENV{'XCATROOT'} : '/opt/xcat';
|
|
}
|
|
|
|
require xCAT::NodeRange;
|
|
require xCAT::NameRange;
|
|
require xCAT::Table;
|
|
require Data::Dumper;
|
|
require Getopt::Long;
|
|
require xCAT::MsgUtils;
|
|
require File::Path;
|
|
use strict;
|
|
use warnings;
|
|
|
|
#
|
|
# Globals
|
|
#
|
|
$::LOGDIR="/var/log/xcat";
|
|
$::LOGFILE="rollupdate.log";
|
|
|
|
#------------------------------------------------------------------------------
|
|
|
|
=head1 rollupdate
|
|
|
|
This program module file supports the cluster rolling update functions.
|
|
|
|
Supported commands:
|
|
rollupdate - Create scheduler job command files and submit the jobs
|
|
runrollupdate - Reboot the updategroup in response to request from scheduler
|
|
job
|
|
|
|
If adding to this file, please take a moment to ensure that:
|
|
|
|
1. Your contrib has a readable pod header describing the purpose and use of
|
|
the subroutine.
|
|
|
|
2. Your contrib is under the correct heading and is in alphabetical order
|
|
under that heading.
|
|
|
|
3. You have run tidypod on this file and saved the html file
|
|
|
|
=cut
|
|
|
|
#------------------------------------------------------------------------------
|
|
|
|
=head2 Cluster Rolling Update
|
|
|
|
=cut
|
|
|
|
#------------------------------------------------------------------------------
|
|
|
|
#----------------------------------------------------------------------------
|
|
|
|
=head3 handled_commands
|
|
|
|
Return a list of commands handled by this plugin
|
|
|
|
=cut
|
|
|
|
#-----------------------------------------------------------------------------
|
|
sub handled_commands {
|
|
return {
|
|
rollupdate => "rollupdate",
|
|
runrollupdate => "rollupdate"
|
|
};
|
|
}
|
|
|
|
#----------------------------------------------------------------------------
|
|
|
|
=head3 preprocess_request
|
|
|
|
|
|
Arguments:
|
|
|
|
Returns:
|
|
0 - OK
|
|
1 - error
|
|
Globals:
|
|
|
|
Error:
|
|
|
|
Example:
|
|
|
|
Comments:
|
|
=cut
|
|
|
|
#-----------------------------------------------------------------------------
|
|
sub preprocess_request {
|
|
|
|
# set management node as server for all requests
|
|
# any requests sent to service node need to get
|
|
# get sent up to the MN
|
|
|
|
my $req = shift;
|
|
unless ( defined( $req->{_xcatdest} ) ) {
|
|
$req->{_xcatdest} = xCAT::Utils->get_site_Master();
|
|
}
|
|
return [$req];
|
|
}
|
|
|
|
#----------------------------------------------------------------------------
|
|
|
|
=head3 process_request
|
|
|
|
Process the rolling update commands
|
|
|
|
Arguments:
|
|
|
|
Returns:
|
|
0 - OK
|
|
1 - error
|
|
Globals:
|
|
|
|
Error:
|
|
|
|
Example:
|
|
|
|
Comments:
|
|
=cut
|
|
|
|
#-----------------------------------------------------------------------------
|
|
sub process_request {
|
|
$::request = shift;
|
|
$::CALLBACK = shift;
|
|
$::SUBREQ = shift;
|
|
my $ret;
|
|
|
|
# globals used by all subroutines.
|
|
$::command = $::request->{command}->[0];
|
|
$::args = $::request->{arg};
|
|
$::stdindata = $::request->{stdin}->[0];
|
|
|
|
# figure out which cmd and call the subroutine to process
|
|
if ( $::command eq "rollupdate" ) {
|
|
$ret = &rollupdate($::request);
|
|
}
|
|
elsif ( $::command eq "runrollupdate" ) {
|
|
$ret = &runrollupdate($::request);
|
|
}
|
|
|
|
return $ret;
|
|
}
|
|
|
|
#----------------------------------------------------------------------------
|
|
|
|
=head3 rollupdate_usage
|
|
|
|
Arguments:
|
|
Returns:
|
|
Globals:
|
|
|
|
Error:
|
|
|
|
Example:
|
|
|
|
Comments:
|
|
=cut
|
|
|
|
#-----------------------------------------------------------------------------
|
|
|
|
# display the usage
|
|
sub rollupdate_usage {
|
|
my $rsp;
|
|
push @{ $rsp->{data} },
|
|
"\nUsage: rollupdate - Submit cluster rolling update jobs \n";
|
|
push @{ $rsp->{data} }, " rollupdate [-h | --help | -?] \n";
|
|
push @{ $rsp->{data} },
|
|
" rollupdate [-V | --verbose] [-v | --version] \n ";
|
|
push @{ $rsp->{data} },
|
|
" <STDIN> - stanza file, see /opt/xcat/share/xcat/rollupdate/rollupdate.input.sample";
|
|
push @{ $rsp->{data} }, " for example \n";
|
|
xCAT::MsgUtils->message( "I", $rsp, $::CALLBACK );
|
|
return 0;
|
|
}
|
|
|
|
|
|
#----------------------------------------------------------------------------
|
|
|
|
=head3 runrollupdate_usage
|
|
|
|
Arguments:
|
|
Returns:
|
|
Globals:
|
|
|
|
Error:
|
|
|
|
Example:
|
|
|
|
Comments:
|
|
=cut
|
|
|
|
#-----------------------------------------------------------------------------
|
|
|
|
# display the usage
|
|
sub runrollupdate_usage {
|
|
my $rsp;
|
|
push @{ $rsp->{data} },
|
|
"\nUsage: runrollupdate - Run submitted cluster rolling update job \n";
|
|
push @{ $rsp->{data} }, " runrollupdate [-h | --help | -?] \n";
|
|
push @{ $rsp->{data} },
|
|
" runrollupdate [-V | --verbose] [-v | --version] scheduler datafile \n ";
|
|
push @{ $rsp->{data} },
|
|
xCAT::MsgUtils->message( "I", $rsp, $::CALLBACK );
|
|
return 0;
|
|
}
|
|
|
|
#----------------------------------------------------------------------------
|
|
|
|
=head3 processArgs
|
|
|
|
Process the command line and any input files provided on cmd line.
|
|
|
|
Arguments:
|
|
|
|
Returns:
|
|
0 - OK
|
|
1 - just print usage
|
|
2 - error
|
|
Globals:
|
|
|
|
Error:
|
|
|
|
Example:
|
|
|
|
Comments:
|
|
=cut
|
|
|
|
#-----------------------------------------------------------------------------
|
|
sub processArgs {
|
|
my $gotattrs = 0;
|
|
|
|
if ( defined( @{$::args} ) ) {
|
|
@ARGV = @{$::args};
|
|
}
|
|
else {
|
|
|
|
# return 2; # can run with no args right now
|
|
}
|
|
|
|
# if (scalar(@ARGV) <= 0) {
|
|
# return 2;
|
|
# }
|
|
|
|
# parse the options
|
|
# options can be bundled up like -vV, flag unsupported options
|
|
Getopt::Long::Configure( "bundling", "no_ignore_case", "no_pass_through" );
|
|
Getopt::Long::GetOptions(
|
|
'help|h|?' => \$::opt_h,
|
|
'test|t' => \$::opt_t,
|
|
'verbose|V' => \$::opt_V,
|
|
'version|v' => \$::opt_v,
|
|
);
|
|
|
|
# Option -h for Help
|
|
# if user specifies "-t" & "-h" they want a list of valid attrs
|
|
if ( defined($::opt_h) ) {
|
|
return 2;
|
|
}
|
|
|
|
# opt_t not yet supported
|
|
if ( defined($::opt_t) ) {
|
|
$::test = 1;
|
|
$::TEST = 1;
|
|
# my $rsp;
|
|
# push @{ $rsp->{data} }, "The \'-t\' option is not yet implemented.";
|
|
# xCAT::MsgUtils->message( "I", $rsp, $::CALLBACK );
|
|
# return 2;
|
|
}
|
|
|
|
# Option -v for version
|
|
if ( defined($::opt_v) ) {
|
|
my $rsp;
|
|
my $version = xCAT::Utils->Version();
|
|
push @{ $rsp->{data} }, "$::command - $version\n";
|
|
xCAT::MsgUtils->message( "I", $rsp, $::CALLBACK );
|
|
return 1; # no usage - just exit
|
|
}
|
|
|
|
# Option -V for verbose output
|
|
if ( defined($::opt_V) ) {
|
|
$::verbose = 1;
|
|
$::VERBOSE = 1;
|
|
}
|
|
|
|
|
|
if ( $::command eq "rollupdate" ) {
|
|
# process @ARGV
|
|
#while (my $a = shift(@ARGV))
|
|
#{
|
|
# no args for command yet
|
|
#}
|
|
|
|
# process the <stdin> input file
|
|
if ( defined($::stdindata) ) {
|
|
my $rc = readFileInput($::stdindata);
|
|
if ($rc) {
|
|
my $rsp;
|
|
push @{ $rsp->{data} }, "Could not process file input data.\n";
|
|
xCAT::MsgUtils->message( "I", $rsp, $::CALLBACK );
|
|
return 1;
|
|
}
|
|
}
|
|
else {
|
|
# No <stdin> stanza file, print usage
|
|
return 2;
|
|
}
|
|
}
|
|
elsif ( $::command eq "runrollupdate" ) {
|
|
# process @ARGV
|
|
$::scheduler = shift(@ARGV);
|
|
$::datafile = shift(@ARGV);
|
|
$::ll_reservation_id = shift(@ARGV);
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
#----------------------------------------------------------------------------
|
|
|
|
=head3 readFileInput
|
|
|
|
Process the command line input piped in from a stanza file.
|
|
|
|
Arguments:
|
|
Returns:
|
|
0 - OK
|
|
1 - error
|
|
Globals:
|
|
Error:
|
|
Example:
|
|
|
|
Comments:
|
|
Set %::FILEATTRS
|
|
(i.e.- $::FILEATTRS{attr}=[val])
|
|
|
|
=cut
|
|
|
|
#-----------------------------------------------------------------------------
|
|
sub readFileInput {
|
|
my ($filedata) = @_;
|
|
|
|
my @lines = split /\n/, $filedata;
|
|
|
|
my $prev_attr = "";
|
|
foreach my $l (@lines) {
|
|
|
|
# skip blank and comment lines
|
|
next if ( $l =~ /^\s*$/ || $l =~ /^\s*#/ );
|
|
|
|
# process a real line
|
|
if ( $l =~ /^\s*(\w+)\s*=\s*(.*)\s*/ ) {
|
|
my $attr = $1;
|
|
my $val = $2;
|
|
$attr =~ s/^\s*//; # Remove any leading whitespace
|
|
$attr =~ s/\s*$//; # Remove any trailing whitespace
|
|
$attr =~ tr/A-Z/a-z/; # Convert to lowercase
|
|
$val =~ s/^\s*//;
|
|
$val =~ s/\s*$//;
|
|
|
|
# Convert the following values to lowercase
|
|
if ( ($attr eq 'scheduler') ||
|
|
($attr eq 'updateall') ||
|
|
($attr eq 'shutdownrequired') ) {
|
|
$val =~ tr/A-Z/a-z/;
|
|
}
|
|
# set the value in the hash for this entry
|
|
push( @{ $::FILEATTRS{$attr} }, $val );
|
|
if (($prev_attr eq "prescript") && ($attr ne "prescriptnodes")) {
|
|
push ( @{ $::FILEATTRS{'prescriptnodes'} }, 'ALL_NODES_IN_UPDATEGROUP' );
|
|
}
|
|
if (($prev_attr eq "outofbandcmd") && ($attr ne "outofbandnodes")) {
|
|
push ( @{ $::FILEATTRS{'outofbandnodes'} }, 'ALL_NODES_IN_UPDATEGROUP' );
|
|
}
|
|
$prev_attr = $attr;
|
|
}
|
|
} # end while - go to next line
|
|
|
|
return 0;
|
|
}
|
|
|
|
#----------------------------------------------------------------------------
|
|
|
|
=head3 rollupdate
|
|
|
|
Support for the xCAT rollupdate command.
|
|
|
|
Arguments:
|
|
Returns:
|
|
0 - OK
|
|
1 - error
|
|
Globals:
|
|
|
|
Error:
|
|
|
|
Example:
|
|
|
|
Comments:
|
|
=cut
|
|
|
|
#-----------------------------------------------------------------------------
|
|
|
|
sub rollupdate {
|
|
|
|
my $rc = 0;
|
|
my $error = 0;
|
|
|
|
# process the command line
|
|
$rc = &processArgs;
|
|
if ( $rc != 0 ) {
|
|
|
|
# rc: 0 - ok, 1 - return, 2 - help, 3 - error
|
|
if ( $rc != 1 ) {
|
|
&rollupdate_usage;
|
|
}
|
|
return ( $rc - 1 );
|
|
}
|
|
if ($::VERBOSE) {
|
|
my $rsp;
|
|
push @{ $rsp->{data} }, "Running rollupdate command... ";
|
|
xCAT::MsgUtils->message( "I", $rsp, $::CALLBACK );
|
|
open (RULOG, ">>$::LOGDIR/$::LOGFILE");
|
|
print RULOG "\n\n";
|
|
print RULOG localtime()." Running rollupdate command...\n";
|
|
close (RULOG);
|
|
}
|
|
|
|
#
|
|
# Build updategroup nodelists
|
|
#
|
|
my %updategroup;
|
|
if ( defined($::FILEATTRS{updateall}[0]) &&
|
|
( ($::FILEATTRS{updateall}[0] eq 'yes') ||
|
|
($::FILEATTRS{updateall}[0] eq 'y' ) ) ) {
|
|
if ( defined($::FILEATTRS{updateall_nodes}[0])){
|
|
my $ugname = "UPDATEALL".time();
|
|
my $ugval = $::FILEATTRS{updateall_nodes}[0];
|
|
@{ $updategroup{$ugname} } = xCAT::NodeRange::noderange($ugval);
|
|
} else {
|
|
my $rsp;
|
|
push @{ $rsp->{data} },
|
|
"Error processing stanza input: updateall=yes but no updateall_nodes specified. ";
|
|
xCAT::MsgUtils->message( "E", $rsp, $::CALLBACK );
|
|
return 1;
|
|
}
|
|
} else {
|
|
foreach my $ugline ( @{ $::FILEATTRS{'updategroup'} } ) {
|
|
my ( $ugname, $ugval ) = split( /\(/, $ugline );
|
|
$ugval =~ s/\)$//; # remove trailing ')'
|
|
@{ $updategroup{$ugname} } = xCAT::NodeRange::noderange($ugval);
|
|
if ( xCAT::NodeRange::nodesmissed() ) {
|
|
my $rsp;
|
|
push @{ $rsp->{data} }, "Error processing stanza line: ";
|
|
push @{ $rsp->{data} }, "updategroup=" . $ugline;
|
|
push @{ $rsp->{data} }, "Invalid nodes in noderange: "
|
|
. join( ',', xCAT::NodeRange::nodesmissed() );
|
|
xCAT::MsgUtils->message( "E", $rsp, $::CALLBACK );
|
|
return 1;
|
|
}
|
|
if ($::VERBOSE) {
|
|
my $rsp;
|
|
my $prt_ugn = join(',',@{$updategroup{$ugname}});
|
|
push @{ $rsp->{data} }, "Creating update group $ugname with nodes: ";
|
|
push @{ $rsp->{data} }, $prt_ugn;
|
|
xCAT::MsgUtils->message( "I", $rsp, $::CALLBACK );
|
|
open (RULOG, ">>$::LOGDIR/$::LOGFILE");
|
|
print RULOG localtime()." Creating update group $ugname with nodes: \n";
|
|
print RULOG "$prt_ugn\n";
|
|
close (RULOG);
|
|
}
|
|
}
|
|
|
|
foreach my $mgline ( @{ $::FILEATTRS{'mapgroups'} } ) {
|
|
my @ugnamelist = xCAT::NameRange::namerange( $mgline, 0 );
|
|
foreach my $ugname (@ugnamelist) {
|
|
@{ $updategroup{$ugname} } = xCAT::NodeRange::noderange($ugname);
|
|
if ( xCAT::NodeRange::nodesmissed() ) {
|
|
my $rsp;
|
|
push @{ $rsp->{data} }, "Error processing stanza line: ";
|
|
push @{ $rsp->{data} }, "mapgroups=" . $mgline;
|
|
push @{ $rsp->{data} }, "Invalid nodes in group $ugname: "
|
|
. join( ',', xCAT::NodeRange::nodesmissed() );
|
|
xCAT::MsgUtils->message( "E", $rsp, $::CALLBACK );
|
|
return 1;
|
|
}
|
|
if ($::VERBOSE) {
|
|
my $rsp;
|
|
my $prt_ugn = join(',',@{$updategroup{$ugname}});
|
|
push @{ $rsp->{data} }, "Creating update group $ugname with nodes: ";
|
|
push @{ $rsp->{data} }, $prt_ugn;
|
|
xCAT::MsgUtils->message( "I", $rsp, $::CALLBACK );
|
|
open (RULOG, ">>$::LOGDIR/$::LOGFILE");
|
|
print RULOG localtime()." Creating update group $ugname with nodes: \n";
|
|
print RULOG "$prt_ugn\n";
|
|
close (RULOG);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
unless (%updategroup) {
|
|
my $rsp;
|
|
push @{ $rsp->{data} },
|
|
"Error processing stanza input: No updategroup or mapgroups entries found. ";
|
|
xCAT::MsgUtils->message( "E", $rsp, $::CALLBACK );
|
|
return 1;
|
|
}
|
|
|
|
#
|
|
# Build and submit scheduler jobs
|
|
#
|
|
my $scheduler = $::FILEATTRS{'scheduler'}[0];
|
|
if ( ( !$scheduler )
|
|
|| ( $scheduler eq "loadleveler" ) )
|
|
{
|
|
$rc = ll_jobs( \%updategroup );
|
|
}
|
|
else {
|
|
|
|
# TODO: support scheduler plugins here
|
|
my $rsp;
|
|
push @{ $rsp->{data} }, "Error processing stanza line: ";
|
|
push @{ $rsp->{data} }, "scheduler=" . $::FILEATTRS{'scheduler'}[0];
|
|
push @{ $rsp->{data} }, "Scheduler not supported";
|
|
xCAT::MsgUtils->message( "E", $rsp, $::CALLBACK );
|
|
return 1;
|
|
}
|
|
|
|
return $rc;
|
|
}
|
|
|
|
#----------------------------------------------------------------------------
|
|
|
|
=head3 ll_jobs
|
|
|
|
Build and submit LoadLeveler jobs
|
|
|
|
Arguments:
|
|
Returns:
|
|
0 - OK
|
|
1 - error
|
|
Globals:
|
|
|
|
Error:
|
|
|
|
Example:
|
|
|
|
Comments:
|
|
=cut
|
|
|
|
#-----------------------------------------------------------------------------
|
|
|
|
sub ll_jobs {
|
|
my $updategroup = shift;
|
|
my $rc = 0;
|
|
|
|
if ($::VERBOSE) {
|
|
my $rsp;
|
|
push @{ $rsp->{data} }, "Creating LL job command files ";
|
|
xCAT::MsgUtils->message( "I", $rsp, $::CALLBACK );
|
|
open (RULOG, ">>$::LOGDIR/$::LOGFILE");
|
|
print RULOG localtime()." Creating LL job command files \n";
|
|
close (RULOG);
|
|
}
|
|
|
|
$::updateall=0;
|
|
$::updateall_numperupdate=1;
|
|
if ( defined($::FILEATTRS{updateall}[0]) &&
|
|
( ($::FILEATTRS{updateall}[0] eq 'yes') ||
|
|
($::FILEATTRS{updateall}[0] eq 'y' ) ) ) {
|
|
$::updateall=1;
|
|
if ( defined($::FILEATTRS{updateall_numperupdate}[0]) ) {
|
|
$::updateall_numperupdate=$::FILEATTRS{updateall_numperupdate}[0];
|
|
}
|
|
}
|
|
|
|
|
|
# Create LL floating resources for mutual exclusion support
|
|
# and max_updates
|
|
if (&create_LL_mutex_resources > 0) {
|
|
return 1;
|
|
}
|
|
|
|
#
|
|
# Load job command file template
|
|
#
|
|
my $tmpl_file_name = $::FILEATTRS{'jobtemplate'}[0];
|
|
unless ( defined($tmpl_file_name) ) {
|
|
my $rsp;
|
|
push @{ $rsp->{data} },
|
|
"Error processing stanza input: No jobtemplate entries found. ";
|
|
xCAT::MsgUtils->message( "E", $rsp, $::CALLBACK );
|
|
return 1;
|
|
}
|
|
|
|
my $TMPL_FILE;
|
|
unless ( open( $TMPL_FILE, "<", $tmpl_file_name ) ) {
|
|
my $rsp;
|
|
push @{ $rsp->{data} },
|
|
"Error processing stanza input: jobtemplate file $tmpl_file_name not found. ";
|
|
xCAT::MsgUtils->message( "E", $rsp, $::CALLBACK );
|
|
return 1;
|
|
}
|
|
if ($::VERBOSE) {
|
|
my $rsp;
|
|
push @{ $rsp->{data} }, "Reading LL job template file $tmpl_file_name ";
|
|
xCAT::MsgUtils->message( "I", $rsp, $::CALLBACK );
|
|
open (RULOG, ">>$::LOGDIR/$::LOGFILE");
|
|
print RULOG localtime()." Reading LL job template file $tmpl_file_name \n";
|
|
close (RULOG);
|
|
}
|
|
my @lines = <$TMPL_FILE>;
|
|
close $TMPL_FILE;
|
|
|
|
# Query LL for list of machines and their status
|
|
### LL BUG WORKAROUND -- MUST SET LOADL_STATUS_LEVEL=MACHINE
|
|
my $cmd = "LOADL_STATUS_LEVEL=MACHINE llstatus -r %n %sta 2>/dev/null";
|
|
# my $cmd = "llstatus -r %n %sta 2>/dev/null";
|
|
if ($::VERBOSE) {
|
|
my $rsp;
|
|
push @{ $rsp->{data} }, "Running command: $cmd ";
|
|
xCAT::MsgUtils->message( "I", $rsp, $::CALLBACK );
|
|
open (RULOG, ">>$::LOGDIR/$::LOGFILE");
|
|
print RULOG localtime()." Running command: $cmd \n";
|
|
close (RULOG);
|
|
}
|
|
my @llstatus = xCAT::Utils->runcmd( $cmd, 0 );
|
|
if ( $::RUNCMD_RC != 0 ) {
|
|
my $rsp;
|
|
push @{ $rsp->{data} }, "Could not run llstatus command.";
|
|
push @{ $rsp->{data} }, @llstatus;
|
|
xCAT::MsgUtils->message( "E", $rsp, $::CALLBACK );
|
|
open (RULOG, ">>$::LOGDIR/$::LOGFILE");
|
|
print RULOG localtime()." Could not run llstatus command. \n";
|
|
print RULOG @llstatus;
|
|
close (RULOG);
|
|
return 1;
|
|
}
|
|
my %machines;
|
|
foreach my $machineline (@llstatus) {
|
|
my ( $mlong, $mshort, $mstatus );
|
|
( $mlong, $mstatus ) = split( /\!/, $machineline );
|
|
($mshort) = split( /\./, $mlong );
|
|
$machines{$mlong} = { mname => $mlong, mstatus => $mstatus };
|
|
if ( !( $mlong eq $mshort ) ) {
|
|
$machines{$mshort} = { mname => $mlong, mstatus => $mstatus };
|
|
}
|
|
}
|
|
|
|
#
|
|
# Generate job command file for each updategroup
|
|
#
|
|
# Get LL userid
|
|
my $lluser = $::FILEATTRS{scheduser}[0];
|
|
unless ( defined($lluser) ) {
|
|
my $rsp;
|
|
push @{ $rsp->{data} },
|
|
"Error processing stanza input: No scheduser entries found. ";
|
|
xCAT::MsgUtils->message( "E", $rsp, $::CALLBACK );
|
|
return 1;
|
|
}
|
|
my ( $login, $pass, $uid, $gid );
|
|
( $login, $pass, $uid, $gid ) = getpwnam($lluser);
|
|
unless ( defined($uid) ) {
|
|
my $rsp;
|
|
push @{ $rsp->{data} },
|
|
"Error processing stanza input: scheduser userid $lluser not defined in system. ";
|
|
xCAT::MsgUtils->message( "E", $rsp, $::CALLBACK );
|
|
return 1;
|
|
}
|
|
if ( &check_policy($lluser,'runrollupdate') ) {
|
|
my $rsp;
|
|
push @{ $rsp->{data} },
|
|
"Error processing stanza input: scheduser userid $lluser not listed in xCAT policy table for runrollupdate command. Add to policy table and ensure userid has ssh credentials for running xCAT commands. ";
|
|
xCAT::MsgUtils->message( "E", $rsp, $::CALLBACK );
|
|
return 1;
|
|
}
|
|
|
|
my $lljobs_dir = $::FILEATTRS{jobdir}[0];
|
|
unless ( defined($lljobs_dir) ) {
|
|
my $rsp;
|
|
push @{ $rsp->{data} },
|
|
"Error processing stanza input: No jobdir entries found. ";
|
|
xCAT::MsgUtils->message( "E", $rsp, $::CALLBACK );
|
|
return 1;
|
|
}
|
|
unless ( -d $lljobs_dir ) {
|
|
unless ( File::Path::mkpath($lljobs_dir) ) {
|
|
my $rsp;
|
|
push @{ $rsp->{data} }, "Could not create directory $lljobs_dir";
|
|
xCAT::MsgUtils->message( "E", $rsp, $::CALLBACK );
|
|
return 1;
|
|
}
|
|
unless ( chown( $uid, $gid, $lljobs_dir ) ) {
|
|
my $rsp;
|
|
push @{ $rsp->{data} },
|
|
"Could not change owner of directory $lljobs_dir to $lluser";
|
|
xCAT::MsgUtils->message( "E", $rsp, $::CALLBACK );
|
|
return 1;
|
|
}
|
|
}
|
|
|
|
my $nodestatus = $::FILEATTRS{bringupstatus}[0];
|
|
my $appstatus = $::FILEATTRS{bringupappstatus}[0];
|
|
my $df_statusline = "";
|
|
if ( defined($appstatus) ) {
|
|
$df_statusline = "bringupappstatus=$appstatus\n";
|
|
} elsif ( defined($nodestatus) ) {
|
|
$df_statusline = "bringupstatus=$nodestatus\n";
|
|
} else {
|
|
$df_statusline = "bringupstatus=booted\n";
|
|
}
|
|
|
|
my $run_if_down = "cancel";
|
|
if ( defined($::FILEATTRS{update_if_down}[0]) ) {
|
|
$run_if_down = $::FILEATTRS{update_if_down}[0];
|
|
$run_if_down =~ tr/[A-Z]/[a-z]/;
|
|
if ( $run_if_down eq 'y' ) { $run_if_down = 'yes'; }
|
|
if ( $::updateall && ($run_if_down eq 'yes') ) {
|
|
$run_if_down = 'cancel';
|
|
my $rsp;
|
|
push @{ $rsp->{data} }, "update_all=yes, but update_if_down is yes which is not allowed. update_if_down=cancel will be assumed. ";
|
|
xCAT::MsgUtils->message( "I", $rsp, $::CALLBACK );
|
|
}
|
|
}
|
|
|
|
my @calldirectly;
|
|
ugloop: foreach my $ugname ( keys %{$updategroup} ) {
|
|
|
|
$::updateall_feature = " ";
|
|
if ($::updateall) { $::updateall_feature = "XCAT_$ugname"; }
|
|
# Build node substitution strings
|
|
my ( $nodelist, $machinelist );
|
|
my $machinecount = 0;
|
|
foreach my $node ( @{ $updategroup->{$ugname} } ) {
|
|
if ( defined( $machines{$node} )
|
|
&& ( $machines{$node}{'mstatus'} eq "1" ) ) {
|
|
$machinelist .= " $machines{$node}{'mname'}";
|
|
$machinecount++;
|
|
$nodelist .= ",$node";
|
|
} elsif ( $run_if_down eq 'yes' ) {
|
|
if ( defined( $machines{$node} ) ) {
|
|
# llmkres -D will allow reserving down nodes as long
|
|
# as they are present in the machine list
|
|
$machinelist .= " $machines{$node}{'mname'}";
|
|
$machinecount++;
|
|
}
|
|
$nodelist .= ",$node";
|
|
} elsif ( $run_if_down eq 'cancel' ) {
|
|
my $rsp;
|
|
push @{ $rsp->{data} },
|
|
"Node $node is not active in LL and \"update_if_down=cancel\". Update for updategroup $ugname is canceled.";
|
|
xCAT::MsgUtils->message( "E", $rsp, $::CALLBACK );
|
|
++$rc;
|
|
next ugloop;
|
|
}
|
|
}
|
|
|
|
if ( defined($nodelist) ) { $nodelist =~ s/^\,//; }
|
|
# Build updategroup data file
|
|
my @ugdflines;
|
|
push (@ugdflines, "# xCAT Rolling Update data file for update group $ugname \n");
|
|
push (@ugdflines, "\n");
|
|
push (@ugdflines, "updategroup=$ugname\n");
|
|
if ($::updateall){
|
|
push (@ugdflines, "updatefeature=$::updateall_feature\n");
|
|
} else {
|
|
push (@ugdflines, "nodelist=$nodelist\n");
|
|
}
|
|
if (defined($::FILEATTRS{oldfeature}[0])){
|
|
push (@ugdflines, "oldfeature=$::FILEATTRS{oldfeature}[0]\n");
|
|
}
|
|
if (defined($::FILEATTRS{newfeature}[0])){
|
|
push (@ugdflines, "newfeature=$::FILEATTRS{newfeature}[0]\n");
|
|
}
|
|
push (@ugdflines, "\n");
|
|
push (@ugdflines, &get_prescripts($nodelist));
|
|
if (defined($::FILEATTRS{shutdowntimeout}[0])){
|
|
push (@ugdflines, "shutdowntimeout=$::FILEATTRS{shutdowntimeout}[0]\n");
|
|
}
|
|
push (@ugdflines, &get_outofband($nodelist));
|
|
push (@ugdflines, &get_bringuporder($nodelist));
|
|
push (@ugdflines, $df_statusline);
|
|
if (defined($::FILEATTRS{bringuptimeout}[0])){
|
|
push (@ugdflines, "bringuptimeout=$::FILEATTRS{bringuptimeout}[0]\n");
|
|
}
|
|
my $ugdf_file = $lljobs_dir . "/rollupdate_" . $ugname . ".data";
|
|
my $UGDFFILE;
|
|
unless ( open( $UGDFFILE, ">$ugdf_file" ) ) {
|
|
my $rsp;
|
|
push @{ $rsp->{data} }, "Could not open file $ugdf_file";
|
|
xCAT::MsgUtils->message( "E", $rsp, $::CALLBACK );
|
|
return 1;
|
|
}
|
|
if ($::VERBOSE) {
|
|
my $rsp;
|
|
push @{ $rsp->{data} }, "Writing xCAT rolling update data file $ugdf_file ";
|
|
xCAT::MsgUtils->message( "I", $rsp, $::CALLBACK );
|
|
open (RULOG, ">>$::LOGDIR/$::LOGFILE");
|
|
print RULOG localtime()." Writing xCAT rolling update data file $ugdf_file \n";
|
|
close (RULOG);
|
|
}
|
|
print $UGDFFILE @ugdflines;
|
|
close($UGDFFILE);
|
|
chown( $uid, $gid, $ugdf_file );
|
|
|
|
if ( defined($machinelist) || $::updateall ) {
|
|
my $llhl_file;
|
|
if ( !$::updateall ) {
|
|
$machinelist =~ s/^\s+//;
|
|
|
|
# Build LL hostlist file
|
|
my $hllines = $machinelist;
|
|
$hllines =~ s/"//g;
|
|
$hllines =~ s/\s+/\n/g;
|
|
$hllines .= "\n";
|
|
$llhl_file = $lljobs_dir . "/rollupdate_" . $ugname . ".hostlist";
|
|
my $HLFILE;
|
|
unless ( open( $HLFILE, ">$llhl_file" ) ) {
|
|
my $rsp;
|
|
push @{ $rsp->{data} }, "Could not open file $llhl_file";
|
|
xCAT::MsgUtils->message( "E", $rsp, $::CALLBACK );
|
|
return 1;
|
|
}
|
|
if ($::VRBOSE) {
|
|
my $rsp;
|
|
push @{ $rsp->{data} }, "Writing LL hostlist file $llhl_file ";
|
|
xCAT::MsgUtils->message( "I", $rsp, $::CALLBACK );
|
|
}
|
|
print $HLFILE $hllines;
|
|
close($HLFILE);
|
|
chown( $uid, $gid, $llhl_file );
|
|
}
|
|
|
|
# Build reservation callback script
|
|
my @rcblines;
|
|
my $rcbcmd = $::FILEATTRS{'reservationcallback'}[0];
|
|
if (!defined($rcbcmd)){ $rcbcmd = "$::XCATROOT/bin/runrollupdate"; }
|
|
push (@rcblines, "#!/bin/sh \n");
|
|
push (@rcblines, "# LL Reservation Callback script for xCAT Rolling Update group $ugname \n");
|
|
push (@rcblines, "\n");
|
|
push (@rcblines, "if [ \"\$2\" == \"RESERVATION_ACTIVE\" ] ; then\n");
|
|
my $send_verbose = "";
|
|
if ($::VERBOSE) {$send_verbose="--verbose";}
|
|
push (@rcblines, " $rcbcmd $send_verbose loadleveler $ugdf_file \$1 &\n");
|
|
push (@rcblines, "fi \n");
|
|
push (@rcblines, "\n");
|
|
my $llrcb_file = $lljobs_dir . "/rollupdate_" . $ugname . ".rsvcb";
|
|
my $RCBFILE;
|
|
unless ( open( $RCBFILE, ">$llrcb_file" ) ) {
|
|
my $rsp;
|
|
push @{ $rsp->{data} }, "Could not open file $llrcb_file";
|
|
xCAT::MsgUtils->message( "E", $rsp, $::CALLBACK );
|
|
return 1;
|
|
}
|
|
if ($::VERBOSE) {
|
|
my $rsp;
|
|
push @{ $rsp->{data} }, "Writing LL reservation callback script $llrcb_file ";
|
|
xCAT::MsgUtils->message( "I", $rsp, $::CALLBACK );
|
|
open (RULOG, ">>$::LOGDIR/$::LOGFILE");
|
|
print RULOG localtime()." Writing xCAT rolling update data file $ugdf_file \n";
|
|
close (RULOG);
|
|
}
|
|
print $RCBFILE @rcblines;
|
|
close($RCBFILE);
|
|
chown( $uid, $gid, $llrcb_file );
|
|
chmod( 0700, $llrcb_file );
|
|
|
|
# Build output file
|
|
my $mutex_string = &get_mutex($ugname);
|
|
my $lastcount = 0;
|
|
my $llcount = $machinecount;
|
|
if ($::updateall) {
|
|
$lastcount = $machinecount % $::updateall_numperupdate;
|
|
$llcount = $::updateall_numperupdate;
|
|
}
|
|
my @jclines;
|
|
my @jclines2;
|
|
foreach my $line (@lines) {
|
|
my $jcline = $line;
|
|
my $jcline2;
|
|
$jcline =~ s/\[\[NODESET\]\]/$ugname/;
|
|
$jcline =~ s/\[\[JOBDIR\]\]/$lljobs_dir/;
|
|
if (defined($nodelist)){
|
|
$jcline =~ s/\[\[XNODELIST\]\]/$nodelist/;
|
|
} else {
|
|
$jcline =~ s/\[\[XNODELIST\]\]//;
|
|
}
|
|
if (defined($llhl_file)){
|
|
$jcline =~ s/\[\[LLHOSTFILE\]\]/$llhl_file/;
|
|
} else {
|
|
$jcline =~ s/\[\[LLHOSTFILE\]\]//;
|
|
}
|
|
if (defined($::FILEATTRS{oldfeature}[0])){
|
|
$jcline =~ s/\[\[OLDFEATURE\]\]/$::FILEATTRS{oldfeature}[0]/;
|
|
} else {
|
|
$jcline =~ s/\[\[OLDFEATURE\]\]//;
|
|
}
|
|
$jcline =~ s/\[\[UPDATEALLFEATURE\]\]/$::updateall_feature/;
|
|
$jcline =~ s/\[\[MUTEXRESOURCES\]\]/$mutex_string/;
|
|
# LL is VERY picky about extra blanks in Feature string
|
|
if ( $jcline =~ /Feature/ ) {
|
|
$jcline =~ s/\"\s+/\"/g;
|
|
$jcline =~ s/\s+\"/\"/g;
|
|
}
|
|
if ($lastcount) {
|
|
$jcline2 = $jcline;
|
|
$jcline2 =~ s/\[\[LLCOUNT\]\]/$lastcount/;
|
|
push( @jclines2, $jcline2 );
|
|
}
|
|
if ( $jcline =~ /\[\[LLCOUNT\]\]/ ) {
|
|
$jcline =~ s/\[\[LLCOUNT\]\]/$llcount/;
|
|
}
|
|
push( @jclines, $jcline );
|
|
}
|
|
my $lljob_file = $lljobs_dir . "/rollupdate_" . $ugname . ".cmd";
|
|
my $JOBFILE;
|
|
unless ( open( $JOBFILE, ">$lljob_file" ) ) {
|
|
my $rsp;
|
|
push @{ $rsp->{data} }, "Could not open file $lljob_file";
|
|
xCAT::MsgUtils->message( "E", $rsp, $::CALLBACK );
|
|
return 1;
|
|
}
|
|
if ($::VERBOSE) {
|
|
my $rsp;
|
|
push @{ $rsp->{data} }, "Writing LL job command file $lljob_file ";
|
|
xCAT::MsgUtils->message( "I", $rsp, $::CALLBACK );
|
|
open (RULOG, ">>$::LOGDIR/$::LOGFILE");
|
|
print RULOG localtime()." Writing LL job command file $lljob_file \n";
|
|
close (RULOG);
|
|
}
|
|
print $JOBFILE @jclines;
|
|
close($JOBFILE);
|
|
chown( $uid, $gid, $lljob_file );
|
|
my $lljob_file2 = $lljobs_dir . "/rollupdate_LAST_" . $ugname . ".cmd";
|
|
if ($lastcount) {
|
|
my $JOBFILE2;
|
|
unless ( open( $JOBFILE2, ">$lljob_file2" ) ) {
|
|
my $rsp;
|
|
push @{ $rsp->{data} }, "Could not open file $lljob_file2";
|
|
xCAT::MsgUtils->message( "E", $rsp, $::CALLBACK );
|
|
return 1;
|
|
}
|
|
if ($::VERBOSE) {
|
|
my $rsp;
|
|
push @{ $rsp->{data} }, "Writing LL job command file $lljob_file2 ";
|
|
xCAT::MsgUtils->message( "I", $rsp, $::CALLBACK );
|
|
open (RULOG, ">>$::LOGDIR/$::LOGFILE");
|
|
print RULOG localtime()." Writing LL job command file $lljob_file2 \n";
|
|
close (RULOG);
|
|
}
|
|
print $JOBFILE2 @jclines2;
|
|
close($JOBFILE2);
|
|
chown( $uid, $gid, $lljob_file2 );
|
|
}
|
|
|
|
if ($::updateall) {
|
|
&set_LL_feature($machinelist,$::updateall_feature);
|
|
|
|
}
|
|
|
|
# Need to change status before actually submitting LL jobs
|
|
# If LL jobs happen to run right away, the update code checking
|
|
# for the status may run before we've had a chance to actually update it
|
|
my $nltab = xCAT::Table->new('nodelist');
|
|
my @nodes = split( /\,/, $nodelist );
|
|
xCAT::Utils->setAppStatus(\@nodes,"RollingUpdate","update_job_submitted");
|
|
# Submit LL reservation
|
|
my $downnodes = "-D";
|
|
if ($::updateall){$downnodes = " ";}
|
|
my $cmd = qq~su - $lluser "-c llmkres -x -d $::FILEATTRS{'reservationduration'}[0] -f $lljob_file -p $llrcb_file $downnodes "~;
|
|
my $cmd2 = qq~su - $lluser "-c llmkres -x -d $::FILEATTRS{'reservationduration'}[0] -f $lljob_file2 -p $llrcb_file $downnodes "~;
|
|
if ($::VERBOSE) {
|
|
my $rsp;
|
|
push @{ $rsp->{data} }, "Running command: $cmd ";
|
|
xCAT::MsgUtils->message( "I", $rsp, $::CALLBACK );
|
|
open (RULOG, ">>$::LOGDIR/$::LOGFILE");
|
|
print RULOG localtime()." Running command: $cmd \n";
|
|
close (RULOG);
|
|
}
|
|
my @llsubmit;
|
|
if ($::TEST) {
|
|
my $rsp;
|
|
push @{ $rsp->{data} }, "In TEST mode. Will NOT run command: $cmd ";
|
|
xCAT::MsgUtils->message( "I", $rsp, $::CALLBACK );
|
|
$::RUNCMD_RC = 0;
|
|
} else {
|
|
my $submit_count = 1;
|
|
if ($::updateall){
|
|
$submit_count = $machinecount / $::updateall_numperupdate;
|
|
}
|
|
for (1..$submit_count) {
|
|
@llsubmit = xCAT::Utils->runcmd( "$cmd", 0 );
|
|
if ( $::RUNCMD_RC != 0 ) {
|
|
my $rsp;
|
|
push @{ $rsp->{data} }, "Could not run llmkres command.";
|
|
push @{ $rsp->{data} }, @llsubmit;
|
|
xCAT::MsgUtils->message( "E", $rsp, $::CALLBACK );
|
|
return 1;
|
|
}
|
|
if ( $::VERBOSE ) {
|
|
my $rsp;
|
|
push @{ $rsp->{data} }, @llsubmit;
|
|
xCAT::MsgUtils->message( "I", $rsp, $::CALLBACK );
|
|
open (RULOG, ">>$::LOGDIR/$::LOGFILE");
|
|
print RULOG @llsubmit;
|
|
close (RULOG);
|
|
}
|
|
}
|
|
if ($lastcount) {
|
|
if ($::VERBOSE) {
|
|
my $rsp;
|
|
push @{ $rsp->{data} }, "Running command: $cmd2 ";
|
|
xCAT::MsgUtils->message( "I", $rsp, $::CALLBACK );
|
|
open (RULOG, ">>$::LOGDIR/$::LOGFILE");
|
|
print RULOG localtime()." Running command: $cmd2 \n";
|
|
close (RULOG);
|
|
}
|
|
@llsubmit = xCAT::Utils->runcmd( "$cmd2", 0 );
|
|
if ( $::RUNCMD_RC != 0 ) {
|
|
my $rsp;
|
|
push @{ $rsp->{data} }, "Could not run llmkres command.";
|
|
push @{ $rsp->{data} }, @llsubmit;
|
|
xCAT::MsgUtils->message( "E", $rsp, $::CALLBACK );
|
|
open (RULOG, ">>$::LOGDIR/$::LOGFILE");
|
|
print RULOG @llsubmit;
|
|
close (RULOG);
|
|
return 1;
|
|
}
|
|
if ( $::VERBOSE ) {
|
|
my $rsp;
|
|
push @{ $rsp->{data} }, @llsubmit;
|
|
xCAT::MsgUtils->message( "I", $rsp, $::CALLBACK );
|
|
open (RULOG, ">>$::LOGDIR/$::LOGFILE");
|
|
print RULOG @llsubmit;
|
|
close (RULOG);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
elsif ( defined($nodelist) ) {
|
|
|
|
# No nodes in LL to submit job to -- not able to schedule.
|
|
# Call xCAT directly for all other nodes.
|
|
# TODO - this will serialize updating the updategroups
|
|
# is this okay, or do we want forked child processes?
|
|
push @calldirectly, $ugname;
|
|
}
|
|
} # end ugloop
|
|
|
|
if ( scalar(@calldirectly) > 0 ) {
|
|
my @children;
|
|
foreach my $ugname (@calldirectly) {
|
|
my $nodelist = join( ',', @{ $updategroup->{$ugname} } );
|
|
my $ugdf = $lljobs_dir . "/rollupdate_" . $ugname . ".data";
|
|
my $rsp;
|
|
push @{ $rsp->{data} },
|
|
"No active LL nodes in update group $ugname";
|
|
push @{ $rsp->{data} },
|
|
"These nodes will be updated now. This will take a few minutes...";
|
|
xCAT::MsgUtils->message( "I", $rsp, $::CALLBACK );
|
|
if ($::VERBOSE) {
|
|
open (RULOG, ">>$::LOGDIR/$::LOGFILE");
|
|
print RULOG localtime()." No active LL nodes in update group $ugname. These nodes will be updated now.\n";
|
|
close (RULOG);
|
|
}
|
|
my $nltab = xCAT::Table->new('nodelist');
|
|
my @nodes = split( /\,/, $nodelist );
|
|
xCAT::Utils->setAppStatus(\@nodes,"RollingUpdate","update_job_submitted");
|
|
my $childpid = runrollupdate( { command => ['runrollupdate'],
|
|
arg => [ 'internal','loadleveler', $ugdf ]
|
|
});
|
|
if (defined($childpid) && ($childpid != 0)) {
|
|
push (@children, $childpid);
|
|
}
|
|
}
|
|
# wait until all the children are finished before returning
|
|
foreach my $child (@children) {
|
|
if ($::VERBOSE) {
|
|
my $rsp;
|
|
push @{ $rsp->{data} }, "Waiting for child PID $child to complete ";
|
|
xCAT::MsgUtils->message( "I", $rsp, $::CALLBACK );
|
|
open (RULOG, ">>$::LOGDIR/$::LOGFILE");
|
|
print RULOG localtime()." Waiting for child PID $child to complete \n";
|
|
close (RULOG);
|
|
}
|
|
waitpid($child,0);
|
|
}
|
|
}
|
|
|
|
return $rc;
|
|
}
|
|
|
|
|
|
|
|
#----------------------------------------------------------------------------
|
|
|
|
=head3 check_policy
|
|
|
|
Check the policy table to see if userid is authorized to run command
|
|
|
|
Arguments: userid, command
|
|
Returns:
|
|
0 - OK
|
|
1 - error
|
|
Globals:
|
|
Error:
|
|
Example:
|
|
|
|
Comments:
|
|
|
|
=cut
|
|
|
|
#-----------------------------------------------------------------------------
|
|
sub check_policy {
|
|
my $userid = shift;
|
|
my $xcatcmd = shift;
|
|
|
|
my $policytable = xCAT::Table->new('policy');
|
|
unless ($policytable) {
|
|
return 1;
|
|
}
|
|
|
|
my $policies = $policytable->getAllEntries;
|
|
$policytable->close;
|
|
foreach my $rule (@$policies) {
|
|
if ( $rule->{name} &&
|
|
(($rule->{name} eq "*") || ($rule->{name} eq $userid)) ) {
|
|
if ( $rule->{commands} &&
|
|
(($rule->{commands} eq "*") || ($rule->{commands} =~ /$xcatcmd/)) ){
|
|
return 0; # match found
|
|
}
|
|
}
|
|
}
|
|
return 1; # no match found
|
|
}
|
|
|
|
|
|
|
|
#----------------------------------------------------------------------------
|
|
|
|
=head3 set_LL_feature
|
|
|
|
Sets the specified feature for the list of LL machines
|
|
|
|
Arguments: $machinelist - blank delimited list of LL machines
|
|
$feature - feature value to set
|
|
Returns:
|
|
0 - OK
|
|
1 - error
|
|
Globals:
|
|
Error:
|
|
Example:
|
|
|
|
Comments:
|
|
|
|
=cut
|
|
|
|
#-----------------------------------------------------------------------------
|
|
sub set_LL_feature {
|
|
my $machinelist = shift;
|
|
my $feature = shift;
|
|
|
|
# Query current feature
|
|
my $cmd = "llconfig -h $machinelist -d FEATURE";
|
|
if ($::VERBOSE) {
|
|
open (RULOG, ">>$::LOGDIR/$::LOGFILE");
|
|
print RULOG localtime()." Running command \'$cmd\'\n";
|
|
close (RULOG);
|
|
}
|
|
my @llcfgoutput = xCAT::Utils->runcmd( $cmd, 0 );
|
|
if ($::VERBOSE) {
|
|
open (RULOG, ">>$::LOGDIR/$::LOGFILE");
|
|
print RULOG localtime()." Return code: $::RUNCMD_RC\n";
|
|
close (RULOG);
|
|
}
|
|
|
|
my %set_features;
|
|
my %had_cfg;
|
|
foreach my $llcfgout (@llcfgoutput) {
|
|
my @llfeatures;
|
|
my $haveit = 0;
|
|
if ( $llcfgout =~ /:FEATURE =/ ) {
|
|
my ($stuff,$newfeature_string) = split(/=/,$llcfgout);
|
|
my ($machine,$morestuff) = split(/:/,$stuff);
|
|
@llfeatures = split(/\s+/,$newfeature_string);
|
|
foreach my $f (@llfeatures) {
|
|
if ($f eq $feature){
|
|
$haveit = 1;
|
|
}
|
|
}
|
|
if ( !$haveit) {
|
|
$newfeature_string .= " $feature";
|
|
}
|
|
$set_features{$newfeature_string} .= " $machine";
|
|
$had_cfg{$machine} = 1;
|
|
}
|
|
}
|
|
foreach my $m (split(/\s+/,$machinelist)){
|
|
if (! $had_cfg{$m} ){
|
|
$set_features{$feature} .= " $m";
|
|
}
|
|
}
|
|
|
|
# Change in LL database
|
|
foreach my $sf (keys %set_features) {
|
|
if ($set_features{$sf} !~ /^\s*$/){
|
|
$cmd = "llconfig -h $set_features{$sf} -c FEATURE=\"$sf\"";
|
|
if ($::VERBOSE) {
|
|
open (RULOG, ">>$::LOGDIR/$::LOGFILE");
|
|
print RULOG localtime()." Running command \'$cmd\'\n";
|
|
close (RULOG);
|
|
}
|
|
if ($::TEST) {
|
|
my $rsp;
|
|
push @{ $rsp->{data} }, "In TEST mode. Will NOT run command: $cmd ";
|
|
xCAT::MsgUtils->message( "I", $rsp, $::CALLBACK );
|
|
$::RUNCMD_RC = 0;
|
|
} else {
|
|
xCAT::Utils->runcmd( $cmd, 0 );
|
|
###
|
|
### DEBUG -- llconfig problem
|
|
### workaround - sleep to let ll daemons catch up
|
|
sleep 3;
|
|
if ($::VERBOSE) {
|
|
open (RULOG, ">>$::LOGDIR/$::LOGFILE");
|
|
print RULOG localtime()." Return code: $::RUNCMD_RC\n";
|
|
close (RULOG);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
###
|
|
### DEBUG -- llconfig problem
|
|
### need to send a manual reconfig to at least one node to have it
|
|
### register in LL daemons
|
|
### not working - try sleep above instead
|
|
#sleep 3;
|
|
# my @rcfns = split(/\s+/,$machinelist);
|
|
# my @rcfg = `ssh $rcfns[0] llrctl reconfig`;
|
|
# if ($::VERBOSE) {
|
|
# open (RULOG, ">>$::LOGDIR/$::LOGFILE");
|
|
# print RULOG localtime()." llconfig workaround, run \"ssh $rcfns[0] llrctl reconfig \", output: \n";
|
|
# print RULOG @rcfg;
|
|
# close (RULOG);
|
|
# }
|
|
### end DEBUG
|
|
return 0;
|
|
}
|
|
|
|
|
|
#----------------------------------------------------------------------------
|
|
|
|
=head3 get_prescripts
|
|
|
|
Generate the prescripts for this nodelist
|
|
|
|
Arguments:
|
|
Returns:
|
|
0 - OK
|
|
1 - error
|
|
Globals:
|
|
Error:
|
|
Example:
|
|
|
|
Comments:
|
|
|
|
=cut
|
|
|
|
#-----------------------------------------------------------------------------
|
|
sub get_prescripts {
|
|
my $nodelist = shift;
|
|
my @nl = split(/,/, $nodelist);
|
|
|
|
my @datalines;
|
|
my $psindex=0;
|
|
foreach my $psl ( @{ $::FILEATTRS{'prescript'} } ) {
|
|
my $psline = $psl;
|
|
if ($::updateall) {
|
|
push (@datalines, "prescript=".$psline."\n");
|
|
$psindex++;
|
|
next;
|
|
}
|
|
my $psnoderange = $::FILEATTRS{'prescriptnodes'}[$psindex];
|
|
my $executenodelist = "";
|
|
if ($psnoderange eq "ALL_NODES_IN_UPDATEGROUP") {
|
|
$executenodelist = $nodelist;
|
|
} else {
|
|
my @psns = xCAT::NodeRange::noderange($psnoderange);
|
|
unless (@psns) { $psindex++; next; }
|
|
my @executenodes;
|
|
foreach my $node (@nl) {
|
|
push (@executenodes, grep (/^$node$/,@psns));
|
|
}
|
|
$executenodelist = join(',',@executenodes);
|
|
}
|
|
if ($executenodelist ne "") {
|
|
$psline =~ s/\$NODELIST/$executenodelist/g;
|
|
push (@datalines, "prescript=".$psline."\n");
|
|
}
|
|
$psindex++;
|
|
}
|
|
return @datalines;
|
|
}
|
|
|
|
|
|
|
|
#----------------------------------------------------------------------------
|
|
|
|
=head3 get_outofband
|
|
|
|
Generate the out of band scripts for this nodelist
|
|
|
|
Arguments:
|
|
Returns:
|
|
0 - OK
|
|
1 - error
|
|
Globals:
|
|
Error:
|
|
Example:
|
|
|
|
Comments:
|
|
|
|
=cut
|
|
|
|
#-----------------------------------------------------------------------------
|
|
sub get_outofband {
|
|
my $nodelist = shift;
|
|
my @nl = split(/,/, $nodelist);
|
|
|
|
my @datalines;
|
|
my $obindex=0;
|
|
foreach my $obl ( @{ $::FILEATTRS{'outofbandcmd'} } ) {
|
|
my $obline = $obl;
|
|
if ($::updateall) {
|
|
push (@datalines, "outofbandcmd=".$obline."\n");
|
|
$obindex++;
|
|
next;
|
|
}
|
|
my $obnoderange = $::FILEATTRS{'outofbandnodes'}[$obindex];
|
|
my $executenodelist = "";
|
|
if ($obnoderange eq "ALL_NODES_IN_UPDATEGROUP") {
|
|
$executenodelist = $nodelist;
|
|
} else {
|
|
my @obns = xCAT::NodeRange::noderange($obnoderange);
|
|
unless (@obns) { $obindex++; next; }
|
|
my @executenodes;
|
|
foreach my $node (@nl) {
|
|
push (@executenodes, grep (/^$node$/,@obns));
|
|
}
|
|
$executenodelist = join(',',@executenodes);
|
|
}
|
|
if ($executenodelist ne "") {
|
|
$obline =~ s/\$NODELIST/$executenodelist/g;
|
|
push (@datalines, "outofbandcmd=".$obline."\n");
|
|
}
|
|
$obindex++;
|
|
}
|
|
return @datalines;
|
|
}
|
|
|
|
|
|
#----------------------------------------------------------------------------
|
|
|
|
=head3 get_bringuporder
|
|
|
|
Generate the bringup order for this nodelist
|
|
|
|
Arguments:
|
|
Returns:
|
|
0 - OK
|
|
1 - error
|
|
Globals:
|
|
Error:
|
|
Example:
|
|
|
|
Comments:
|
|
|
|
=cut
|
|
|
|
#-----------------------------------------------------------------------------
|
|
sub get_bringuporder {
|
|
my $nodelist = shift;
|
|
my @nl = split(/,/, $nodelist);
|
|
|
|
my @datalines;
|
|
if ($::updateall) {
|
|
$datalines[0]="\n";
|
|
return @datalines;
|
|
}
|
|
foreach my $buline ( @{ $::FILEATTRS{'bringuporder'} } ) {
|
|
my @buns = xCAT::NodeRange::noderange($buline);
|
|
unless (@buns) { next; }
|
|
my @bringupnodes;
|
|
foreach my $node (@nl) {
|
|
if (defined($node)) {
|
|
(my $found) = grep (/^$node$/,@buns);
|
|
if ($found) {
|
|
push (@bringupnodes, $found);
|
|
undef $node; # don't try to use this node again
|
|
}
|
|
}
|
|
}
|
|
my $bringupnodelist = join(',',@bringupnodes);
|
|
if ($bringupnodelist ne "") {
|
|
push (@datalines, "bringuporder=".$bringupnodelist."\n");
|
|
}
|
|
}
|
|
return @datalines;
|
|
}
|
|
|
|
|
|
#----------------------------------------------------------------------------
|
|
|
|
=head3 get_mutex
|
|
|
|
Generate the list of LL mutual exclusion resources for this
|
|
update group
|
|
|
|
Arguments:
|
|
Returns:
|
|
0 - OK
|
|
1 - error
|
|
Globals:
|
|
Error:
|
|
Example:
|
|
|
|
Comments:
|
|
|
|
=cut
|
|
|
|
#-----------------------------------------------------------------------------
|
|
sub get_mutex {
|
|
my $ugname = shift;
|
|
|
|
my $mutex_string = "";
|
|
|
|
my $max_updates = $::FILEATTRS{'maxupdates'}[0];
|
|
if ( defined($max_updates) && ($max_updates ne 'all') ) {
|
|
$mutex_string .= "XCATROLLINGUPDATE_MAXUPDATES(1) ";
|
|
}
|
|
|
|
if ($::updateall){
|
|
return $mutex_string;
|
|
}
|
|
|
|
my $mutex_count = scalar @::MUTEX;
|
|
if ( $mutex_count > 0 ) {
|
|
foreach my $row (0..($mutex_count-1)) {
|
|
foreach my $ugi (0..(@{$::MUTEX[$row]} - 1)) {
|
|
if ( defined($::MUTEX[$row][$ugi]) && ($ugname eq $::MUTEX[$row][$ugi]) ) {
|
|
$mutex_string .= "XCATROLLINGUPDATE_MUTEX".$row."(1) ";
|
|
last;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return $mutex_string;
|
|
}
|
|
|
|
|
|
#----------------------------------------------------------------------------
|
|
|
|
=head3 create_LL_mutex_resources
|
|
|
|
Create all required LL mutex resources
|
|
|
|
Arguments:
|
|
Returns:
|
|
0 - OK
|
|
1 - error
|
|
Globals:
|
|
Error:
|
|
Example:
|
|
|
|
Comments: Reads the mutex entries in $::FILEATTRS and
|
|
sets the global array $::MUTEX
|
|
which is a multi-dimensional array:
|
|
Each row is an array of mutually exclusive update
|
|
group names
|
|
A LL Floating Resource will be created for each
|
|
row of the array.
|
|
|
|
=cut
|
|
|
|
#-----------------------------------------------------------------------------
|
|
sub create_LL_mutex_resources {
|
|
|
|
|
|
$::LL_MUTEX_RESOURCES_CREATED = 0;
|
|
my $mxindex=0;
|
|
foreach my $mxline ( @{ $::FILEATTRS{'mutex'} } ) {
|
|
my @mxparts = split(/,/,$mxline);
|
|
if ( scalar @mxparts < 2 ) {
|
|
my $rsp;
|
|
push @{ $rsp->{data} }, "Error processing stanza line: ";
|
|
push @{ $rsp->{data} }, "mutex=" . $mxline;
|
|
push @{ $rsp->{data} }, "Value must contain at least 2 update groups";
|
|
xCAT::MsgUtils->message( "E", $rsp, $::CALLBACK );
|
|
return 1;
|
|
}
|
|
|
|
my $mxpi = 0;
|
|
my $mxindexmax = $mxindex;
|
|
my @ugnames;
|
|
foreach my $mxpart ( @mxparts ) {
|
|
my $mxindex2 = $mxindex;
|
|
my @ugnamelist = xCAT::NameRange::namerange( $mxpart, 0 );
|
|
foreach my $ugname (@ugnamelist) {
|
|
$::MUTEX[$mxindex2][$mxpi] = $ugname;
|
|
$mxindex2++;
|
|
}
|
|
$mxindexmax = ($mxindex2 > $mxindexmax) ? $mxindex : $mxindexmax;
|
|
$mxpi++;
|
|
}
|
|
$mxindex = $mxindexmax;
|
|
}
|
|
|
|
my $resource_string = "";
|
|
my $max_updates = $::FILEATTRS{'maxupdates'}[0];
|
|
if ( ! defined($max_updates) || ($max_updates eq 'all') ) {
|
|
$max_updates = 0;
|
|
} else {
|
|
$resource_string .= "XCATROLLINGUPDATE_MAXUPDATES($max_updates) ";
|
|
}
|
|
|
|
my $mutex_count = scalar @::MUTEX;
|
|
if ( $mutex_count > 0 ) {
|
|
foreach my $row (0..($mutex_count-1)) {
|
|
# TODO -- UNCOMMENT/REPLACE WHEN LL PROVIDES RESERVATION RESOURCES
|
|
$resource_string .= "XCATROLLINGUPDATE_MUTEX".$row."(1) ";
|
|
}
|
|
}
|
|
|
|
if ( $resource_string ) {
|
|
my $cmd = "llconfig -d FLOATING_RESOURCES SCHEDULE_BY_RESOURCES CENTRAL_MANAGER_LIST RESOURCE_MGR_LIST";
|
|
my @llcfg_d = xCAT::Utils->runcmd( $cmd, 0 );
|
|
my $curSCHED = "";
|
|
my $curFLOAT = "";
|
|
my $llcms = "";
|
|
my $llrms = "";
|
|
foreach my $cfgo (@llcfg_d) {
|
|
chomp $cfgo;
|
|
my($llattr,$llval) = split (/ = /,$cfgo);
|
|
if ( $llattr =~ /SCHEDULE_BY_RESOURCES/ ) {
|
|
$curSCHED = $llval; }
|
|
if ( $llattr =~ /FLOATING_RESOURCES/ ) {
|
|
$curFLOAT = $llval; }
|
|
if ( $llattr =~ /CENTRAL_MANAGER_LIST/ ) {
|
|
$llcms = $llval; }
|
|
if ( $llattr =~ /RESOURCE_MGR_LIST/ ) {
|
|
$llrms = $llval; }
|
|
}
|
|
$cmd = "llconfig -c ";
|
|
my $updateFLOAT = 0;
|
|
my $updateSCHED = 0;
|
|
foreach my $float (split(/\s+/,$resource_string)) {
|
|
$float =~ s/\(/./g;
|
|
$float =~ s/\)/./g;
|
|
if ( $curFLOAT !~ /$float/ ) {
|
|
$updateFLOAT = 1;
|
|
last;
|
|
}
|
|
}
|
|
if ($updateFLOAT) {
|
|
$curFLOAT =~ s/XCATROLLINGUPDATE_MUTEX(\d)*\((\d)*\)//g;
|
|
$curFLOAT =~ s/XCATROLLINGUPDATE_MAXUPDATES(\d)*\((\d)*\)//g;
|
|
$curFLOAT .= $resource_string;
|
|
$cmd .= "FLOATING_RESOURCES=\"$curFLOAT\" ";
|
|
}
|
|
$resource_string =~ s/\((\d)*\)//g;
|
|
foreach my $sched (split(/\s+/,$resource_string)) {
|
|
if ( $curSCHED !~ /$sched/ ) {
|
|
$updateSCHED = 1;
|
|
last;
|
|
}
|
|
}
|
|
if ($updateSCHED) {
|
|
$curSCHED =~ s/XCATROLLINGUPDATE_MUTEX(\d)*//g;
|
|
$curSCHED =~ s/XCATROLLINGUPDATE_MAXUPDATES(\d)*//g;
|
|
$curSCHED .= $resource_string;
|
|
$cmd .= "SCHEDULE_BY_RESOURCES=\"$curSCHED\" ";
|
|
}
|
|
if ( $updateFLOAT || $updateSCHED ) {
|
|
# TODO -- WAITING ON LLCONFIG OPTION TO NOT SEND CFG CMD TO ALL
|
|
#### NODES. NEED TO CHANGE CMD WHEN AVAILABLE.
|
|
my @llcfg_c;
|
|
if ($::TEST) {
|
|
my $rsp;
|
|
push @{ $rsp->{data} }, "In TEST mode. Will NOT run command: $cmd ";
|
|
xCAT::MsgUtils->message( "I", $rsp, $::CALLBACK );
|
|
$::RUNCMD_RC = 0;
|
|
} else {
|
|
@llcfg_c = xCAT::Utils->runcmd( $cmd, 0 );
|
|
}
|
|
$cmd = "llrctl reconfig";
|
|
my @llms = split(/\s+/,$llcms." ".$llrms);
|
|
my %have = ();
|
|
my @llnodes;
|
|
foreach my $m (@llms) {
|
|
my ($sm,$rest) = split(/\./,$m);
|
|
push(@llnodes, $sm) unless $have{$sm}++;
|
|
}
|
|
if ($::TEST) {
|
|
my $rsp;
|
|
push @{ $rsp->{data} }, "In TEST mode. Will NOT run command: xdsh <llcm,llrm> $cmd ";
|
|
xCAT::MsgUtils->message( "I", $rsp, $::CALLBACK );
|
|
$::RUNCMD_RC = 0;
|
|
} else {
|
|
xCAT::Utils->runxcmd(
|
|
{ command => ['xdsh'],
|
|
node => \@llnodes,
|
|
arg => [ "-v", $cmd ]
|
|
},
|
|
$::SUBREQ, -1);
|
|
}
|
|
}
|
|
}
|
|
|
|
$::LL_MUTEX_RESOURCES_CREATED = 1;
|
|
return 0;
|
|
}
|
|
|
|
|
|
|
|
|
|
#----------------------------------------------------------------------------
|
|
|
|
=head3 runrollupdate
|
|
|
|
Reboot updategroup in response to request from scheduler job
|
|
|
|
Arguments:
|
|
Returns:
|
|
0 - OK
|
|
1 - error
|
|
Globals:
|
|
|
|
Error:
|
|
|
|
Example:
|
|
|
|
Comments:
|
|
Note that since this command only gets called from the daemon
|
|
through a port request from a node, there is no active callback
|
|
to return messages to. Log only critical errors to the system log.
|
|
|
|
This subroutine will fork a child process to do the actual
|
|
work of shutting down and rebooting nodes. The parent will return
|
|
immediately to the caller so that other reboot requests can be
|
|
handled in parallel. It is the caller's responsibility to ensure
|
|
the parent process does not go away and take these children down
|
|
with it. (Not a problem when called from xcatd, since that is
|
|
"long-running".)
|
|
=cut
|
|
|
|
#-----------------------------------------------------------------------------
|
|
|
|
sub runrollupdate {
|
|
|
|
my $reboot_request = shift;
|
|
my @reboot_args = @{$reboot_request->{arg}};
|
|
my $internal = 0;
|
|
if ( $reboot_args[0] eq "internal" ) { $internal = 1; }
|
|
|
|
my $rc = 0;
|
|
my $error = 0;
|
|
|
|
# process the command line
|
|
if ( $internal ) {
|
|
$::scheduler = $reboot_args[1];
|
|
$::datafile = $reboot_args[2];
|
|
$::ll_reservation_id = "";
|
|
} else {
|
|
$rc = &processArgs;
|
|
if ( $rc != 0 ) {
|
|
|
|
# rc: 0 - ok, 1 - return, 2 - help, 3 - error
|
|
if ( $rc != 1 ) {
|
|
&runrollupdate_usage;
|
|
}
|
|
return ( $rc - 1 );
|
|
}
|
|
}
|
|
$::scheduler =~ tr/[A-Z]/[a-z]/;
|
|
|
|
|
|
if ($::VERBOSE) {
|
|
unless (-d $::LOGDIR) { File::Path::mkpath($::LOGDIR); }
|
|
open (RULOG, ">>$::LOGDIR/$::LOGFILE");
|
|
print RULOG localtime()."runrollupdate request for $::scheduler $::datafile $::ll_reservation_id \n";
|
|
close (RULOG);
|
|
}
|
|
|
|
my $childpid = xCAT::Utils->xfork();
|
|
unless (defined $childpid) { die "Fork failed" };
|
|
if ($childpid != 0) {
|
|
# This is the parent process, just return and let the child do all
|
|
# the work.
|
|
return $childpid;
|
|
}
|
|
|
|
# This is now the child process
|
|
|
|
# Load the datafile
|
|
&readDataFile($::datafile);
|
|
$::ug_name = $::DATAATTRS{updategroup}[0];
|
|
my ($statusattr,$statusval,$statustimeout);
|
|
if (defined($::DATAATTRS{bringupappstatus}[0])) {
|
|
$statusattr = "appstatus";
|
|
$statusval = $::DATAATTRS{bringupappstatus}[0];
|
|
} elsif (defined($::DATAATTRS{bringupstatus}[0])) {
|
|
$statusattr="status";
|
|
$statusval = $::DATAATTRS{bringupstatus}[0];
|
|
} else {
|
|
$statusattr="status";
|
|
$statusval = "booted";
|
|
}
|
|
if (defined($::DATAATTRS{bringuptimeout}[0])) {
|
|
$statustimeout = $::DATAATTRS{bringuptimeout}[0];
|
|
} else {
|
|
$statustimeout = 10;
|
|
}
|
|
|
|
# make sure nodes are in correct state
|
|
my $hostlist = &get_hostlist;
|
|
if (! $hostlist ) {
|
|
if ($::VERBOSE) {
|
|
open (RULOG, ">>$::LOGDIR/$::LOGFILE");
|
|
print RULOG localtime()."$::ug_name: Cannot determine nodelist for this request. \n";
|
|
close (RULOG);
|
|
}
|
|
my $rsp;
|
|
xCAT::MsgUtils->message( "S",
|
|
"rollupdate failure: $::ug_name: Cannot determine nodelist for this request. ");
|
|
exit(1);
|
|
}
|
|
|
|
my $nltab = xCAT::Table->new('nodelist');
|
|
my @nodes = split( /\,/, $hostlist );
|
|
my $appstatus=xCAT::Utils->getAppStatus(\@nodes,"RollingUpdate");
|
|
foreach my $node (@nodes) {
|
|
unless ( defined($appstatus->{$node})
|
|
&& ( $appstatus->{$node} eq "update_job_submitted" ) )
|
|
{
|
|
if ($::VERBOSE) {
|
|
open (RULOG, ">>$::LOGDIR/$::LOGFILE");
|
|
print RULOG localtime()." $::ug_name: Node $node appstatus not in valid state for rolling update\n";
|
|
print RULOG "The following nodelist will not be processed:\n $hostlist \n";
|
|
close (RULOG);
|
|
}
|
|
my $rsp;
|
|
xCAT::MsgUtils->message(
|
|
"S",
|
|
"ROLLUPDATE failure: $::ug_name: Node $node appstatus not in valid state for rolling update "
|
|
);
|
|
exit(1);
|
|
}
|
|
}
|
|
|
|
# Run prescripts for this update group
|
|
xCAT::Utils->setAppStatus(\@nodes,"RollingUpdate","running_prescripts");
|
|
foreach my $psline ( @{ $::DATAATTRS{'prescript'} } ) {
|
|
$psline =~ s/\$NODELIST/$hostlist/g;
|
|
# Run the command
|
|
if ($::VERBOSE) {
|
|
open (RULOG, ">>$::LOGDIR/$::LOGFILE");
|
|
print RULOG localtime()." $::ug_name: Running prescript \'$psline\'\n";
|
|
close (RULOG);
|
|
}
|
|
my @psoutput;
|
|
if ($::TEST) {
|
|
open (RULOG, ">>$::LOGDIR/$::LOGFILE");
|
|
print RULOG localtime()." $::ug_name: In TEST mode. Will NOT run prescript \'$psline\'\n";
|
|
close (RULOG);
|
|
} else {
|
|
@psoutput = xCAT::Utils->runcmd( $psline, 0 );
|
|
}
|
|
if ($::VERBOSE) {
|
|
open (RULOG, ">>$::LOGDIR/$::LOGFILE");
|
|
print RULOG localtime()." Prescript output:\n";
|
|
foreach my $psoline (@psoutput) {
|
|
print RULOG $psoline."\n";
|
|
}
|
|
close (RULOG);
|
|
}
|
|
}
|
|
|
|
|
|
# Shutdown the nodes
|
|
# FUTURE: Replace if we ever develop cluster shutdown function
|
|
xCAT::Utils->setAppStatus(\@nodes,"RollingUpdate","shutting_down");
|
|
my $shutdown_cmd = "shutdown -F &";
|
|
if ($::VERBOSE) {
|
|
open (RULOG, ">>$::LOGDIR/$::LOGFILE");
|
|
print RULOG localtime()." $::ug_name: Running command \'xdsh $hostlist -v $shutdown_cmd\' \n";
|
|
close (RULOG);
|
|
}
|
|
if ($::TEST) {
|
|
open (RULOG, ">>$::LOGDIR/$::LOGFILE");
|
|
print RULOG localtime()." $::ug_name: In TEST mode. Will NOT run command \'xdsh $hostlist -v $shutdown_cmd\' \n";
|
|
close (RULOG);
|
|
} else {
|
|
xCAT::Utils->runxcmd( { command => ['xdsh'],
|
|
node => \@nodes,
|
|
arg => [ "-v", $shutdown_cmd ]
|
|
}, $::SUBREQ, -1);
|
|
}
|
|
my $slept = 0;
|
|
my $alldown = 1;
|
|
my $nodelist = join( ',', @nodes );
|
|
if (! $::TEST) {
|
|
do {
|
|
$alldown = 1;
|
|
my $shutdownmax = 5;
|
|
if ( defined($::DATAATTRS{shutdowntimeout}[0] ) ) {
|
|
$shutdownmax = $::DATAATTRS{shutdowntimeout}[0];
|
|
}
|
|
my $pwrstat_cmd = "rpower $nodelist stat";
|
|
if ($::VERBOSE) {
|
|
open (RULOG, ">>$::LOGDIR/$::LOGFILE");
|
|
print RULOG localtime()." $::ug_name: Running command \'$pwrstat_cmd\' \n";
|
|
close (RULOG);
|
|
}
|
|
my $pwrstat = xCAT::Utils->runxcmd( $pwrstat_cmd, $::SUBREQ, -1, 1 );
|
|
foreach my $pline (@{$pwrstat}) {
|
|
my ( $pnode, $pstat, $rest ) = split( /\s+/, $pline );
|
|
if ( ( $pstat eq "Running" )
|
|
|| ( $pstat eq "Shutting" )
|
|
|| ( $pstat eq "on" ) )
|
|
{
|
|
|
|
# give up on shutdown after requested wait time and force the
|
|
# node off
|
|
if ( $slept >= ($shutdownmax * 60) ) {
|
|
$pnode =~ s/://g;
|
|
my $pwroff_cmd = "rpower $pnode off";
|
|
if ($::VERBOSE) {
|
|
open (RULOG, ">>$::LOGDIR/$::LOGFILE");
|
|
print RULOG localtime()." $::ug_name: Running command \'$pwroff_cmd\' \n";
|
|
close (RULOG);
|
|
}
|
|
xCAT::Utils->runxcmd( $pwroff_cmd, $::SUBREQ, -1 );
|
|
}
|
|
else {
|
|
$alldown = 0;
|
|
last;
|
|
}
|
|
}
|
|
}
|
|
|
|
# If all nodes are not down yet, wait some more
|
|
unless ($alldown) {
|
|
sleep(20);
|
|
$slept += 20;
|
|
}
|
|
} until ($alldown);
|
|
} # end not TEST
|
|
|
|
# Run out-of-band commands for this update group
|
|
xCAT::Utils->setAppStatus(\@nodes,"RollingUpdate","running_outofbandcmds");
|
|
foreach my $obline ( @{ $::DATAATTRS{'outofbandcmd'} } ) {
|
|
$obline =~ s/\$NODELIST/$hostlist/g;
|
|
# Run the command
|
|
if ($::VERBOSE) {
|
|
open (RULOG, ">>$::LOGDIR/$::LOGFILE");
|
|
print RULOG localtime()." $::ug_name: Running out-of-band command \'$obline\'\n";
|
|
close (RULOG);
|
|
}
|
|
my @oboutput;
|
|
if ($::TEST) {
|
|
open (RULOG, ">>$::LOGDIR/$::LOGFILE");
|
|
print RULOG localtime()." $::ug_name: In TEST mode. Will NOT run out-of-band command \'$obline\'\n";
|
|
close (RULOG);
|
|
} else {
|
|
@oboutput = xCAT::Utils->runcmd( $obline, 0 );
|
|
}
|
|
if ($::VERBOSE) {
|
|
open (RULOG, ">>$::LOGDIR/$::LOGFILE");
|
|
print RULOG localtime()." $::ug_name: Out-of-band command output:\n";
|
|
foreach my $oboline (@oboutput) {
|
|
print RULOG $oboline."\n";
|
|
}
|
|
close (RULOG);
|
|
}
|
|
}
|
|
|
|
|
|
|
|
# reboot the nodes
|
|
# Use bringuporder defined in datafile
|
|
|
|
my $numboots = 0;
|
|
if ( defined($::DATAATTRS{bringuporder}[0]) ) {
|
|
$numboots = scalar( @{$::DATAATTRS{bringuporder}} );
|
|
}
|
|
my @remaining_nodes = @nodes;
|
|
foreach my $bootindex (0..$numboots){
|
|
my @bootnodes;
|
|
if (defined($::DATAATTRS{bringuporder}[$bootindex])) {
|
|
@bootnodes = split(/,/,$::DATAATTRS{bringuporder}[$bootindex]);
|
|
foreach my $rn (@remaining_nodes) {
|
|
if ((defined($rn)) && (grep(/^$rn$/,@bootnodes))) {
|
|
undef($rn);
|
|
}
|
|
}
|
|
} else {
|
|
foreach my $rn (@remaining_nodes) {
|
|
if (defined($rn)) {
|
|
push (@bootnodes,$rn);
|
|
}
|
|
}
|
|
}
|
|
if (!scalar (@bootnodes)) { next; }
|
|
|
|
# reboot command determined by nodehm power/mgt attributes
|
|
my $hmtab = xCAT::Table->new('nodehm');
|
|
my @rpower_nodes;
|
|
my @rnetboot_nodes;
|
|
my $hmtab_entries =
|
|
$hmtab->getNodesAttribs( \@bootnodes, [ 'node', 'mgt', 'power' ] );
|
|
foreach my $node (@bootnodes) {
|
|
my $pwr = $hmtab_entries->{$node}->[0]->{power};
|
|
unless ( defined($pwr) ) { $pwr = $hmtab_entries->{$node}->[0]->{mgt}; }
|
|
if ( $pwr eq 'hmc' ) {
|
|
push( @rnetboot_nodes, $node );
|
|
}
|
|
else {
|
|
push( @rpower_nodes, $node );
|
|
}
|
|
}
|
|
|
|
xCAT::Utils->setAppStatus(\@bootnodes,"RollingUpdate","rebooting");
|
|
if ( scalar(@rnetboot_nodes) > 0 ) {
|
|
my $rnb_nodelist = join( ',', @rnetboot_nodes );
|
|
# my $cmd = "rnetboot $rnb_nodelist -f";
|
|
#### TODO: DO WE STILL NEED 2 LISTS?
|
|
#### RUNNING rpower FOR ALL BOOTS NOW! MUCH FASTER!!!
|
|
my $cmd = "rpower $rnb_nodelist on";
|
|
if ($::VERBOSE) {
|
|
open (RULOG, ">>$::LOGDIR/$::LOGFILE");
|
|
print RULOG localtime()." $::ug_name: Running command \'$cmd\' \n";
|
|
close (RULOG);
|
|
}
|
|
if ($::TEST) {
|
|
open (RULOG, ">>$::LOGDIR/$::LOGFILE");
|
|
print RULOG localtime()." $::ug_name: In TEST mode. Will NOT run command \'$cmd\' \n";
|
|
close (RULOG);
|
|
} else {
|
|
xCAT::Utils->runxcmd( $cmd, $::SUBREQ, 0 );
|
|
}
|
|
} elsif ( scalar(@rpower_nodes) > 0 ) {
|
|
my $rp_nodelist = join( ',', @rpower_nodes );
|
|
my $cmd = "rpower $rp_nodelist boot";
|
|
if ($::VERBOSE) {
|
|
open (RULOG, ">>$::LOGDIR/$::LOGFILE");
|
|
print RULOG localtime()." $::ug_name: Running command \'$cmd\' \n";
|
|
close (RULOG);
|
|
}
|
|
if ($::TEST) {
|
|
open (RULOG, ">>$::LOGDIR/$::LOGFILE");
|
|
print RULOG localtime()." $::ug_name: In TEST mode. Will NOT run command \'$cmd\' \n";
|
|
close (RULOG);
|
|
} else {
|
|
xCAT::Utils->runxcmd( $cmd, $::SUBREQ, 0 );
|
|
}
|
|
}
|
|
|
|
# wait for bringupstatus to be set
|
|
my $not_done = 1;
|
|
my $totalwait = 0;
|
|
while ($not_done && $totalwait < ($statustimeout * 60)) {
|
|
if ($::VERBOSE) {
|
|
open (RULOG, ">>$::LOGDIR/$::LOGFILE");
|
|
print RULOG localtime()." $::ug_name: Checking xCAT database $statusattr for value $statusval \n";
|
|
close (RULOG);
|
|
}
|
|
my $nltab_stats =
|
|
$nltab->getNodesAttribs( \@bootnodes, [ 'node', $statusattr ] );
|
|
my %ll_res;
|
|
$not_done = 0;
|
|
foreach my $bn (@bootnodes) {
|
|
if ( $nltab_stats->{$bn}->[0]->{$statusattr} !~ /$statusval/ ) {
|
|
$not_done = 1;
|
|
} else {
|
|
$ll_res{$bn}{remove}=1;
|
|
}
|
|
}
|
|
if (($::scheduler eq "loadleveler") &&
|
|
($::ll_reservation_id)){
|
|
my @remove_res;
|
|
foreach my $bn (keys %ll_res) {
|
|
if ($ll_res{$bn}{remove} && ! $ll_res{$bn}{removed} ){
|
|
push (@remove_res,$bn);
|
|
$ll_res{$bn}{removed} = 1;
|
|
}
|
|
}
|
|
if (@remove_res) {
|
|
&remove_LL_reservations(\@remove_res);
|
|
xCAT::Utils->setAppStatus(\@remove_res,"RollingUpdate","update_complete");
|
|
}
|
|
}
|
|
if ($not_done) {
|
|
sleep(20);
|
|
$totalwait += 20;
|
|
}
|
|
}
|
|
if ($not_done) {
|
|
open (RULOG, ">>$::LOGDIR/$::LOGFILE");
|
|
print RULOG "\n";
|
|
print RULOG localtime()." ERROR: Update group $::ug_name: Reached bringuptimeout before all nodes completed bringup. Some nodes may not have been updated. \n";
|
|
print RULOG "\n";
|
|
close (RULOG);
|
|
last;
|
|
}
|
|
|
|
}
|
|
if ($::VERBOSE) {
|
|
open (RULOG, ">>$::LOGDIR/$::LOGFILE");
|
|
print RULOG localtime()." $::ug_name: Rolling update complete.\n\n";
|
|
close (RULOG);
|
|
}
|
|
|
|
exit(0);
|
|
}
|
|
|
|
#----------------------------------------------------------------------------
|
|
|
|
=head3 readDataFile
|
|
|
|
Process the command line input piped in from a stanza file.
|
|
|
|
Arguments:
|
|
Returns:
|
|
0 - OK
|
|
1 - error
|
|
Globals:
|
|
Error:
|
|
Example:
|
|
|
|
Comments:
|
|
Set %::DATAATTRS
|
|
(i.e.- $::DATAATTRS{attr}=[val])
|
|
|
|
=cut
|
|
|
|
#-----------------------------------------------------------------------------
|
|
sub readDataFile {
|
|
my $filedata = shift;
|
|
|
|
my $DF;
|
|
unless ( open( $DF, "<", $filedata ) ) {
|
|
open (RULOG, ">>$::LOGDIR/$::LOGFILE");
|
|
print RULOG localtime()." runrollupdate cannot open datafile $filedata\n";
|
|
close (RULOG);
|
|
return 1;
|
|
}
|
|
if ($::VERBOSE) {
|
|
open (RULOG, ">>$::LOGDIR/$::LOGFILE");
|
|
print RULOG localtime()." runrollupdate reading datafile $filedata\n";
|
|
close (RULOG);
|
|
}
|
|
my @lines = <$DF>;
|
|
close $DF;
|
|
|
|
foreach my $l (@lines) {
|
|
|
|
# skip blank and comment lines
|
|
next if ( $l =~ /^\s*$/ || $l =~ /^\s*#/ );
|
|
|
|
# process a real line
|
|
if ( $l =~ /^\s*(\w+)\s*=\s*(.*)\s*/ ) {
|
|
my $attr = $1;
|
|
my $val = $2;
|
|
$attr =~ s/^\s*//; # Remove any leading whitespace
|
|
$attr =~ s/\s*$//; # Remove any trailing whitespace
|
|
$attr =~ tr/A-Z/a-z/; # Convert to lowercase
|
|
$val =~ s/^\s*//;
|
|
$val =~ s/\s*$//;
|
|
|
|
# set the value in the hash for this entry
|
|
push( @{ $::DATAATTRS{$attr} }, $val );
|
|
}
|
|
} # end while - go to next line
|
|
|
|
return 0;
|
|
}
|
|
|
|
|
|
#----------------------------------------------------------------------------
|
|
|
|
=head3 get_hostlist
|
|
|
|
Returns a comma-delimited hostlist string for this updategroup
|
|
|
|
Arguments:
|
|
Returns:
|
|
0 - OK
|
|
1 - error
|
|
Globals:
|
|
Error:
|
|
Example:
|
|
|
|
Comments:
|
|
|
|
=cut
|
|
|
|
#-----------------------------------------------------------------------------
|
|
sub get_hostlist {
|
|
if ( defined($::DATAATTRS{nodelist}[0]) ){
|
|
return $::DATAATTRS{nodelist}[0];
|
|
}
|
|
my $cmd;
|
|
if ($::VERBOSE) {
|
|
$cmd = "llqres -r -s -R $::ll_reservation_id 2>>$::LOGDIR/$::LOGFILE";
|
|
} else {
|
|
$cmd = "llqres -r -s -R $::ll_reservation_id";
|
|
}
|
|
if ($::VERBOSE) {
|
|
open (RULOG, ">>$::LOGDIR/$::LOGFILE");
|
|
print RULOG localtime()." $::ug_name: Running command \'$cmd\'\n";
|
|
close (RULOG);
|
|
}
|
|
my ($llstatus) = xCAT::Utils->runcmd( $cmd, 0 );
|
|
if ($::VERBOSE) {
|
|
open (RULOG, ">>$::LOGDIR/$::LOGFILE");
|
|
print RULOG localtime()." Return code: $::RUNCMD_RC\n";
|
|
print RULOG localtime()." $llstatus \n";
|
|
close (RULOG);
|
|
}
|
|
my @status_fields = split(/!/,$llstatus);
|
|
if ($::VERBOSE) {
|
|
open (RULOG, ">>$::LOGDIR/$::LOGFILE");
|
|
print RULOG localtime()." Hostlist: $status_fields[22] \n";
|
|
close (RULOG);
|
|
}
|
|
return $status_fields[22];
|
|
}
|
|
|
|
|
|
|
|
#----------------------------------------------------------------------------
|
|
|
|
=head3 remove_LL_reservations
|
|
|
|
Changes the LL feature for nodes from oldfeature to newfeature
|
|
Remove nodes from LL reservation
|
|
|
|
Arguments:
|
|
Returns:
|
|
0 - OK
|
|
1 - error
|
|
Globals:
|
|
Error:
|
|
Example:
|
|
|
|
Comments:
|
|
|
|
=cut
|
|
|
|
#-----------------------------------------------------------------------------
|
|
sub remove_LL_reservations {
|
|
my $nodes = shift;
|
|
|
|
my $cmd;
|
|
if ($::VERBOSE) {
|
|
$cmd = "llqres -r -s -R $::ll_reservation_id 2>>$::LOGDIR/$::LOGFILE";
|
|
} else {
|
|
$cmd = "llqres -r -s -R $::ll_reservation_id";
|
|
}
|
|
if ($::VERBOSE) {
|
|
open (RULOG, ">>$::LOGDIR/$::LOGFILE");
|
|
my $nl = join(",",@{$nodes});
|
|
print RULOG localtime()." $::ug_name: remove_LL_reservations for $nl \n";
|
|
print RULOG localtime()." $::ug_name: Running command \'$cmd\'\n";
|
|
close (RULOG);
|
|
}
|
|
my ($llstatus) = xCAT::Utils->runcmd( $cmd, 0 );
|
|
if ($::VERBOSE) {
|
|
open (RULOG, ">>$::LOGDIR/$::LOGFILE");
|
|
print RULOG localtime()." Return code: $::RUNCMD_RC\n";
|
|
print RULOG localtime()." $llstatus \n";
|
|
close (RULOG);
|
|
}
|
|
# if ( $::RUNCMD_RC != 0 ) {
|
|
# return 0;
|
|
# }
|
|
my @status_fields = split(/!/,$llstatus);
|
|
my $llnodelist = $status_fields[22];
|
|
my @llnodes = split(/,/,$llnodelist);
|
|
my $llnode_count = scalar (@llnodes);
|
|
my $remove_count = 0;
|
|
my $remove_reservation = 0;
|
|
my $remove_cmd = "llchres -R $::ll_reservation_id -h -";
|
|
|
|
foreach my $n (@{$nodes}) {
|
|
if ( grep(/^$n$/,@llnodes) ) {
|
|
$remove_count++;
|
|
# change features for this node
|
|
&change_LL_feature($n);
|
|
if ( $remove_count < $llnode_count ) {
|
|
$remove_cmd .= " $n";
|
|
} else {
|
|
$remove_reservation = 1;
|
|
last;
|
|
}
|
|
}
|
|
}
|
|
###
|
|
### DEBUG -- llconfig problem
|
|
### need to send a manual reconfig to at least one node to have it
|
|
### register in LL daemons
|
|
### not working - try sleep instead
|
|
sleep 3;
|
|
# my $n1 = $nodes->[0];
|
|
# my @rcfg = `ssh $n1 llrctl reconfig`;
|
|
# if ($::VERBOSE) {
|
|
# open (RULOG, ">>$::LOGDIR/$::LOGFILE");
|
|
# print RULOG localtime()." llconfig workaround, run \"ssh $n1 llrctl reconfig \", output: \n";
|
|
# print RULOG @rcfg;
|
|
# close (RULOG);
|
|
# }
|
|
### end DEBUG
|
|
# Verify that the config change has been registerd and that updatefeature
|
|
# has been removed according to what the LL daemons report
|
|
if (defined($::DATAATTRS{updatefeature}[0])) {
|
|
my $machinelist = join(" ",@{$nodes});
|
|
my $llset;
|
|
my $statrun = 0;
|
|
do {
|
|
sleep 1;
|
|
my $llstatus = "llstatus -Lmachine -h $machinelist -l | grep -i feature | grep -i \" $::DATAATTRS{updatefeature}[0] \"";
|
|
if ($::VERBOSE) {
|
|
open (RULOG, ">>$::LOGDIR/$::LOGFILE");
|
|
print RULOG localtime()." Running command \'$llstatus\'\n";
|
|
close (RULOG);
|
|
}
|
|
my @stat_out = xCAT::Utils->runcmd( $llstatus, 0 );
|
|
if ($::VERBOSE) {
|
|
open (RULOG, ">>$::LOGDIR/$::LOGFILE");
|
|
print RULOG localtime()." Return code: $::RUNCMD_RC\n";
|
|
close (RULOG);
|
|
}
|
|
# Are there any nodes with this feature still set?
|
|
$llset = $stat_out[0];
|
|
### DEBUG - print output to see what's going on
|
|
if ($llset) {
|
|
open (RULOG, ">>$::LOGDIR/$::LOGFILE");
|
|
print RULOG localtime()." llset: $llset\n";
|
|
close (RULOG);
|
|
}
|
|
### end DEBUG
|
|
$statrun ++;
|
|
} until ((! $llset) || ($statrun > 10) );
|
|
}
|
|
|
|
if ($remove_reservation) {
|
|
$cmd = "llrmres -R $::ll_reservation_id";
|
|
} elsif ( $remove_count > 0 ) {
|
|
$cmd = $remove_cmd;
|
|
} else {
|
|
return 0; # no LL nodes to remove
|
|
}
|
|
if ($::VERBOSE) {
|
|
open (RULOG, ">>$::LOGDIR/$::LOGFILE");
|
|
print RULOG localtime()." $::ug_name: Running command \'$cmd\'\n";
|
|
close (RULOG);
|
|
}
|
|
# if ($::TEST) {
|
|
# open (RULOG, ">>$::LOGDIR/$::LOGFILE");
|
|
# print RULOG localtime()." $::ug_name: In TEST mode. Will NOT run command \'$cmd\' \n";
|
|
# close (RULOG);
|
|
# } else {
|
|
xCAT::Utils->runcmd( $cmd, 0 );
|
|
if ( $::RUNCMD_RC != 0 ) {
|
|
open (RULOG, ">>$::LOGDIR/$::LOGFILE");
|
|
print RULOG localtime()." $::ug_name: Error running command \'$cmd\'\n";
|
|
print RULOG localtime()." Return code: $::RUNCMD_RC\n";
|
|
close (RULOG);
|
|
return 1;
|
|
}
|
|
# }
|
|
|
|
return 0;
|
|
}
|
|
|
|
|
|
#----------------------------------------------------------------------------
|
|
|
|
=head3 change_LL_feature
|
|
|
|
Changes the LL feature for the node from oldfeature to newfeature
|
|
and removes updatefeature
|
|
|
|
Arguments:
|
|
Returns:
|
|
0 - OK
|
|
1 - error
|
|
Globals:
|
|
Error:
|
|
Example:
|
|
|
|
Comments:
|
|
|
|
=cut
|
|
|
|
#-----------------------------------------------------------------------------
|
|
sub change_LL_feature {
|
|
my $node = shift;
|
|
|
|
if (!defined($::DATAATTRS{updatefeature}[0]) &&
|
|
!defined($::DATAATTRS{oldfeature}[0]) &&
|
|
!defined($::DATAATTRS{newfeature}[0]) ) {
|
|
return 0;
|
|
}
|
|
# Query current feature
|
|
my $cmd = "llconfig -h $node -d FEATURE";
|
|
if ($::VERBOSE) {
|
|
open (RULOG, ">>$::LOGDIR/$::LOGFILE");
|
|
print RULOG localtime()." $::ug_name: Running command \'$cmd\'\n";
|
|
close (RULOG);
|
|
}
|
|
my ($llcfgout) = xCAT::Utils->runcmd( $cmd, 0 );
|
|
if ($::VERBOSE) {
|
|
open (RULOG, ">>$::LOGDIR/$::LOGFILE");
|
|
print RULOG localtime()." Return code: $::RUNCMD_RC\n";
|
|
close (RULOG);
|
|
}
|
|
|
|
# Remove old feature
|
|
my $newfeature_string = "";
|
|
my @llfeatures;
|
|
if ( $llcfgout =~ /:FEATURE =/ ) {
|
|
my ($stuff,$curfeature_string) = split(/=/,$llcfgout);
|
|
@llfeatures = split(/\s+/,$curfeature_string);
|
|
my $oldfeature = " ";
|
|
my $updateallfeature = " ";
|
|
if (defined($::DATAATTRS{oldfeature}[0])) {
|
|
$oldfeature = $::DATAATTRS{oldfeature}[0];
|
|
}
|
|
if (defined($::DATAATTRS{updatefeature}[0])) {
|
|
$updateallfeature = $::DATAATTRS{updatefeature}[0];
|
|
}
|
|
foreach my $f (@llfeatures) {
|
|
if (($f eq $oldfeature) || ($f eq $updateallfeature)) {
|
|
$f = " ";
|
|
}
|
|
}
|
|
$newfeature_string = join(" ",@llfeatures);
|
|
}
|
|
|
|
# Add new feature
|
|
if ( defined($::DATAATTRS{newfeature}[0]) ) {
|
|
my $haveit = 0;
|
|
foreach my $f (@llfeatures) {
|
|
if ($f eq $::DATAATTRS{newfeature}[0]){
|
|
$haveit = 1;
|
|
}
|
|
}
|
|
if ( !$haveit) {
|
|
$newfeature_string .= " $::DATAATTRS{newfeature}[0]";
|
|
}
|
|
}
|
|
# Change in LL database
|
|
$cmd = "llconfig -h $node -c FEATURE=\"$newfeature_string\"";
|
|
if ($::VERBOSE) {
|
|
open (RULOG, ">>$::LOGDIR/$::LOGFILE");
|
|
print RULOG localtime()." $::ug_name: Running command \'$cmd\'\n";
|
|
close (RULOG);
|
|
}
|
|
xCAT::Utils->runcmd( $cmd, 0 );
|
|
if ($::VERBOSE) {
|
|
open (RULOG, ">>$::LOGDIR/$::LOGFILE");
|
|
print RULOG localtime()." Return code: $::RUNCMD_RC\n";
|
|
close (RULOG);
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
|
|
|
|
1;
|