2008-10-22 16:02:14 +00:00
#!/usr/bin/env perl -w
# IBM(c) 2007 EPL license http://www.eclipse.org/legal/epl-v10.html
#####################################################
#
# xCAT plugin package to handle rolling updates
#
#####################################################
package xCAT_plugin::rollupdate ;
2010-09-14 22:29:41 +00:00
BEGIN
{
$ ::XCATROOT = $ ENV { 'XCATROOT' } ? $ ENV { 'XCATROOT' } : '/opt/xcat' ;
}
2008-10-22 16:02:14 +00:00
require xCAT::NodeRange ;
require xCAT::NameRange ;
require xCAT::Table ;
require Data::Dumper ;
require Getopt::Long ;
require xCAT::MsgUtils ;
require File::Path ;
use strict ;
use warnings ;
#
# Globals
#
2008-10-27 17:11:23 +00:00
$ ::LOGDIR = "/var/log/xcat" ;
$ ::LOGFILE = "rollupdate.log" ;
2008-10-22 16:02:14 +00:00
#------------------------------------------------------------------------------
= head1 rollupdate
This program module file supports the cluster rolling update functions .
Supported commands:
2010-09-14 22:29:41 +00:00
rollupdate - Create scheduler job command files and submit the jobs
runrollupdate - Reboot the updategroup in response to request from scheduler
job
2008-10-22 16:02:14 +00:00
If adding to this file , please take a moment to ensure that:
1 . Your contrib has a readable pod header describing the purpose and use of
the subroutine .
2 . Your contrib is under the correct heading and is in alphabetical order
under that heading .
3 . You have run tidypod on this file and saved the html file
= cut
#------------------------------------------------------------------------------
= head2 Cluster Rolling Update
= cut
#------------------------------------------------------------------------------
#----------------------------------------------------------------------------
= head3 handled_commands
Return a list of commands handled by this plugin
= cut
#-----------------------------------------------------------------------------
sub handled_commands {
return {
rollupdate = > "rollupdate" ,
2010-09-14 22:29:41 +00:00
runrollupdate = > "rollupdate"
2008-10-22 16:02:14 +00:00
} ;
}
#----------------------------------------------------------------------------
= head3 preprocess_request
Arguments:
Returns:
0 - OK
1 - error
Globals:
Error:
Example:
Comments:
= cut
#-----------------------------------------------------------------------------
sub preprocess_request {
# set management node as server for all requests
# any requests sent to service node need to get
# get sent up to the MN
my $ req = shift ;
unless ( defined ( $ req - > { _xcatdest } ) ) {
$ req - > { _xcatdest } = xCAT::Utils - > get_site_Master ( ) ;
}
return [ $ req ] ;
}
#----------------------------------------------------------------------------
= head3 process_request
Process the rolling update commands
Arguments:
Returns:
0 - OK
1 - error
Globals:
Error:
Example:
Comments:
= cut
#-----------------------------------------------------------------------------
sub process_request {
$ ::request = shift ;
$ ::CALLBACK = shift ;
$ ::SUBREQ = shift ;
my $ ret ;
# globals used by all subroutines.
$ ::command = $ ::request - > { command } - > [ 0 ] ;
$ ::args = $ ::request - > { arg } ;
$ ::stdindata = $ ::request - > { stdin } - > [ 0 ] ;
# figure out which cmd and call the subroutine to process
if ( $ ::command eq "rollupdate" ) {
2008-10-22 18:52:40 +00:00
$ ret = & rollupdate ( $ ::request ) ;
2008-10-22 16:02:14 +00:00
}
2010-09-14 22:29:41 +00:00
elsif ( $ ::command eq "runrollupdate" ) {
$ ret = & runrollupdate ( $ ::request ) ;
2008-10-22 16:02:14 +00:00
}
return $ ret ;
}
#----------------------------------------------------------------------------
= head3 rollupdate_usage
Arguments:
Returns:
Globals:
Error:
Example:
Comments:
= cut
#-----------------------------------------------------------------------------
# display the usage
sub rollupdate_usage {
my $ rsp ;
push @ { $ rsp - > { data } } ,
"\nUsage: rollupdate - Submit cluster rolling update jobs \n" ;
push @ { $ rsp - > { data } } , " rollupdate [-h | --help | -?] \n" ;
push @ { $ rsp - > { data } } ,
" rollupdate [-V | --verbose] [-v | --version] \n " ;
push @ { $ rsp - > { data } } ,
" <STDIN> - stanza file, see /opt/xcat/share/xcat/rollupdate/rollupdate.input.sample" ;
push @ { $ rsp - > { data } } , " for example \n" ;
xCAT::MsgUtils - > message ( "I" , $ rsp , $ ::CALLBACK ) ;
return 0 ;
}
2010-09-14 22:29:41 +00:00
#----------------------------------------------------------------------------
= head3 runrollupdate_usage
Arguments:
Returns:
Globals:
Error:
Example:
Comments:
= cut
#-----------------------------------------------------------------------------
# display the usage
sub runrollupdate_usage {
my $ rsp ;
push @ { $ rsp - > { data } } ,
"\nUsage: runrollupdate - Run submitted cluster rolling update job \n" ;
push @ { $ rsp - > { data } } , " runrollupdate [-h | --help | -?] \n" ;
push @ { $ rsp - > { data } } ,
" runrollupdate [-V | --verbose] [-v | --version] scheduler datafile \n " ;
push @ { $ rsp - > { data } } ,
xCAT::MsgUtils - > message ( "I" , $ rsp , $ ::CALLBACK ) ;
return 0 ;
}
2008-10-22 16:02:14 +00:00
#----------------------------------------------------------------------------
= head3 processArgs
Process the command line and any input files provided on cmd line .
Arguments:
Returns:
0 - OK
1 - just print usage
2010-09-14 22:29:41 +00:00
2 - error
2008-10-22 16:02:14 +00:00
Globals:
Error:
Example:
Comments:
= cut
#-----------------------------------------------------------------------------
sub processArgs {
my $ gotattrs = 0 ;
if ( defined ( @ { $ ::args } ) ) {
@ ARGV = @ { $ ::args } ;
}
else {
# return 2; # can run with no args right now
}
# if (scalar(@ARGV) <= 0) {
# return 2;
# }
# parse the options
# options can be bundled up like -vV, flag unsupported options
Getopt::Long:: Configure ( "bundling" , "no_ignore_case" , "no_pass_through" ) ;
Getopt::Long:: GetOptions (
'help|h|?' = > \ $ ::opt_h ,
'test|t' = > \ $ ::opt_t ,
'verbose|V' = > \ $ ::opt_V ,
'version|v' = > \ $ ::opt_v ,
) ;
# Option -h for Help
# if user specifies "-t" & "-h" they want a list of valid attrs
if ( defined ( $ ::opt_h ) ) {
return 2 ;
}
# opt_t not yet supported
if ( defined ( $ ::opt_t ) ) {
2010-09-14 22:29:41 +00:00
$ ::test = 1 ;
$ ::TEST = 1 ;
# my $rsp;
# push @{ $rsp->{data} }, "The \'-t\' option is not yet implemented.";
# xCAT::MsgUtils->message( "I", $rsp, $::CALLBACK );
# return 2;
2008-10-22 16:02:14 +00:00
}
# Option -v for version
if ( defined ( $ ::opt_v ) ) {
my $ rsp ;
my $ version = xCAT::Utils - > Version ( ) ;
push @ { $ rsp - > { data } } , "$::command - $version\n" ;
xCAT::MsgUtils - > message ( "I" , $ rsp , $ ::CALLBACK ) ;
return 1 ; # no usage - just exit
}
# Option -V for verbose output
if ( defined ( $ ::opt_V ) ) {
$ ::verbose = 1 ;
$ ::VERBOSE = 1 ;
}
2010-09-14 22:29:41 +00:00
if ( $ ::command eq "rollupdate" ) {
# process @ARGV
#while (my $a = shift(@ARGV))
#{
# no args for command yet
#}
# process the <stdin> input file
if ( defined ( $ ::stdindata ) ) {
my $ rc = readFileInput ( $ ::stdindata ) ;
if ( $ rc ) {
my $ rsp ;
push @ { $ rsp - > { data } } , "Could not process file input data.\n" ;
xCAT::MsgUtils - > message ( "I" , $ rsp , $ ::CALLBACK ) ;
return 1 ;
}
}
else {
# No <stdin> stanza file, print usage
return 2 ;
2008-10-22 16:02:14 +00:00
}
}
2010-09-14 22:29:41 +00:00
elsif ( $ ::command eq "runrollupdate" ) {
# process @ARGV
$ ::scheduler = shift ( @ ARGV ) ;
$ ::datafile = shift ( @ ARGV ) ;
$ ::ll_reservation_id = shift ( @ ARGV ) ;
2008-10-22 16:02:14 +00:00
}
return 0 ;
}
#----------------------------------------------------------------------------
= head3 readFileInput
Process the command line input piped in from a stanza file .
Arguments:
Returns:
0 - OK
1 - error
Globals:
Error:
Example:
Comments:
Set % ::FILEATTRS
( i . e . - $ ::FILEATTRS { attr } = [ val ] )
= cut
#-----------------------------------------------------------------------------
sub readFileInput {
my ( $ filedata ) = @ _ ;
my @ lines = split /\n/ , $ filedata ;
2010-09-14 22:29:41 +00:00
my $ prev_attr = "" ;
2008-10-22 16:02:14 +00:00
foreach my $ l ( @ lines ) {
# skip blank and comment lines
next if ( $ l =~ /^\s*$/ || $ l =~ /^\s*#/ ) ;
# process a real line
if ( $ l =~ /^\s*(\w+)\s*=\s*(.*)\s*/ ) {
my $ attr = $ 1 ;
my $ val = $ 2 ;
$ attr =~ s/^\s*// ; # Remove any leading whitespace
$ attr =~ s/\s*$// ; # Remove any trailing whitespace
$ attr =~ tr /A-Z/ a - z / ; # Convert to lowercase
$ val =~ s/^\s*// ;
$ val =~ s/\s*$// ;
2010-09-14 22:29:41 +00:00
# Convert the following values to lowercase
if ( ( $ attr eq 'scheduler' ) ||
( $ attr eq 'updateall' ) ||
( $ attr eq 'shutdownrequired' ) ) {
$ val =~ tr /A-Z/ a - z / ;
}
2008-10-22 16:02:14 +00:00
# set the value in the hash for this entry
push ( @ { $ ::FILEATTRS { $ attr } } , $ val ) ;
2010-09-14 22:29:41 +00:00
if ( ( $ prev_attr eq "prescript" ) && ( $ attr ne "prescriptnodes" ) ) {
push ( @ { $ ::FILEATTRS { 'prescriptnodes' } } , 'ALL_NODES_IN_UPDATEGROUP' ) ;
}
if ( ( $ prev_attr eq "outofbandcmd" ) && ( $ attr ne "outofbandnodes" ) ) {
push ( @ { $ ::FILEATTRS { 'outofbandnodes' } } , 'ALL_NODES_IN_UPDATEGROUP' ) ;
}
$ prev_attr = $ attr ;
2008-10-22 16:02:14 +00:00
}
} # end while - go to next line
return 0 ;
}
#----------------------------------------------------------------------------
= head3 rollupdate
Support for the xCAT rollupdate command .
Arguments:
Returns:
0 - OK
1 - error
Globals:
Error:
Example:
Comments:
= cut
#-----------------------------------------------------------------------------
sub rollupdate {
my $ rc = 0 ;
my $ error = 0 ;
# process the command line
$ rc = & processArgs ;
if ( $ rc != 0 ) {
# rc: 0 - ok, 1 - return, 2 - help, 3 - error
if ( $ rc != 1 ) {
& rollupdate_usage ;
}
return ( $ rc - 1 ) ;
}
2010-09-14 22:29:41 +00:00
if ( $ ::VERBOSE ) {
my $ rsp ;
push @ { $ rsp - > { data } } , "Running rollupdate command... " ;
xCAT::MsgUtils - > message ( "I" , $ rsp , $ ::CALLBACK ) ;
open ( RULOG , ">>$::LOGDIR/$::LOGFILE" ) ;
print RULOG "\n\n" ;
print RULOG localtime ( ) . " Running rollupdate command...\n" ;
close ( RULOG ) ;
}
2008-10-22 16:02:14 +00:00
#
# Build updategroup nodelists
#
my % updategroup ;
2010-09-14 22:29:41 +00:00
if ( defined ( $ ::FILEATTRS { updateall } [ 0 ] ) &&
( ( $ ::FILEATTRS { updateall } [ 0 ] eq 'yes' ) ||
( $ ::FILEATTRS { updateall } [ 0 ] eq 'y' ) ) ) {
if ( defined ( $ ::FILEATTRS { updateall_nodes } [ 0 ] ) ) {
my $ ugname = "UPDATEALL" . time ( ) ;
my $ ugval = $ ::FILEATTRS { updateall_nodes } [ 0 ] ;
@ { $ updategroup { $ ugname } } = xCAT::NodeRange:: noderange ( $ ugval ) ;
} else {
2008-10-22 16:02:14 +00:00
my $ rsp ;
2010-09-14 22:29:41 +00:00
push @ { $ rsp - > { data } } ,
"Error processing stanza input: updateall=yes but no updateall_nodes specified. " ;
2008-10-22 16:02:14 +00:00
xCAT::MsgUtils - > message ( "E" , $ rsp , $ ::CALLBACK ) ;
return 1 ;
}
2010-09-14 22:29:41 +00:00
} else {
foreach my $ ugline ( @ { $ ::FILEATTRS { 'updategroup' } } ) {
my ( $ ugname , $ ugval ) = split ( /\(/ , $ ugline ) ;
$ ugval =~ s/\)$// ; # remove trailing ')'
@ { $ updategroup { $ ugname } } = xCAT::NodeRange:: noderange ( $ ugval ) ;
2008-10-22 16:02:14 +00:00
if ( xCAT::NodeRange:: nodesmissed ( ) ) {
my $ rsp ;
push @ { $ rsp - > { data } } , "Error processing stanza line: " ;
2010-09-14 22:29:41 +00:00
push @ { $ rsp - > { data } } , "updategroup=" . $ ugline ;
push @ { $ rsp - > { data } } , "Invalid nodes in noderange: "
2008-10-22 16:02:14 +00:00
. join ( ',' , xCAT::NodeRange:: nodesmissed ( ) ) ;
xCAT::MsgUtils - > message ( "E" , $ rsp , $ ::CALLBACK ) ;
return 1 ;
}
2010-09-14 22:29:41 +00:00
if ( $ ::VERBOSE ) {
my $ rsp ;
my $ prt_ugn = join ( ',' , @ { $ updategroup { $ ugname } } ) ;
push @ { $ rsp - > { data } } , "Creating update group $ugname with nodes: " ;
push @ { $ rsp - > { data } } , $ prt_ugn ;
xCAT::MsgUtils - > message ( "I" , $ rsp , $ ::CALLBACK ) ;
open ( RULOG , ">>$::LOGDIR/$::LOGFILE" ) ;
print RULOG localtime ( ) . " Creating update group $ugname with nodes: \n" ;
print RULOG "$prt_ugn\n" ;
close ( RULOG ) ;
}
}
foreach my $ mgline ( @ { $ ::FILEATTRS { 'mapgroups' } } ) {
my @ ugnamelist = xCAT::NameRange:: namerange ( $ mgline , 0 ) ;
foreach my $ ugname ( @ ugnamelist ) {
@ { $ updategroup { $ ugname } } = xCAT::NodeRange:: noderange ( $ ugname ) ;
if ( xCAT::NodeRange:: nodesmissed ( ) ) {
my $ rsp ;
push @ { $ rsp - > { data } } , "Error processing stanza line: " ;
push @ { $ rsp - > { data } } , "mapgroups=" . $ mgline ;
push @ { $ rsp - > { data } } , "Invalid nodes in group $ugname: "
. join ( ',' , xCAT::NodeRange:: nodesmissed ( ) ) ;
xCAT::MsgUtils - > message ( "E" , $ rsp , $ ::CALLBACK ) ;
return 1 ;
}
if ( $ ::VERBOSE ) {
my $ rsp ;
my $ prt_ugn = join ( ',' , @ { $ updategroup { $ ugname } } ) ;
push @ { $ rsp - > { data } } , "Creating update group $ugname with nodes: " ;
push @ { $ rsp - > { data } } , $ prt_ugn ;
xCAT::MsgUtils - > message ( "I" , $ rsp , $ ::CALLBACK ) ;
open ( RULOG , ">>$::LOGDIR/$::LOGFILE" ) ;
print RULOG localtime ( ) . " Creating update group $ugname with nodes: \n" ;
print RULOG "$prt_ugn\n" ;
close ( RULOG ) ;
}
}
2008-10-22 16:02:14 +00:00
}
}
unless ( % updategroup ) {
my $ rsp ;
push @ { $ rsp - > { data } } ,
"Error processing stanza input: No updategroup or mapgroups entries found. " ;
xCAT::MsgUtils - > message ( "E" , $ rsp , $ ::CALLBACK ) ;
return 1 ;
}
#
# Build and submit scheduler jobs
#
my $ scheduler = $ ::FILEATTRS { 'scheduler' } [ 0 ] ;
if ( ( ! $ scheduler )
|| ( $ scheduler eq "loadleveler" ) )
{
$ rc = ll_jobs ( \ % updategroup ) ;
}
else {
# TODO: support scheduler plugins here
my $ rsp ;
push @ { $ rsp - > { data } } , "Error processing stanza line: " ;
push @ { $ rsp - > { data } } , "scheduler=" . $ ::FILEATTRS { 'scheduler' } [ 0 ] ;
push @ { $ rsp - > { data } } , "Scheduler not supported" ;
xCAT::MsgUtils - > message ( "E" , $ rsp , $ ::CALLBACK ) ;
return 1 ;
}
return $ rc ;
}
#----------------------------------------------------------------------------
= head3 ll_jobs
Build and submit LoadLeveler jobs
Arguments:
Returns:
0 - OK
1 - error
Globals:
Error:
Example:
Comments:
= cut
#-----------------------------------------------------------------------------
sub ll_jobs {
my $ updategroup = shift ;
my $ rc = 0 ;
2010-09-14 22:29:41 +00:00
if ( $ ::VERBOSE ) {
my $ rsp ;
push @ { $ rsp - > { data } } , "Creating LL job command files " ;
xCAT::MsgUtils - > message ( "I" , $ rsp , $ ::CALLBACK ) ;
open ( RULOG , ">>$::LOGDIR/$::LOGFILE" ) ;
print RULOG localtime ( ) . " Creating LL job command files \n" ;
close ( RULOG ) ;
}
$ ::updateall = 0 ;
$ ::updateall_numperupdate = 1 ;
if ( defined ( $ ::FILEATTRS { updateall } [ 0 ] ) &&
( ( $ ::FILEATTRS { updateall } [ 0 ] eq 'yes' ) ||
( $ ::FILEATTRS { updateall } [ 0 ] eq 'y' ) ) ) {
$ ::updateall = 1 ;
if ( defined ( $ ::FILEATTRS { updateall_numperupdate } [ 0 ] ) ) {
$ ::updateall_numperupdate = $ ::FILEATTRS { updateall_numperupdate } [ 0 ] ;
}
}
# Create LL floating resources for mutual exclusion support
# and max_updates
if ( & create_LL_mutex_resources > 0 ) {
return 1 ;
}
2008-10-22 16:02:14 +00:00
#
# Load job command file template
#
my $ tmpl_file_name = $ ::FILEATTRS { 'jobtemplate' } [ 0 ] ;
unless ( defined ( $ tmpl_file_name ) ) {
my $ rsp ;
push @ { $ rsp - > { data } } ,
"Error processing stanza input: No jobtemplate entries found. " ;
xCAT::MsgUtils - > message ( "E" , $ rsp , $ ::CALLBACK ) ;
return 1 ;
}
my $ TMPL_FILE ;
unless ( open ( $ TMPL_FILE , "<" , $ tmpl_file_name ) ) {
my $ rsp ;
push @ { $ rsp - > { data } } ,
"Error processing stanza input: jobtemplate file $tmpl_file_name not found. " ;
xCAT::MsgUtils - > message ( "E" , $ rsp , $ ::CALLBACK ) ;
return 1 ;
}
2010-09-14 22:29:41 +00:00
if ( $ ::VERBOSE ) {
my $ rsp ;
push @ { $ rsp - > { data } } , "Reading LL job template file $tmpl_file_name " ;
xCAT::MsgUtils - > message ( "I" , $ rsp , $ ::CALLBACK ) ;
open ( RULOG , ">>$::LOGDIR/$::LOGFILE" ) ;
print RULOG localtime ( ) . " Reading LL job template file $tmpl_file_name \n" ;
close ( RULOG ) ;
}
2008-10-22 16:02:14 +00:00
my @ lines = <$TMPL_FILE> ;
close $ TMPL_FILE ;
# Query LL for list of machines and their status
2010-09-14 22:29:41 +00:00
### LL BUG WORKAROUND -- MUST SET LOADL_STATUS_LEVEL=MACHINE
my $ cmd = "LOADL_STATUS_LEVEL=MACHINE llstatus -r %n %sta 2>/dev/null" ;
# my $cmd = "llstatus -r %n %sta 2>/dev/null";
if ( $ ::VERBOSE ) {
my $ rsp ;
push @ { $ rsp - > { data } } , "Running command: $cmd " ;
xCAT::MsgUtils - > message ( "I" , $ rsp , $ ::CALLBACK ) ;
open ( RULOG , ">>$::LOGDIR/$::LOGFILE" ) ;
print RULOG localtime ( ) . " Running command: $cmd \n" ;
close ( RULOG ) ;
}
2008-10-22 16:02:14 +00:00
my @ llstatus = xCAT::Utils - > runcmd ( $ cmd , 0 ) ;
if ( $ ::RUNCMD_RC != 0 ) {
my $ rsp ;
push @ { $ rsp - > { data } } , "Could not run llstatus command." ;
push @ { $ rsp - > { data } } , @ llstatus ;
xCAT::MsgUtils - > message ( "E" , $ rsp , $ ::CALLBACK ) ;
2010-09-14 22:29:41 +00:00
open ( RULOG , ">>$::LOGDIR/$::LOGFILE" ) ;
print RULOG localtime ( ) . " Could not run llstatus command. \n" ;
print RULOG @ llstatus ;
close ( RULOG ) ;
2008-10-22 16:02:14 +00:00
return 1 ;
}
my % machines ;
foreach my $ machineline ( @ llstatus ) {
my ( $ mlong , $ mshort , $ mstatus ) ;
( $ mlong , $ mstatus ) = split ( /\!/ , $ machineline ) ;
( $ mshort ) = split ( /\./ , $ mlong ) ;
$ machines { $ mlong } = { mname = > $ mlong , mstatus = > $ mstatus } ;
if ( ! ( $ mlong eq $ mshort ) ) {
$ machines { $ mshort } = { mname = > $ mlong , mstatus = > $ mstatus } ;
}
}
#
# Generate job command file for each updategroup
#
# Get LL userid
my $ lluser = $ ::FILEATTRS { scheduser } [ 0 ] ;
unless ( defined ( $ lluser ) ) {
my $ rsp ;
push @ { $ rsp - > { data } } ,
"Error processing stanza input: No scheduser entries found. " ;
xCAT::MsgUtils - > message ( "E" , $ rsp , $ ::CALLBACK ) ;
return 1 ;
}
my ( $ login , $ pass , $ uid , $ gid ) ;
( $ login , $ pass , $ uid , $ gid ) = getpwnam ( $ lluser ) ;
unless ( defined ( $ uid ) ) {
my $ rsp ;
push @ { $ rsp - > { data } } ,
2010-09-14 22:29:41 +00:00
"Error processing stanza input: scheduser userid $lluser not defined in system. " ;
xCAT::MsgUtils - > message ( "E" , $ rsp , $ ::CALLBACK ) ;
return 1 ;
}
if ( & check_policy ( $ lluser , 'runrollupdate' ) ) {
my $ rsp ;
push @ { $ rsp - > { data } } ,
"Error processing stanza input: scheduser userid $lluser not listed in xCAT policy table for runrollupdate command. Add to policy table and ensure userid has ssh credentials for running xCAT commands. " ;
2008-10-22 16:02:14 +00:00
xCAT::MsgUtils - > message ( "E" , $ rsp , $ ::CALLBACK ) ;
return 1 ;
}
2010-09-14 22:29:41 +00:00
2008-10-22 16:02:14 +00:00
my $ lljobs_dir = $ ::FILEATTRS { jobdir } [ 0 ] ;
unless ( defined ( $ lljobs_dir ) ) {
my $ rsp ;
push @ { $ rsp - > { data } } ,
"Error processing stanza input: No jobdir entries found. " ;
xCAT::MsgUtils - > message ( "E" , $ rsp , $ ::CALLBACK ) ;
return 1 ;
}
unless ( - d $ lljobs_dir ) {
2008-10-27 17:11:23 +00:00
unless ( File::Path:: mkpath ( $ lljobs_dir ) ) {
2008-10-22 16:02:14 +00:00
my $ rsp ;
push @ { $ rsp - > { data } } , "Could not create directory $lljobs_dir" ;
xCAT::MsgUtils - > message ( "E" , $ rsp , $ ::CALLBACK ) ;
return 1 ;
}
unless ( chown ( $ uid , $ gid , $ lljobs_dir ) ) {
my $ rsp ;
push @ { $ rsp - > { data } } ,
"Could not change owner of directory $lljobs_dir to $lluser" ;
xCAT::MsgUtils - > message ( "E" , $ rsp , $ ::CALLBACK ) ;
return 1 ;
}
}
2010-09-14 22:29:41 +00:00
my $ nodestatus = $ ::FILEATTRS { bringupstatus } [ 0 ] ;
my $ appstatus = $ ::FILEATTRS { bringupappstatus } [ 0 ] ;
my $ df_statusline = "" ;
if ( defined ( $ appstatus ) ) {
$ df_statusline = "bringupappstatus=$appstatus\n" ;
} elsif ( defined ( $ nodestatus ) ) {
$ df_statusline = "bringupstatus=$nodestatus\n" ;
} else {
$ df_statusline = "bringupstatus=booted\n" ;
}
my $ run_if_down = "cancel" ;
if ( defined ( $ ::FILEATTRS { update_if_down } [ 0 ] ) ) {
$ run_if_down = $ ::FILEATTRS { update_if_down } [ 0 ] ;
$ run_if_down =~ tr /[A-Z]/ [ a - z ] / ;
if ( $ run_if_down eq 'y' ) { $ run_if_down = 'yes' ; }
if ( $ ::updateall && ( $ run_if_down eq 'yes' ) ) {
$ run_if_down = 'cancel' ;
my $ rsp ;
push @ { $ rsp - > { data } } , "update_all=yes, but update_if_down is yes which is not allowed. update_if_down=cancel will be assumed. " ;
xCAT::MsgUtils - > message ( "I" , $ rsp , $ ::CALLBACK ) ;
}
}
2008-10-22 16:02:14 +00:00
my @ calldirectly ;
ugloop: foreach my $ ugname ( keys % { $ updategroup } ) {
2010-09-14 22:29:41 +00:00
$ ::updateall_feature = " " ;
if ( $ ::updateall ) { $ ::updateall_feature = "XCAT_$ugname" ; }
# Build node substitution strings
2008-10-22 16:02:14 +00:00
my ( $ nodelist , $ machinelist ) ;
my $ machinecount = 0 ;
foreach my $ node ( @ { $ updategroup - > { $ ugname } } ) {
if ( defined ( $ machines { $ node } )
2010-09-14 22:29:41 +00:00
&& ( $ machines { $ node } { 'mstatus' } eq "1" ) ) {
$ machinelist . = " $machines{$node}{'mname'}" ;
2008-10-22 16:02:14 +00:00
$ machinecount + + ;
$ nodelist . = ",$node" ;
2010-09-14 22:29:41 +00:00
} elsif ( $ run_if_down eq 'yes' ) {
if ( defined ( $ machines { $ node } ) ) {
# llmkres -D will allow reserving down nodes as long
# as they are present in the machine list
$ machinelist . = " $machines{$node}{'mname'}" ;
$ machinecount + + ;
}
2008-10-22 16:02:14 +00:00
$ nodelist . = ",$node" ;
2010-09-14 22:29:41 +00:00
} elsif ( $ run_if_down eq 'cancel' ) {
2008-10-22 16:02:14 +00:00
my $ rsp ;
push @ { $ rsp - > { data } } ,
"Node $node is not active in LL and \"update_if_down=cancel\". Update for updategroup $ugname is canceled." ;
xCAT::MsgUtils - > message ( "E" , $ rsp , $ ::CALLBACK ) ;
+ + $ rc ;
next ugloop ;
}
}
2010-09-14 22:29:41 +00:00
if ( defined ( $ nodelist ) ) { $ nodelist =~ s/^\,// ; }
# Build updategroup data file
my @ ugdflines ;
push ( @ ugdflines , "# xCAT Rolling Update data file for update group $ugname \n" ) ;
push ( @ ugdflines , "\n" ) ;
push ( @ ugdflines , "updategroup=$ugname\n" ) ;
if ( $ ::updateall ) {
push ( @ ugdflines , "updatefeature=$::updateall_feature\n" ) ;
} else {
push ( @ ugdflines , "nodelist=$nodelist\n" ) ;
}
if ( defined ( $ ::FILEATTRS { oldfeature } [ 0 ] ) ) {
push ( @ ugdflines , "oldfeature=$::FILEATTRS{oldfeature}[0]\n" ) ;
}
if ( defined ( $ ::FILEATTRS { newfeature } [ 0 ] ) ) {
push ( @ ugdflines , "newfeature=$::FILEATTRS{newfeature}[0]\n" ) ;
}
push ( @ ugdflines , "\n" ) ;
push ( @ ugdflines , & get_prescripts ( $ nodelist ) ) ;
if ( defined ( $ ::FILEATTRS { shutdowntimeout } [ 0 ] ) ) {
push ( @ ugdflines , "shutdowntimeout=$::FILEATTRS{shutdowntimeout}[0]\n" ) ;
}
push ( @ ugdflines , & get_outofband ( $ nodelist ) ) ;
push ( @ ugdflines , & get_bringuporder ( $ nodelist ) ) ;
push ( @ ugdflines , $ df_statusline ) ;
if ( defined ( $ ::FILEATTRS { bringuptimeout } [ 0 ] ) ) {
push ( @ ugdflines , "bringuptimeout=$::FILEATTRS{bringuptimeout}[0]\n" ) ;
}
my $ ugdf_file = $ lljobs_dir . "/rollupdate_" . $ ugname . ".data" ;
my $ UGDFFILE ;
unless ( open ( $ UGDFFILE , ">$ugdf_file" ) ) {
my $ rsp ;
push @ { $ rsp - > { data } } , "Could not open file $ugdf_file" ;
xCAT::MsgUtils - > message ( "E" , $ rsp , $ ::CALLBACK ) ;
return 1 ;
}
if ( $ ::VERBOSE ) {
my $ rsp ;
push @ { $ rsp - > { data } } , "Writing xCAT rolling update data file $ugdf_file " ;
xCAT::MsgUtils - > message ( "I" , $ rsp , $ ::CALLBACK ) ;
open ( RULOG , ">>$::LOGDIR/$::LOGFILE" ) ;
print RULOG localtime ( ) . " Writing xCAT rolling update data file $ugdf_file \n" ;
close ( RULOG ) ;
}
print $ UGDFFILE @ ugdflines ;
close ( $ UGDFFILE ) ;
chown ( $ uid , $ gid , $ ugdf_file ) ;
if ( defined ( $ machinelist ) || $ ::updateall ) {
my $ llhl_file ;
if ( ! $ ::updateall ) {
$ machinelist =~ s/^\s+// ;
# Build LL hostlist file
my $ hllines = $ machinelist ;
$ hllines =~ s/"//g ;
$ hllines =~ s/\s+/\n/g ;
$ hllines . = "\n" ;
$ llhl_file = $ lljobs_dir . "/rollupdate_" . $ ugname . ".hostlist" ;
my $ HLFILE ;
unless ( open ( $ HLFILE , ">$llhl_file" ) ) {
my $ rsp ;
push @ { $ rsp - > { data } } , "Could not open file $llhl_file" ;
xCAT::MsgUtils - > message ( "E" , $ rsp , $ ::CALLBACK ) ;
return 1 ;
}
if ( $ ::VRBOSE ) {
my $ rsp ;
push @ { $ rsp - > { data } } , "Writing LL hostlist file $llhl_file " ;
xCAT::MsgUtils - > message ( "I" , $ rsp , $ ::CALLBACK ) ;
}
print $ HLFILE $ hllines ;
close ( $ HLFILE ) ;
chown ( $ uid , $ gid , $ llhl_file ) ;
}
# Build reservation callback script
my @ rcblines ;
my $ rcbcmd = $ ::FILEATTRS { 'reservationcallback' } [ 0 ] ;
if ( ! defined ( $ rcbcmd ) ) { $ rcbcmd = "$::XCATROOT/bin/runrollupdate" ; }
push ( @ rcblines , "#!/bin/sh \n" ) ;
push ( @ rcblines , "# LL Reservation Callback script for xCAT Rolling Update group $ugname \n" ) ;
push ( @ rcblines , "\n" ) ;
push ( @ rcblines , "if [ \"\$2\" == \"RESERVATION_ACTIVE\" ] ; then\n" ) ;
my $ send_verbose = "" ;
if ( $ ::VERBOSE ) { $ send_verbose = "--verbose" ; }
push ( @ rcblines , " $rcbcmd $send_verbose loadleveler $ugdf_file \$1 &\n" ) ;
push ( @ rcblines , "fi \n" ) ;
push ( @ rcblines , "\n" ) ;
my $ llrcb_file = $ lljobs_dir . "/rollupdate_" . $ ugname . ".rsvcb" ;
my $ RCBFILE ;
unless ( open ( $ RCBFILE , ">$llrcb_file" ) ) {
my $ rsp ;
push @ { $ rsp - > { data } } , "Could not open file $llrcb_file" ;
xCAT::MsgUtils - > message ( "E" , $ rsp , $ ::CALLBACK ) ;
return 1 ;
}
if ( $ ::VERBOSE ) {
my $ rsp ;
push @ { $ rsp - > { data } } , "Writing LL reservation callback script $llrcb_file " ;
xCAT::MsgUtils - > message ( "I" , $ rsp , $ ::CALLBACK ) ;
open ( RULOG , ">>$::LOGDIR/$::LOGFILE" ) ;
print RULOG localtime ( ) . " Writing xCAT rolling update data file $ugdf_file \n" ;
close ( RULOG ) ;
}
print $ RCBFILE @ rcblines ;
close ( $ RCBFILE ) ;
chown ( $ uid , $ gid , $ llrcb_file ) ;
chmod ( 0700 , $ llrcb_file ) ;
2008-10-22 16:02:14 +00:00
# Build output file
2010-09-14 22:29:41 +00:00
my $ mutex_string = & get_mutex ( $ ugname ) ;
my $ lastcount = 0 ;
my $ llcount = $ machinecount ;
if ( $ ::updateall ) {
$ lastcount = $ machinecount % $ ::updateall_numperupdate ;
$ llcount = $ ::updateall_numperupdate ;
}
2008-10-22 16:02:14 +00:00
my @ jclines ;
2010-09-14 22:29:41 +00:00
my @ jclines2 ;
2008-10-22 16:02:14 +00:00
foreach my $ line ( @ lines ) {
my $ jcline = $ line ;
2010-09-14 22:29:41 +00:00
my $ jcline2 ;
2008-10-22 16:02:14 +00:00
$ jcline =~ s/\[\[NODESET\]\]/$ugname/ ;
2010-09-14 22:29:41 +00:00
$ jcline =~ s/\[\[JOBDIR\]\]/$lljobs_dir/ ;
if ( defined ( $ nodelist ) ) {
$ jcline =~ s/\[\[XNODELIST\]\]/$nodelist/ ;
} else {
$ jcline =~ s/\[\[XNODELIST\]\]// ;
}
if ( defined ( $ llhl_file ) ) {
$ jcline =~ s/\[\[LLHOSTFILE\]\]/$llhl_file/ ;
} else {
$ jcline =~ s/\[\[LLHOSTFILE\]\]// ;
}
if ( defined ( $ ::FILEATTRS { oldfeature } [ 0 ] ) ) {
$ jcline =~ s/\[\[OLDFEATURE\]\]/$::FILEATTRS{oldfeature}[0]/ ;
} else {
$ jcline =~ s/\[\[OLDFEATURE\]\]// ;
}
$ jcline =~ s/\[\[UPDATEALLFEATURE\]\]/$::updateall_feature/ ;
$ jcline =~ s/\[\[MUTEXRESOURCES\]\]/$mutex_string/ ;
# LL is VERY picky about extra blanks in Feature string
if ( $ jcline =~ /Feature/ ) {
$ jcline =~ s/\"\s+/\"/g ;
$ jcline =~ s/\s+\"/\"/g ;
}
if ( $ lastcount ) {
$ jcline2 = $ jcline ;
$ jcline2 =~ s/\[\[LLCOUNT\]\]/$lastcount/ ;
push ( @ jclines2 , $ jcline2 ) ;
}
if ( $ jcline =~ /\[\[LLCOUNT\]\]/ ) {
$ jcline =~ s/\[\[LLCOUNT\]\]/$llcount/ ;
}
2008-10-22 16:02:14 +00:00
push ( @ jclines , $ jcline ) ;
}
2008-10-29 13:44:36 +00:00
my $ lljob_file = $ lljobs_dir . "/rollupdate_" . $ ugname . ".cmd" ;
2008-10-22 16:02:14 +00:00
my $ JOBFILE ;
unless ( open ( $ JOBFILE , ">$lljob_file" ) ) {
my $ rsp ;
push @ { $ rsp - > { data } } , "Could not open file $lljob_file" ;
xCAT::MsgUtils - > message ( "E" , $ rsp , $ ::CALLBACK ) ;
return 1 ;
}
2010-09-14 22:29:41 +00:00
if ( $ ::VERBOSE ) {
my $ rsp ;
push @ { $ rsp - > { data } } , "Writing LL job command file $lljob_file " ;
xCAT::MsgUtils - > message ( "I" , $ rsp , $ ::CALLBACK ) ;
open ( RULOG , ">>$::LOGDIR/$::LOGFILE" ) ;
print RULOG localtime ( ) . " Writing LL job command file $lljob_file \n" ;
close ( RULOG ) ;
}
2008-10-22 16:02:14 +00:00
print $ JOBFILE @ jclines ;
close ( $ JOBFILE ) ;
chown ( $ uid , $ gid , $ lljob_file ) ;
2010-09-14 22:29:41 +00:00
my $ lljob_file2 = $ lljobs_dir . "/rollupdate_LAST_" . $ ugname . ".cmd" ;
if ( $ lastcount ) {
my $ JOBFILE2 ;
unless ( open ( $ JOBFILE2 , ">$lljob_file2" ) ) {
my $ rsp ;
push @ { $ rsp - > { data } } , "Could not open file $lljob_file2" ;
xCAT::MsgUtils - > message ( "E" , $ rsp , $ ::CALLBACK ) ;
return 1 ;
}
if ( $ ::VERBOSE ) {
my $ rsp ;
push @ { $ rsp - > { data } } , "Writing LL job command file $lljob_file2 " ;
xCAT::MsgUtils - > message ( "I" , $ rsp , $ ::CALLBACK ) ;
open ( RULOG , ">>$::LOGDIR/$::LOGFILE" ) ;
print RULOG localtime ( ) . " Writing LL job command file $lljob_file2 \n" ;
close ( RULOG ) ;
}
print $ JOBFILE2 @ jclines2 ;
close ( $ JOBFILE2 ) ;
chown ( $ uid , $ gid , $ lljob_file2 ) ;
}
if ( $ ::updateall ) {
& set_LL_feature ( $ machinelist , $ ::updateall_feature ) ;
2010-09-15 21:08:19 +00:00
2010-09-14 22:29:41 +00:00
}
2008-10-22 16:02:14 +00:00
2010-09-14 22:29:41 +00:00
# Need to change status before actually submitting LL jobs
# If LL jobs happen to run right away, the update code checking
# for the status may run before we've had a chance to actually update it
2009-01-05 18:02:00 +00:00
my $ nltab = xCAT::Table - > new ( 'nodelist' ) ;
my @ nodes = split ( /\,/ , $ nodelist ) ;
2010-09-14 22:29:41 +00:00
xCAT::Utils - > setAppStatus ( \ @ nodes , "RollingUpdate" , "update_job_submitted" ) ;
# Submit LL reservation
my $ downnodes = "-D" ;
if ( $ ::updateall ) { $ downnodes = " " ; }
my $ cmd = qq~su - $lluser "-c llmkres -x -d $::FILEATTRS{'reservationduration'}[0] -f $lljob_file -p $llrcb_file $downnodes "~ ;
my $ cmd2 = qq~su - $lluser "-c llmkres -x -d $::FILEATTRS{'reservationduration'}[0] -f $lljob_file2 -p $llrcb_file $downnodes "~ ;
if ( $ ::VERBOSE ) {
2008-10-22 16:02:14 +00:00
my $ rsp ;
2010-09-14 22:29:41 +00:00
push @ { $ rsp - > { data } } , "Running command: $cmd " ;
xCAT::MsgUtils - > message ( "I" , $ rsp , $ ::CALLBACK ) ;
open ( RULOG , ">>$::LOGDIR/$::LOGFILE" ) ;
print RULOG localtime ( ) . " Running command: $cmd \n" ;
close ( RULOG ) ;
2008-10-22 16:02:14 +00:00
}
2010-09-14 22:29:41 +00:00
my @ llsubmit ;
if ( $ ::TEST ) {
my $ rsp ;
push @ { $ rsp - > { data } } , "In TEST mode. Will NOT run command: $cmd " ;
xCAT::MsgUtils - > message ( "I" , $ rsp , $ ::CALLBACK ) ;
$ ::RUNCMD_RC = 0 ;
} else {
my $ submit_count = 1 ;
if ( $ ::updateall ) {
$ submit_count = $ machinecount / $ ::updateall_numperupdate ;
}
for ( 1 .. $ submit_count ) {
@ llsubmit = xCAT::Utils - > runcmd ( "$cmd" , 0 ) ;
if ( $ ::RUNCMD_RC != 0 ) {
my $ rsp ;
push @ { $ rsp - > { data } } , "Could not run llmkres command." ;
push @ { $ rsp - > { data } } , @ llsubmit ;
xCAT::MsgUtils - > message ( "E" , $ rsp , $ ::CALLBACK ) ;
return 1 ;
}
if ( $ ::VERBOSE ) {
my $ rsp ;
push @ { $ rsp - > { data } } , @ llsubmit ;
xCAT::MsgUtils - > message ( "I" , $ rsp , $ ::CALLBACK ) ;
open ( RULOG , ">>$::LOGDIR/$::LOGFILE" ) ;
print RULOG @ llsubmit ;
close ( RULOG ) ;
}
}
if ( $ lastcount ) {
if ( $ ::VERBOSE ) {
my $ rsp ;
push @ { $ rsp - > { data } } , "Running command: $cmd2 " ;
xCAT::MsgUtils - > message ( "I" , $ rsp , $ ::CALLBACK ) ;
open ( RULOG , ">>$::LOGDIR/$::LOGFILE" ) ;
print RULOG localtime ( ) . " Running command: $cmd2 \n" ;
close ( RULOG ) ;
}
@ llsubmit = xCAT::Utils - > runcmd ( "$cmd2" , 0 ) ;
if ( $ ::RUNCMD_RC != 0 ) {
my $ rsp ;
push @ { $ rsp - > { data } } , "Could not run llmkres command." ;
push @ { $ rsp - > { data } } , @ llsubmit ;
xCAT::MsgUtils - > message ( "E" , $ rsp , $ ::CALLBACK ) ;
open ( RULOG , ">>$::LOGDIR/$::LOGFILE" ) ;
print RULOG @ llsubmit ;
close ( RULOG ) ;
return 1 ;
}
if ( $ ::VERBOSE ) {
my $ rsp ;
push @ { $ rsp - > { data } } , @ llsubmit ;
xCAT::MsgUtils - > message ( "I" , $ rsp , $ ::CALLBACK ) ;
open ( RULOG , ">>$::LOGDIR/$::LOGFILE" ) ;
print RULOG @ llsubmit ;
close ( RULOG ) ;
}
}
}
2008-10-22 16:02:14 +00:00
}
elsif ( defined ( $ nodelist ) ) {
# No nodes in LL to submit job to -- not able to schedule.
# Call xCAT directly for all other nodes.
# TODO - this will serialize updating the updategroups
2010-09-14 22:29:41 +00:00
# is this okay, or do we want forked child processes?
2008-10-22 16:02:14 +00:00
push @ calldirectly , $ ugname ;
}
2010-09-14 22:29:41 +00:00
} # end ugloop
2008-10-22 16:02:14 +00:00
if ( scalar ( @ calldirectly ) > 0 ) {
2010-09-14 22:29:41 +00:00
my @ children ;
2008-10-22 16:02:14 +00:00
foreach my $ ugname ( @ calldirectly ) {
my $ nodelist = join ( ',' , @ { $ updategroup - > { $ ugname } } ) ;
2010-09-14 22:29:41 +00:00
my $ ugdf = $ lljobs_dir . "/rollupdate_" . $ ugname . ".data" ;
2008-10-22 16:02:14 +00:00
my $ rsp ;
push @ { $ rsp - > { data } } ,
"No active LL nodes in update group $ugname" ;
push @ { $ rsp - > { data } } ,
"These nodes will be updated now. This will take a few minutes..." ;
xCAT::MsgUtils - > message ( "I" , $ rsp , $ ::CALLBACK ) ;
2010-09-14 22:29:41 +00:00
if ( $ ::VERBOSE ) {
open ( RULOG , ">>$::LOGDIR/$::LOGFILE" ) ;
print RULOG localtime ( ) . " No active LL nodes in update group $ugname. These nodes will be updated now.\n" ;
close ( RULOG ) ;
}
2008-10-22 16:02:14 +00:00
my $ nltab = xCAT::Table - > new ( 'nodelist' ) ;
my @ nodes = split ( /\,/ , $ nodelist ) ;
2010-09-14 22:29:41 +00:00
xCAT::Utils - > setAppStatus ( \ @ nodes , "RollingUpdate" , "update_job_submitted" ) ;
my $ childpid = runrollupdate ( { command = > [ 'runrollupdate' ] ,
arg = > [ 'internal' , 'loadleveler' , $ ugdf ]
} ) ;
if ( defined ( $ childpid ) && ( $ childpid != 0 ) ) {
push ( @ children , $ childpid ) ;
}
}
# wait until all the children are finished before returning
foreach my $ child ( @ children ) {
if ( $ ::VERBOSE ) {
my $ rsp ;
push @ { $ rsp - > { data } } , "Waiting for child PID $child to complete " ;
xCAT::MsgUtils - > message ( "I" , $ rsp , $ ::CALLBACK ) ;
open ( RULOG , ">>$::LOGDIR/$::LOGFILE" ) ;
print RULOG localtime ( ) . " Waiting for child PID $child to complete \n" ;
close ( RULOG ) ;
}
waitpid ( $ child , 0 ) ;
}
2008-10-22 16:02:14 +00:00
}
return $ rc ;
}
2010-09-14 22:29:41 +00:00
2008-10-22 16:02:14 +00:00
#----------------------------------------------------------------------------
2010-09-14 22:29:41 +00:00
= head3 check_policy
2008-10-22 16:02:14 +00:00
2010-09-14 22:29:41 +00:00
Check the policy table to see if userid is authorized to run command
2008-10-22 16:02:14 +00:00
2010-09-14 22:29:41 +00:00
Arguments: userid , command
2008-10-22 16:02:14 +00:00
Returns:
0 - OK
1 - error
Globals:
Error:
Example:
Comments:
2010-09-14 22:29:41 +00:00
2008-10-22 16:02:14 +00:00
= cut
#-----------------------------------------------------------------------------
2010-09-14 22:29:41 +00:00
sub check_policy {
my $ userid = shift ;
my $ xcatcmd = shift ;
my $ policytable = xCAT::Table - > new ( 'policy' ) ;
unless ( $ policytable ) {
return 1 ;
}
my $ policies = $ policytable - > getAllEntries ;
$ policytable - > close ;
foreach my $ rule ( @$ policies ) {
if ( $ rule - > { name } &&
( ( $ rule - > { name } eq "*" ) || ( $ rule - > { name } eq $ userid ) ) ) {
if ( $ rule - > { commands } &&
( ( $ rule - > { commands } eq "*" ) || ( $ rule - > { commands } =~ /$xcatcmd/ ) ) ) {
return 0 ; # match found
}
}
}
return 1 ; # no match found
}
2008-10-22 16:02:14 +00:00
2008-10-22 18:52:40 +00:00
2010-09-14 22:29:41 +00:00
#----------------------------------------------------------------------------
2008-10-22 16:02:14 +00:00
2010-09-14 22:29:41 +00:00
= head3 set_LL_feature
Sets the specified feature for the list of LL machines
Arguments: $ machinelist - blank delimited list of LL machines
$ feature - feature value to set
Returns:
0 - OK
1 - error
Globals:
Error:
Example:
Comments:
2008-10-22 16:02:14 +00:00
2010-09-14 22:29:41 +00:00
= cut
2009-01-05 18:02:00 +00:00
2010-09-14 22:29:41 +00:00
#-----------------------------------------------------------------------------
sub set_LL_feature {
my $ machinelist = shift ;
my $ feature = shift ;
# Query current feature
my $ cmd = "llconfig -h $machinelist -d FEATURE" ;
if ( $ ::VERBOSE ) {
open ( RULOG , ">>$::LOGDIR/$::LOGFILE" ) ;
print RULOG localtime ( ) . " Running command \'$cmd\'\n" ;
close ( RULOG ) ;
}
my @ llcfgoutput = xCAT::Utils - > runcmd ( $ cmd , 0 ) ;
if ( $ ::VERBOSE ) {
open ( RULOG , ">>$::LOGDIR/$::LOGFILE" ) ;
print RULOG localtime ( ) . " Return code: $::RUNCMD_RC\n" ;
close ( RULOG ) ;
}
2008-10-22 16:02:14 +00:00
2010-09-14 22:29:41 +00:00
my % set_features ;
my % had_cfg ;
foreach my $ llcfgout ( @ llcfgoutput ) {
my @ llfeatures ;
my $ haveit = 0 ;
2010-09-15 21:08:19 +00:00
if ( $ llcfgout =~ /:FEATURE =/ ) {
2010-09-14 22:29:41 +00:00
my ( $ stuff , $ newfeature_string ) = split ( /=/ , $ llcfgout ) ;
my ( $ machine , $ morestuff ) = split ( /:/ , $ stuff ) ;
@ llfeatures = split ( /\s+/ , $ newfeature_string ) ;
foreach my $ f ( @ llfeatures ) {
if ( $ f eq $ feature ) {
$ haveit = 1 ;
}
2008-10-22 16:02:14 +00:00
}
2010-09-14 22:29:41 +00:00
if ( ! $ haveit ) {
$ newfeature_string . = " $feature" ;
2008-10-22 16:02:14 +00:00
}
2010-09-14 22:29:41 +00:00
$ set_features { $ newfeature_string } . = " $machine" ;
$ had_cfg { $ machine } = 1 ;
}
}
foreach my $ m ( split ( /\s+/ , $ machinelist ) ) {
if ( ! $ had_cfg { $ m } ) {
$ set_features { $ feature } . = " $m" ;
2008-10-22 16:02:14 +00:00
}
}
2010-09-14 22:29:41 +00:00
# Change in LL database
foreach my $ sf ( keys % set_features ) {
if ( $ set_features { $ sf } !~ /^\s*$/ ) {
$ cmd = "llconfig -h $set_features{$sf} -c FEATURE=\"$sf\"" ;
if ( $ ::VERBOSE ) {
open ( RULOG , ">>$::LOGDIR/$::LOGFILE" ) ;
print RULOG localtime ( ) . " Running command \'$cmd\'\n" ;
close ( RULOG ) ;
}
if ( $ ::TEST ) {
my $ rsp ;
push @ { $ rsp - > { data } } , "In TEST mode. Will NOT run command: $cmd " ;
xCAT::MsgUtils - > message ( "I" , $ rsp , $ ::CALLBACK ) ;
$ ::RUNCMD_RC = 0 ;
} else {
xCAT::Utils - > runcmd ( $ cmd , 0 ) ;
2010-09-15 21:08:19 +00:00
###
### DEBUG -- llconfig problem
### workaround - sleep to let ll daemons catch up
sleep 3 ;
2010-09-14 22:29:41 +00:00
if ( $ ::VERBOSE ) {
open ( RULOG , ">>$::LOGDIR/$::LOGFILE" ) ;
print RULOG localtime ( ) . " Return code: $::RUNCMD_RC\n" ;
close ( RULOG ) ;
2008-10-22 16:02:14 +00:00
}
}
}
2010-09-14 22:29:41 +00:00
}
2010-09-15 21:08:19 +00:00
###
### DEBUG -- llconfig problem
### need to send a manual reconfig to at least one node to have it
### register in LL daemons
### not working - try sleep above instead
#sleep 3;
# my @rcfns = split(/\s+/,$machinelist);
# my @rcfg = `ssh $rcfns[0] llrctl reconfig`;
# if ($::VERBOSE) {
# open (RULOG, ">>$::LOGDIR/$::LOGFILE");
# print RULOG localtime()." llconfig workaround, run \"ssh $rcfns[0] llrctl reconfig \", output: \n";
# print RULOG @rcfg;
# close (RULOG);
# }
### end DEBUG
2010-09-14 22:29:41 +00:00
return 0 ;
}
2008-10-22 16:02:14 +00:00
2010-09-14 22:29:41 +00:00
#----------------------------------------------------------------------------
= head3 get_prescripts
Generate the prescripts for this nodelist
Arguments:
Returns:
0 - OK
1 - error
Globals:
Error:
Example:
Comments:
= cut
#-----------------------------------------------------------------------------
sub get_prescripts {
my $ nodelist = shift ;
my @ nl = split ( /,/ , $ nodelist ) ;
my @ datalines ;
my $ psindex = 0 ;
foreach my $ psl ( @ { $ ::FILEATTRS { 'prescript' } } ) {
my $ psline = $ psl ;
if ( $ ::updateall ) {
push ( @ datalines , "prescript=" . $ psline . "\n" ) ;
$ psindex + + ;
next ;
2008-10-22 16:02:14 +00:00
}
2010-09-14 22:29:41 +00:00
my $ psnoderange = $ ::FILEATTRS { 'prescriptnodes' } [ $ psindex ] ;
my $ executenodelist = "" ;
if ( $ psnoderange eq "ALL_NODES_IN_UPDATEGROUP" ) {
$ executenodelist = $ nodelist ;
} else {
my @ psns = xCAT::NodeRange:: noderange ( $ psnoderange ) ;
unless ( @ psns ) { $ psindex + + ; next ; }
my @ executenodes ;
foreach my $ node ( @ nl ) {
push ( @ executenodes , grep ( /^$node$/ , @ psns ) ) ;
}
$ executenodelist = join ( ',' , @ executenodes ) ;
}
if ( $ executenodelist ne "" ) {
$ psline =~ s/\$NODELIST/$executenodelist/g ;
push ( @ datalines , "prescript=" . $ psline . "\n" ) ;
}
$ psindex + + ;
2008-10-22 16:02:14 +00:00
}
2010-09-14 22:29:41 +00:00
return @ datalines ;
2008-10-22 16:02:14 +00:00
}
2010-09-14 22:29:41 +00:00
#----------------------------------------------------------------------------
= head3 get_outofband
Generate the out of band scripts for this nodelist
Arguments:
Returns:
0 - OK
1 - error
Globals:
Error:
Example:
Comments:
= cut
#-----------------------------------------------------------------------------
sub get_outofband {
my $ nodelist = shift ;
my @ nl = split ( /,/ , $ nodelist ) ;
my @ datalines ;
my $ obindex = 0 ;
foreach my $ obl ( @ { $ ::FILEATTRS { 'outofbandcmd' } } ) {
my $ obline = $ obl ;
if ( $ ::updateall ) {
push ( @ datalines , "outofbandcmd=" . $ obline . "\n" ) ;
$ obindex + + ;
next ;
}
my $ obnoderange = $ ::FILEATTRS { 'outofbandnodes' } [ $ obindex ] ;
my $ executenodelist = "" ;
if ( $ obnoderange eq "ALL_NODES_IN_UPDATEGROUP" ) {
$ executenodelist = $ nodelist ;
} else {
my @ obns = xCAT::NodeRange:: noderange ( $ obnoderange ) ;
unless ( @ obns ) { $ obindex + + ; next ; }
my @ executenodes ;
foreach my $ node ( @ nl ) {
push ( @ executenodes , grep ( /^$node$/ , @ obns ) ) ;
}
$ executenodelist = join ( ',' , @ executenodes ) ;
}
if ( $ executenodelist ne "" ) {
$ obline =~ s/\$NODELIST/$executenodelist/g ;
push ( @ datalines , "outofbandcmd=" . $ obline . "\n" ) ;
}
$ obindex + + ;
}
return @ datalines ;
}
#----------------------------------------------------------------------------
= head3 get_bringuporder
Generate the bringup order for this nodelist
Arguments:
Returns:
0 - OK
1 - error
Globals:
Error:
Example:
Comments:
= cut
#-----------------------------------------------------------------------------
sub get_bringuporder {
my $ nodelist = shift ;
my @ nl = split ( /,/ , $ nodelist ) ;
my @ datalines ;
if ( $ ::updateall ) {
$ datalines [ 0 ] = "\n" ;
return @ datalines ;
}
foreach my $ buline ( @ { $ ::FILEATTRS { 'bringuporder' } } ) {
my @ buns = xCAT::NodeRange:: noderange ( $ buline ) ;
unless ( @ buns ) { next ; }
my @ bringupnodes ;
foreach my $ node ( @ nl ) {
if ( defined ( $ node ) ) {
( my $ found ) = grep ( /^$node$/ , @ buns ) ;
if ( $ found ) {
push ( @ bringupnodes , $ found ) ;
undef $ node ; # don't try to use this node again
}
}
}
my $ bringupnodelist = join ( ',' , @ bringupnodes ) ;
if ( $ bringupnodelist ne "" ) {
push ( @ datalines , "bringuporder=" . $ bringupnodelist . "\n" ) ;
}
}
return @ datalines ;
}
#----------------------------------------------------------------------------
= head3 get_mutex
Generate the list of LL mutual exclusion resources for this
update group
Arguments:
Returns:
0 - OK
1 - error
Globals:
Error:
Example:
Comments:
= cut
#-----------------------------------------------------------------------------
sub get_mutex {
my $ ugname = shift ;
my $ mutex_string = "" ;
my $ max_updates = $ ::FILEATTRS { 'maxupdates' } [ 0 ] ;
if ( defined ( $ max_updates ) && ( $ max_updates ne 'all' ) ) {
$ mutex_string . = "XCATROLLINGUPDATE_MAXUPDATES(1) " ;
}
if ( $ ::updateall ) {
return $ mutex_string ;
}
my $ mutex_count = scalar @ ::MUTEX ;
if ( $ mutex_count > 0 ) {
foreach my $ row ( 0 .. ( $ mutex_count - 1 ) ) {
foreach my $ ugi ( 0 .. ( @ { $ ::MUTEX [ $ row ] } - 1 ) ) {
if ( defined ( $ ::MUTEX [ $ row ] [ $ ugi ] ) && ( $ ugname eq $ ::MUTEX [ $ row ] [ $ ugi ] ) ) {
$ mutex_string . = "XCATROLLINGUPDATE_MUTEX" . $ row . "(1) " ;
last ;
}
}
}
}
return $ mutex_string ;
}
#----------------------------------------------------------------------------
= head3 create_LL_mutex_resources
Create all required LL mutex resources
Arguments:
Returns:
0 - OK
1 - error
Globals:
Error:
Example:
Comments: Reads the mutex entries in $ ::FILEATTRS and
sets the global array $ ::MUTEX
which is a multi - dimensional array:
Each row is an array of mutually exclusive update
group names
A LL Floating Resource will be created for each
row of the array .
= cut
#-----------------------------------------------------------------------------
sub create_LL_mutex_resources {
$ ::LL_MUTEX_RESOURCES_CREATED = 0 ;
my $ mxindex = 0 ;
foreach my $ mxline ( @ { $ ::FILEATTRS { 'mutex' } } ) {
my @ mxparts = split ( /,/ , $ mxline ) ;
if ( scalar @ mxparts < 2 ) {
my $ rsp ;
push @ { $ rsp - > { data } } , "Error processing stanza line: " ;
push @ { $ rsp - > { data } } , "mutex=" . $ mxline ;
push @ { $ rsp - > { data } } , "Value must contain at least 2 update groups" ;
xCAT::MsgUtils - > message ( "E" , $ rsp , $ ::CALLBACK ) ;
return 1 ;
}
my $ mxpi = 0 ;
my $ mxindexmax = $ mxindex ;
my @ ugnames ;
foreach my $ mxpart ( @ mxparts ) {
my $ mxindex2 = $ mxindex ;
my @ ugnamelist = xCAT::NameRange:: namerange ( $ mxpart , 0 ) ;
foreach my $ ugname ( @ ugnamelist ) {
$ ::MUTEX [ $ mxindex2 ] [ $ mxpi ] = $ ugname ;
$ mxindex2 + + ;
}
$ mxindexmax = ( $ mxindex2 > $ mxindexmax ) ? $ mxindex : $ mxindexmax ;
$ mxpi + + ;
}
$ mxindex = $ mxindexmax ;
}
my $ resource_string = "" ;
my $ max_updates = $ ::FILEATTRS { 'maxupdates' } [ 0 ] ;
if ( ! defined ( $ max_updates ) || ( $ max_updates eq 'all' ) ) {
$ max_updates = 0 ;
} else {
$ resource_string . = "XCATROLLINGUPDATE_MAXUPDATES($max_updates) " ;
}
my $ mutex_count = scalar @ ::MUTEX ;
if ( $ mutex_count > 0 ) {
foreach my $ row ( 0 .. ( $ mutex_count - 1 ) ) {
# TODO -- UNCOMMENT/REPLACE WHEN LL PROVIDES RESERVATION RESOURCES
$ resource_string . = "XCATROLLINGUPDATE_MUTEX" . $ row . "(1) " ;
}
}
if ( $ resource_string ) {
my $ cmd = "llconfig -d FLOATING_RESOURCES SCHEDULE_BY_RESOURCES CENTRAL_MANAGER_LIST RESOURCE_MGR_LIST" ;
my @ llcfg_d = xCAT::Utils - > runcmd ( $ cmd , 0 ) ;
my $ curSCHED = "" ;
my $ curFLOAT = "" ;
my $ llcms = "" ;
my $ llrms = "" ;
foreach my $ cfgo ( @ llcfg_d ) {
chomp $ cfgo ;
my ( $ llattr , $ llval ) = split ( / = / , $ cfgo ) ;
if ( $ llattr =~ /SCHEDULE_BY_RESOURCES/ ) {
$ curSCHED = $ llval ; }
if ( $ llattr =~ /FLOATING_RESOURCES/ ) {
$ curFLOAT = $ llval ; }
if ( $ llattr =~ /CENTRAL_MANAGER_LIST/ ) {
$ llcms = $ llval ; }
if ( $ llattr =~ /RESOURCE_MGR_LIST/ ) {
$ llrms = $ llval ; }
}
$ cmd = "llconfig -c " ;
my $ updateFLOAT = 0 ;
my $ updateSCHED = 0 ;
foreach my $ float ( split ( /\s+/ , $ resource_string ) ) {
$ float =~ s/\(/./g ;
$ float =~ s/\)/./g ;
if ( $ curFLOAT !~ /$float/ ) {
$ updateFLOAT = 1 ;
last ;
}
}
if ( $ updateFLOAT ) {
$ curFLOAT =~ s/XCATROLLINGUPDATE_MUTEX(\d)*\((\d)*\)//g ;
$ curFLOAT =~ s/XCATROLLINGUPDATE_MAXUPDATES(\d)*\((\d)*\)//g ;
$ curFLOAT . = $ resource_string ;
$ cmd . = "FLOATING_RESOURCES=\"$curFLOAT\" " ;
}
$ resource_string =~ s/\((\d)*\)//g ;
foreach my $ sched ( split ( /\s+/ , $ resource_string ) ) {
if ( $ curSCHED !~ /$sched/ ) {
$ updateSCHED = 1 ;
last ;
}
}
if ( $ updateSCHED ) {
$ curSCHED =~ s/XCATROLLINGUPDATE_MUTEX(\d)*//g ;
$ curSCHED =~ s/XCATROLLINGUPDATE_MAXUPDATES(\d)*//g ;
$ curSCHED . = $ resource_string ;
$ cmd . = "SCHEDULE_BY_RESOURCES=\"$curSCHED\" " ;
}
if ( $ updateFLOAT || $ updateSCHED ) {
# TODO -- WAITING ON LLCONFIG OPTION TO NOT SEND CFG CMD TO ALL
#### NODES. NEED TO CHANGE CMD WHEN AVAILABLE.
my @ llcfg_c ;
if ( $ ::TEST ) {
my $ rsp ;
push @ { $ rsp - > { data } } , "In TEST mode. Will NOT run command: $cmd " ;
xCAT::MsgUtils - > message ( "I" , $ rsp , $ ::CALLBACK ) ;
$ ::RUNCMD_RC = 0 ;
} else {
@ llcfg_c = xCAT::Utils - > runcmd ( $ cmd , 0 ) ;
}
$ cmd = "llrctl reconfig" ;
my @ llms = split ( /\s+/ , $ llcms . " " . $ llrms ) ;
my % have = ( ) ;
my @ llnodes ;
foreach my $ m ( @ llms ) {
my ( $ sm , $ rest ) = split ( /\./ , $ m ) ;
push ( @ llnodes , $ sm ) unless $ have { $ sm } + + ;
}
if ( $ ::TEST ) {
my $ rsp ;
push @ { $ rsp - > { data } } , "In TEST mode. Will NOT run command: xdsh <llcm,llrm> $cmd " ;
xCAT::MsgUtils - > message ( "I" , $ rsp , $ ::CALLBACK ) ;
$ ::RUNCMD_RC = 0 ;
} else {
xCAT::Utils - > runxcmd (
{ command = > [ 'xdsh' ] ,
node = > \ @ llnodes ,
arg = > [ "-v" , $ cmd ]
} ,
$ ::SUBREQ , - 1 ) ;
}
}
}
$ ::LL_MUTEX_RESOURCES_CREATED = 1 ;
return 0 ;
}
#----------------------------------------------------------------------------
= head3 runrollupdate
Reboot updategroup in response to request from scheduler job
Arguments:
Returns:
0 - OK
1 - error
Globals:
Error:
Example:
Comments:
Note that since this command only gets called from the daemon
through a port request from a node , there is no active callback
to return messages to . Log only critical errors to the system log .
This subroutine will fork a child process to do the actual
work of shutting down and rebooting nodes . The parent will return
immediately to the caller so that other reboot requests can be
handled in parallel . It is the caller ' s responsibility to ensure
the parent process does not go away and take these children down
with it . ( Not a problem when called from xcatd , since that is
"long-running" . )
= cut
#-----------------------------------------------------------------------------
sub runrollupdate {
my $ reboot_request = shift ;
my @ reboot_args = @ { $ reboot_request - > { arg } } ;
my $ internal = 0 ;
if ( $ reboot_args [ 0 ] eq "internal" ) { $ internal = 1 ; }
my $ rc = 0 ;
my $ error = 0 ;
# process the command line
if ( $ internal ) {
$ ::scheduler = $ reboot_args [ 1 ] ;
$ ::datafile = $ reboot_args [ 2 ] ;
$ ::ll_reservation_id = "" ;
} else {
$ rc = & processArgs ;
if ( $ rc != 0 ) {
# rc: 0 - ok, 1 - return, 2 - help, 3 - error
if ( $ rc != 1 ) {
& runrollupdate_usage ;
}
return ( $ rc - 1 ) ;
}
}
$ ::scheduler =~ tr /[A-Z]/ [ a - z ] / ;
if ( $ ::VERBOSE ) {
unless ( - d $ ::LOGDIR ) { File::Path:: mkpath ( $ ::LOGDIR ) ; }
open ( RULOG , ">>$::LOGDIR/$::LOGFILE" ) ;
print RULOG localtime ( ) . "runrollupdate request for $::scheduler $::datafile $::ll_reservation_id \n" ;
close ( RULOG ) ;
}
my $ childpid = xCAT::Utils - > xfork ( ) ;
unless ( defined $ childpid ) { die "Fork failed" } ;
if ( $ childpid != 0 ) {
# This is the parent process, just return and let the child do all
# the work.
return $ childpid ;
}
# This is now the child process
# Load the datafile
& readDataFile ( $ ::datafile ) ;
$ ::ug_name = $ ::DATAATTRS { updategroup } [ 0 ] ;
my ( $ statusattr , $ statusval , $ statustimeout ) ;
if ( defined ( $ ::DATAATTRS { bringupappstatus } [ 0 ] ) ) {
$ statusattr = "appstatus" ;
$ statusval = $ ::DATAATTRS { bringupappstatus } [ 0 ] ;
} elsif ( defined ( $ ::DATAATTRS { bringupstatus } [ 0 ] ) ) {
$ statusattr = "status" ;
$ statusval = $ ::DATAATTRS { bringupstatus } [ 0 ] ;
} else {
$ statusattr = "status" ;
$ statusval = "booted" ;
}
if ( defined ( $ ::DATAATTRS { bringuptimeout } [ 0 ] ) ) {
$ statustimeout = $ ::DATAATTRS { bringuptimeout } [ 0 ] ;
} else {
$ statustimeout = 10 ;
}
# make sure nodes are in correct state
my $ hostlist = & get_hostlist ;
if ( ! $ hostlist ) {
if ( $ ::VERBOSE ) {
open ( RULOG , ">>$::LOGDIR/$::LOGFILE" ) ;
print RULOG localtime ( ) . "$::ug_name: Cannot determine nodelist for this request. \n" ;
close ( RULOG ) ;
}
my $ rsp ;
xCAT::MsgUtils - > message ( "S" ,
"rollupdate failure: $::ug_name: Cannot determine nodelist for this request. " ) ;
exit ( 1 ) ;
}
my $ nltab = xCAT::Table - > new ( 'nodelist' ) ;
my @ nodes = split ( /\,/ , $ hostlist ) ;
my $ appstatus = xCAT::Utils - > getAppStatus ( \ @ nodes , "RollingUpdate" ) ;
foreach my $ node ( @ nodes ) {
unless ( defined ( $ appstatus - > { $ node } )
&& ( $ appstatus - > { $ node } eq "update_job_submitted" ) )
{
if ( $ ::VERBOSE ) {
open ( RULOG , ">>$::LOGDIR/$::LOGFILE" ) ;
print RULOG localtime ( ) . " $::ug_name: Node $node appstatus not in valid state for rolling update\n" ;
print RULOG "The following nodelist will not be processed:\n $hostlist \n" ;
close ( RULOG ) ;
}
my $ rsp ;
xCAT::MsgUtils - > message (
"S" ,
"ROLLUPDATE failure: $::ug_name: Node $node appstatus not in valid state for rolling update "
) ;
exit ( 1 ) ;
}
}
# Run prescripts for this update group
xCAT::Utils - > setAppStatus ( \ @ nodes , "RollingUpdate" , "running_prescripts" ) ;
foreach my $ psline ( @ { $ ::DATAATTRS { 'prescript' } } ) {
$ psline =~ s/\$NODELIST/$hostlist/g ;
# Run the command
if ( $ ::VERBOSE ) {
open ( RULOG , ">>$::LOGDIR/$::LOGFILE" ) ;
print RULOG localtime ( ) . " $::ug_name: Running prescript \'$psline\'\n" ;
close ( RULOG ) ;
}
my @ psoutput ;
if ( $ ::TEST ) {
open ( RULOG , ">>$::LOGDIR/$::LOGFILE" ) ;
print RULOG localtime ( ) . " $::ug_name: In TEST mode. Will NOT run prescript \'$psline\'\n" ;
close ( RULOG ) ;
} else {
@ psoutput = xCAT::Utils - > runcmd ( $ psline , 0 ) ;
}
if ( $ ::VERBOSE ) {
open ( RULOG , ">>$::LOGDIR/$::LOGFILE" ) ;
print RULOG localtime ( ) . " Prescript output:\n" ;
foreach my $ psoline ( @ psoutput ) {
print RULOG $ psoline . "\n" ;
}
close ( RULOG ) ;
}
}
# Shutdown the nodes
# FUTURE: Replace if we ever develop cluster shutdown function
xCAT::Utils - > setAppStatus ( \ @ nodes , "RollingUpdate" , "shutting_down" ) ;
my $ shutdown_cmd = "shutdown -F &" ;
if ( $ ::VERBOSE ) {
open ( RULOG , ">>$::LOGDIR/$::LOGFILE" ) ;
print RULOG localtime ( ) . " $::ug_name: Running command \'xdsh $hostlist -v $shutdown_cmd\' \n" ;
close ( RULOG ) ;
}
if ( $ ::TEST ) {
open ( RULOG , ">>$::LOGDIR/$::LOGFILE" ) ;
print RULOG localtime ( ) . " $::ug_name: In TEST mode. Will NOT run command \'xdsh $hostlist -v $shutdown_cmd\' \n" ;
close ( RULOG ) ;
} else {
xCAT::Utils - > runxcmd ( { command = > [ 'xdsh' ] ,
node = > \ @ nodes ,
arg = > [ "-v" , $ shutdown_cmd ]
} , $ ::SUBREQ , - 1 ) ;
}
my $ slept = 0 ;
my $ alldown = 1 ;
my $ nodelist = join ( ',' , @ nodes ) ;
if ( ! $ ::TEST ) {
do {
$ alldown = 1 ;
my $ shutdownmax = 5 ;
if ( defined ( $ ::DATAATTRS { shutdowntimeout } [ 0 ] ) ) {
$ shutdownmax = $ ::DATAATTRS { shutdowntimeout } [ 0 ] ;
}
my $ pwrstat_cmd = "rpower $nodelist stat" ;
if ( $ ::VERBOSE ) {
open ( RULOG , ">>$::LOGDIR/$::LOGFILE" ) ;
print RULOG localtime ( ) . " $::ug_name: Running command \'$pwrstat_cmd\' \n" ;
close ( RULOG ) ;
}
my $ pwrstat = xCAT::Utils - > runxcmd ( $ pwrstat_cmd , $ ::SUBREQ , - 1 , 1 ) ;
foreach my $ pline ( @ { $ pwrstat } ) {
my ( $ pnode , $ pstat , $ rest ) = split ( /\s+/ , $ pline ) ;
if ( ( $ pstat eq "Running" )
|| ( $ pstat eq "Shutting" )
|| ( $ pstat eq "on" ) )
{
# give up on shutdown after requested wait time and force the
# node off
if ( $ slept >= ( $ shutdownmax * 60 ) ) {
$ pnode =~ s/://g ;
my $ pwroff_cmd = "rpower $pnode off" ;
if ( $ ::VERBOSE ) {
open ( RULOG , ">>$::LOGDIR/$::LOGFILE" ) ;
print RULOG localtime ( ) . " $::ug_name: Running command \'$pwroff_cmd\' \n" ;
close ( RULOG ) ;
}
xCAT::Utils - > runxcmd ( $ pwroff_cmd , $ ::SUBREQ , - 1 ) ;
}
else {
$ alldown = 0 ;
last ;
}
}
}
# If all nodes are not down yet, wait some more
unless ( $ alldown ) {
sleep ( 20 ) ;
$ slept += 20 ;
}
} until ( $ alldown ) ;
} # end not TEST
# Run out-of-band commands for this update group
xCAT::Utils - > setAppStatus ( \ @ nodes , "RollingUpdate" , "running_outofbandcmds" ) ;
foreach my $ obline ( @ { $ ::DATAATTRS { 'outofbandcmd' } } ) {
$ obline =~ s/\$NODELIST/$hostlist/g ;
# Run the command
if ( $ ::VERBOSE ) {
open ( RULOG , ">>$::LOGDIR/$::LOGFILE" ) ;
print RULOG localtime ( ) . " $::ug_name: Running out-of-band command \'$obline\'\n" ;
close ( RULOG ) ;
}
my @ oboutput ;
if ( $ ::TEST ) {
open ( RULOG , ">>$::LOGDIR/$::LOGFILE" ) ;
print RULOG localtime ( ) . " $::ug_name: In TEST mode. Will NOT run out-of-band command \'$obline\'\n" ;
close ( RULOG ) ;
} else {
@ oboutput = xCAT::Utils - > runcmd ( $ obline , 0 ) ;
}
if ( $ ::VERBOSE ) {
open ( RULOG , ">>$::LOGDIR/$::LOGFILE" ) ;
print RULOG localtime ( ) . " $::ug_name: Out-of-band command output:\n" ;
foreach my $ oboline ( @ oboutput ) {
print RULOG $ oboline . "\n" ;
}
close ( RULOG ) ;
}
}
# reboot the nodes
# Use bringuporder defined in datafile
my $ numboots = 0 ;
if ( defined ( $ ::DATAATTRS { bringuporder } [ 0 ] ) ) {
$ numboots = scalar ( @ { $ ::DATAATTRS { bringuporder } } ) ;
}
my @ remaining_nodes = @ nodes ;
foreach my $ bootindex ( 0 .. $ numboots ) {
my @ bootnodes ;
if ( defined ( $ ::DATAATTRS { bringuporder } [ $ bootindex ] ) ) {
@ bootnodes = split ( /,/ , $ ::DATAATTRS { bringuporder } [ $ bootindex ] ) ;
foreach my $ rn ( @ remaining_nodes ) {
if ( ( defined ( $ rn ) ) && ( grep ( /^$rn$/ , @ bootnodes ) ) ) {
undef ( $ rn ) ;
}
}
} else {
foreach my $ rn ( @ remaining_nodes ) {
if ( defined ( $ rn ) ) {
push ( @ bootnodes , $ rn ) ;
}
}
}
if ( ! scalar ( @ bootnodes ) ) { next ; }
# reboot command determined by nodehm power/mgt attributes
my $ hmtab = xCAT::Table - > new ( 'nodehm' ) ;
my @ rpower_nodes ;
my @ rnetboot_nodes ;
my $ hmtab_entries =
$ hmtab - > getNodesAttribs ( \ @ bootnodes , [ 'node' , 'mgt' , 'power' ] ) ;
foreach my $ node ( @ bootnodes ) {
my $ pwr = $ hmtab_entries - > { $ node } - > [ 0 ] - > { power } ;
unless ( defined ( $ pwr ) ) { $ pwr = $ hmtab_entries - > { $ node } - > [ 0 ] - > { mgt } ; }
if ( $ pwr eq 'hmc' ) {
push ( @ rnetboot_nodes , $ node ) ;
}
else {
push ( @ rpower_nodes , $ node ) ;
}
}
xCAT::Utils - > setAppStatus ( \ @ bootnodes , "RollingUpdate" , "rebooting" ) ;
if ( scalar ( @ rnetboot_nodes ) > 0 ) {
my $ rnb_nodelist = join ( ',' , @ rnetboot_nodes ) ;
# my $cmd = "rnetboot $rnb_nodelist -f";
#### TODO: DO WE STILL NEED 2 LISTS?
#### RUNNING rpower FOR ALL BOOTS NOW! MUCH FASTER!!!
my $ cmd = "rpower $rnb_nodelist on" ;
if ( $ ::VERBOSE ) {
open ( RULOG , ">>$::LOGDIR/$::LOGFILE" ) ;
print RULOG localtime ( ) . " $::ug_name: Running command \'$cmd\' \n" ;
close ( RULOG ) ;
}
if ( $ ::TEST ) {
open ( RULOG , ">>$::LOGDIR/$::LOGFILE" ) ;
print RULOG localtime ( ) . " $::ug_name: In TEST mode. Will NOT run command \'$cmd\' \n" ;
close ( RULOG ) ;
} else {
xCAT::Utils - > runxcmd ( $ cmd , $ ::SUBREQ , 0 ) ;
}
} elsif ( scalar ( @ rpower_nodes ) > 0 ) {
my $ rp_nodelist = join ( ',' , @ rpower_nodes ) ;
my $ cmd = "rpower $rp_nodelist boot" ;
if ( $ ::VERBOSE ) {
open ( RULOG , ">>$::LOGDIR/$::LOGFILE" ) ;
print RULOG localtime ( ) . " $::ug_name: Running command \'$cmd\' \n" ;
close ( RULOG ) ;
}
if ( $ ::TEST ) {
open ( RULOG , ">>$::LOGDIR/$::LOGFILE" ) ;
print RULOG localtime ( ) . " $::ug_name: In TEST mode. Will NOT run command \'$cmd\' \n" ;
close ( RULOG ) ;
} else {
xCAT::Utils - > runxcmd ( $ cmd , $ ::SUBREQ , 0 ) ;
}
}
# wait for bringupstatus to be set
my $ not_done = 1 ;
my $ totalwait = 0 ;
while ( $ not_done && $ totalwait < ( $ statustimeout * 60 ) ) {
if ( $ ::VERBOSE ) {
open ( RULOG , ">>$::LOGDIR/$::LOGFILE" ) ;
print RULOG localtime ( ) . " $::ug_name: Checking xCAT database $statusattr for value $statusval \n" ;
close ( RULOG ) ;
}
my $ nltab_stats =
$ nltab - > getNodesAttribs ( \ @ bootnodes , [ 'node' , $ statusattr ] ) ;
my % ll_res ;
$ not_done = 0 ;
foreach my $ bn ( @ bootnodes ) {
if ( $ nltab_stats - > { $ bn } - > [ 0 ] - > { $ statusattr } !~ /$statusval/ ) {
$ not_done = 1 ;
} else {
$ ll_res { $ bn } { remove } = 1 ;
}
}
if ( ( $ ::scheduler eq "loadleveler" ) &&
( $ ::ll_reservation_id ) ) {
my @ remove_res ;
foreach my $ bn ( keys % ll_res ) {
if ( $ ll_res { $ bn } { remove } && ! $ ll_res { $ bn } { removed } ) {
push ( @ remove_res , $ bn ) ;
$ ll_res { $ bn } { removed } = 1 ;
}
}
if ( @ remove_res ) {
& remove_LL_reservations ( \ @ remove_res ) ;
xCAT::Utils - > setAppStatus ( \ @ remove_res , "RollingUpdate" , "update_complete" ) ;
}
}
if ( $ not_done ) {
sleep ( 20 ) ;
$ totalwait += 20 ;
}
}
if ( $ not_done ) {
open ( RULOG , ">>$::LOGDIR/$::LOGFILE" ) ;
print RULOG "\n" ;
print RULOG localtime ( ) . " ERROR: Update group $::ug_name: Reached bringuptimeout before all nodes completed bringup. Some nodes may not have been updated. \n" ;
print RULOG "\n" ;
close ( RULOG ) ;
last ;
}
}
if ( $ ::VERBOSE ) {
open ( RULOG , ">>$::LOGDIR/$::LOGFILE" ) ;
print RULOG localtime ( ) . " $::ug_name: Rolling update complete.\n\n" ;
close ( RULOG ) ;
}
exit ( 0 ) ;
}
#----------------------------------------------------------------------------
= head3 readDataFile
Process the command line input piped in from a stanza file .
Arguments:
Returns:
0 - OK
1 - error
Globals:
Error:
Example:
Comments:
Set % ::DATAATTRS
( i . e . - $ ::DATAATTRS { attr } = [ val ] )
= cut
#-----------------------------------------------------------------------------
sub readDataFile {
my $ filedata = shift ;
my $ DF ;
unless ( open ( $ DF , "<" , $ filedata ) ) {
open ( RULOG , ">>$::LOGDIR/$::LOGFILE" ) ;
print RULOG localtime ( ) . " runrollupdate cannot open datafile $filedata\n" ;
close ( RULOG ) ;
return 1 ;
}
if ( $ ::VERBOSE ) {
open ( RULOG , ">>$::LOGDIR/$::LOGFILE" ) ;
print RULOG localtime ( ) . " runrollupdate reading datafile $filedata\n" ;
close ( RULOG ) ;
}
my @ lines = <$DF> ;
close $ DF ;
foreach my $ l ( @ lines ) {
# skip blank and comment lines
next if ( $ l =~ /^\s*$/ || $ l =~ /^\s*#/ ) ;
# process a real line
if ( $ l =~ /^\s*(\w+)\s*=\s*(.*)\s*/ ) {
my $ attr = $ 1 ;
my $ val = $ 2 ;
$ attr =~ s/^\s*// ; # Remove any leading whitespace
$ attr =~ s/\s*$// ; # Remove any trailing whitespace
$ attr =~ tr /A-Z/ a - z / ; # Convert to lowercase
$ val =~ s/^\s*// ;
$ val =~ s/\s*$// ;
# set the value in the hash for this entry
push ( @ { $ ::DATAATTRS { $ attr } } , $ val ) ;
}
} # end while - go to next line
return 0 ;
}
#----------------------------------------------------------------------------
= head3 get_hostlist
Returns a comma - delimited hostlist string for this updategroup
Arguments:
Returns:
0 - OK
1 - error
Globals:
Error:
Example:
Comments:
= cut
#-----------------------------------------------------------------------------
sub get_hostlist {
if ( defined ( $ ::DATAATTRS { nodelist } [ 0 ] ) ) {
return $ ::DATAATTRS { nodelist } [ 0 ] ;
}
my $ cmd ;
if ( $ ::VERBOSE ) {
$ cmd = "llqres -r -s -R $::ll_reservation_id 2>>$::LOGDIR/$::LOGFILE" ;
} else {
$ cmd = "llqres -r -s -R $::ll_reservation_id" ;
}
if ( $ ::VERBOSE ) {
open ( RULOG , ">>$::LOGDIR/$::LOGFILE" ) ;
print RULOG localtime ( ) . " $::ug_name: Running command \'$cmd\'\n" ;
close ( RULOG ) ;
}
my ( $ llstatus ) = xCAT::Utils - > runcmd ( $ cmd , 0 ) ;
if ( $ ::VERBOSE ) {
open ( RULOG , ">>$::LOGDIR/$::LOGFILE" ) ;
print RULOG localtime ( ) . " Return code: $::RUNCMD_RC\n" ;
print RULOG localtime ( ) . " $llstatus \n" ;
close ( RULOG ) ;
}
my @ status_fields = split ( /!/ , $ llstatus ) ;
if ( $ ::VERBOSE ) {
open ( RULOG , ">>$::LOGDIR/$::LOGFILE" ) ;
print RULOG localtime ( ) . " Hostlist: $status_fields[22] \n" ;
close ( RULOG ) ;
}
return $ status_fields [ 22 ] ;
}
#----------------------------------------------------------------------------
= head3 remove_LL_reservations
Changes the LL feature for nodes from oldfeature to newfeature
Remove nodes from LL reservation
Arguments:
Returns:
0 - OK
1 - error
Globals:
Error:
Example:
Comments:
= cut
#-----------------------------------------------------------------------------
sub remove_LL_reservations {
my $ nodes = shift ;
my $ cmd ;
if ( $ ::VERBOSE ) {
$ cmd = "llqres -r -s -R $::ll_reservation_id 2>>$::LOGDIR/$::LOGFILE" ;
} else {
$ cmd = "llqres -r -s -R $::ll_reservation_id" ;
}
if ( $ ::VERBOSE ) {
open ( RULOG , ">>$::LOGDIR/$::LOGFILE" ) ;
my $ nl = join ( "," , @ { $ nodes } ) ;
print RULOG localtime ( ) . " $::ug_name: remove_LL_reservations for $nl \n" ;
print RULOG localtime ( ) . " $::ug_name: Running command \'$cmd\'\n" ;
close ( RULOG ) ;
}
my ( $ llstatus ) = xCAT::Utils - > runcmd ( $ cmd , 0 ) ;
if ( $ ::VERBOSE ) {
open ( RULOG , ">>$::LOGDIR/$::LOGFILE" ) ;
print RULOG localtime ( ) . " Return code: $::RUNCMD_RC\n" ;
print RULOG localtime ( ) . " $llstatus \n" ;
close ( RULOG ) ;
}
# if ( $::RUNCMD_RC != 0 ) {
# return 0;
# }
my @ status_fields = split ( /!/ , $ llstatus ) ;
my $ llnodelist = $ status_fields [ 22 ] ;
my @ llnodes = split ( /,/ , $ llnodelist ) ;
my $ llnode_count = scalar ( @ llnodes ) ;
my $ remove_count = 0 ;
my $ remove_reservation = 0 ;
my $ remove_cmd = "llchres -R $::ll_reservation_id -h -" ;
foreach my $ n ( @ { $ nodes } ) {
if ( grep ( /^$n$/ , @ llnodes ) ) {
$ remove_count + + ;
# change features for this node
& change_LL_feature ( $ n ) ;
if ( $ remove_count < $ llnode_count ) {
$ remove_cmd . = " $n" ;
} else {
$ remove_reservation = 1 ;
last ;
}
}
}
2010-09-15 21:08:19 +00:00
###
### DEBUG -- llconfig problem
### need to send a manual reconfig to at least one node to have it
### register in LL daemons
### not working - try sleep instead
sleep 3 ;
# my $n1 = $nodes->[0];
# my @rcfg = `ssh $n1 llrctl reconfig`;
# if ($::VERBOSE) {
# open (RULOG, ">>$::LOGDIR/$::LOGFILE");
# print RULOG localtime()." llconfig workaround, run \"ssh $n1 llrctl reconfig \", output: \n";
# print RULOG @rcfg;
# close (RULOG);
# }
### end DEBUG
2010-09-14 22:29:41 +00:00
# Verify that the config change has been registerd and that updatefeature
# has been removed according to what the LL daemons report
if ( defined ( $ ::DATAATTRS { updatefeature } [ 0 ] ) ) {
my $ machinelist = join ( " " , @ { $ nodes } ) ;
my $ llset ;
my $ statrun = 0 ;
do {
sleep 1 ;
my $ llstatus = "llstatus -Lmachine -h $machinelist -l | grep -i feature | grep -i \" $::DATAATTRS{updatefeature}[0] \"" ;
if ( $ ::VERBOSE ) {
open ( RULOG , ">>$::LOGDIR/$::LOGFILE" ) ;
print RULOG localtime ( ) . " Running command \'$llstatus\'\n" ;
close ( RULOG ) ;
}
my @ stat_out = xCAT::Utils - > runcmd ( $ llstatus , 0 ) ;
if ( $ ::VERBOSE ) {
open ( RULOG , ">>$::LOGDIR/$::LOGFILE" ) ;
print RULOG localtime ( ) . " Return code: $::RUNCMD_RC\n" ;
close ( RULOG ) ;
}
# Are there any nodes with this feature still set?
$ llset = $ stat_out [ 0 ] ;
### DEBUG - print output to see what's going on
if ( $ llset ) {
open ( RULOG , ">>$::LOGDIR/$::LOGFILE" ) ;
print RULOG localtime ( ) . " llset: $llset\n" ;
close ( RULOG ) ;
}
### end DEBUG
$ statrun + + ;
} until ( ( ! $ llset ) || ( $ statrun > 10 ) ) ;
}
if ( $ remove_reservation ) {
$ cmd = "llrmres -R $::ll_reservation_id" ;
} elsif ( $ remove_count > 0 ) {
$ cmd = $ remove_cmd ;
} else {
return 0 ; # no LL nodes to remove
}
if ( $ ::VERBOSE ) {
open ( RULOG , ">>$::LOGDIR/$::LOGFILE" ) ;
print RULOG localtime ( ) . " $::ug_name: Running command \'$cmd\'\n" ;
close ( RULOG ) ;
}
# if ($::TEST) {
# open (RULOG, ">>$::LOGDIR/$::LOGFILE");
# print RULOG localtime()." $::ug_name: In TEST mode. Will NOT run command \'$cmd\' \n";
# close (RULOG);
# } else {
xCAT::Utils - > runcmd ( $ cmd , 0 ) ;
if ( $ ::RUNCMD_RC != 0 ) {
open ( RULOG , ">>$::LOGDIR/$::LOGFILE" ) ;
print RULOG localtime ( ) . " $::ug_name: Error running command \'$cmd\'\n" ;
print RULOG localtime ( ) . " Return code: $::RUNCMD_RC\n" ;
close ( RULOG ) ;
return 1 ;
}
# }
return 0 ;
}
#----------------------------------------------------------------------------
= head3 change_LL_feature
Changes the LL feature for the node from oldfeature to newfeature
and removes updatefeature
Arguments:
Returns:
0 - OK
1 - error
Globals:
Error:
Example:
Comments:
= cut
#-----------------------------------------------------------------------------
sub change_LL_feature {
my $ node = shift ;
if ( ! defined ( $ ::DATAATTRS { updatefeature } [ 0 ] ) &&
! defined ( $ ::DATAATTRS { oldfeature } [ 0 ] ) &&
! defined ( $ ::DATAATTRS { newfeature } [ 0 ] ) ) {
return 0 ;
}
# Query current feature
my $ cmd = "llconfig -h $node -d FEATURE" ;
if ( $ ::VERBOSE ) {
open ( RULOG , ">>$::LOGDIR/$::LOGFILE" ) ;
print RULOG localtime ( ) . " $::ug_name: Running command \'$cmd\'\n" ;
close ( RULOG ) ;
}
my ( $ llcfgout ) = xCAT::Utils - > runcmd ( $ cmd , 0 ) ;
if ( $ ::VERBOSE ) {
open ( RULOG , ">>$::LOGDIR/$::LOGFILE" ) ;
print RULOG localtime ( ) . " Return code: $::RUNCMD_RC\n" ;
close ( RULOG ) ;
}
# Remove old feature
my $ newfeature_string = "" ;
my @ llfeatures ;
2010-09-15 21:08:19 +00:00
if ( $ llcfgout =~ /:FEATURE =/ ) {
2010-09-14 22:29:41 +00:00
my ( $ stuff , $ curfeature_string ) = split ( /=/ , $ llcfgout ) ;
@ llfeatures = split ( /\s+/ , $ curfeature_string ) ;
my $ oldfeature = " " ;
my $ updateallfeature = " " ;
if ( defined ( $ ::DATAATTRS { oldfeature } [ 0 ] ) ) {
$ oldfeature = $ ::DATAATTRS { oldfeature } [ 0 ] ;
}
if ( defined ( $ ::DATAATTRS { updatefeature } [ 0 ] ) ) {
$ updateallfeature = $ ::DATAATTRS { updatefeature } [ 0 ] ;
}
foreach my $ f ( @ llfeatures ) {
if ( ( $ f eq $ oldfeature ) || ( $ f eq $ updateallfeature ) ) {
$ f = " " ;
}
}
$ newfeature_string = join ( " " , @ llfeatures ) ;
}
# Add new feature
if ( defined ( $ ::DATAATTRS { newfeature } [ 0 ] ) ) {
my $ haveit = 0 ;
foreach my $ f ( @ llfeatures ) {
if ( $ f eq $ ::DATAATTRS { newfeature } [ 0 ] ) {
$ haveit = 1 ;
}
}
if ( ! $ haveit ) {
$ newfeature_string . = " $::DATAATTRS{newfeature}[0]" ;
}
}
# Change in LL database
$ cmd = "llconfig -h $node -c FEATURE=\"$newfeature_string\"" ;
if ( $ ::VERBOSE ) {
open ( RULOG , ">>$::LOGDIR/$::LOGFILE" ) ;
print RULOG localtime ( ) . " $::ug_name: Running command \'$cmd\'\n" ;
close ( RULOG ) ;
}
xCAT::Utils - > runcmd ( $ cmd , 0 ) ;
if ( $ ::VERBOSE ) {
open ( RULOG , ">>$::LOGDIR/$::LOGFILE" ) ;
print RULOG localtime ( ) . " Return code: $::RUNCMD_RC\n" ;
close ( RULOG ) ;
}
return 0 ;
}
2008-10-22 16:02:14 +00:00
1 ;
