mirror of
https://github.com/xcat2/xcat-core.git
synced 2025-05-30 09:36:41 +00:00
1f1ef56e41
The IVP expects xcatd to be fully operational by the time is begins a run. It is possible for xcatd to be delayed. The delay should not ause a corruption of the zvmnotify property in the site table. In addition, tests for tabdump failures should be corrected and the log file should be set up sooner so that debug information is available for similar problems. Change-Id: I2b0e3e875cc28f740a35b348faa0cc74f5acb7bf
3025 lines
110 KiB
Perl
3025 lines
110 KiB
Perl
#!/usr/bin/perl
|
|
###############################################################################
|
|
# IBM (C) Copyright 2016 Eclipse Public License
|
|
# http://www.eclipse.org/org/documents/epl-v10.html
|
|
###############################################################################
|
|
# COMPONENT: verifynode
|
|
#
|
|
# Verify the capabilities of a node.
|
|
###############################################################################
|
|
|
|
package xCAT_plugin::verifynode;
|
|
BEGIN
|
|
{
|
|
$::XCATROOT = $ENV{'XCATROOT'} ? $ENV{'XCATROOT'} : '/opt/xcat';
|
|
}
|
|
|
|
$XML::Simple::PREFERRED_PARSER='XML::Parser';
|
|
|
|
use strict;
|
|
use warnings;
|
|
|
|
use Data::Dumper qw(Dumper);
|
|
use File::Basename;
|
|
use File::Path;
|
|
use File::Spec;
|
|
use File::Temp;
|
|
use Getopt::Long;
|
|
use Getopt::Long qw(GetOptionsFromString);
|
|
use lib "$::XCATROOT/lib/perl";
|
|
use MIME::Base64;
|
|
use POSIX qw/strftime/;
|
|
use Scalar::Util qw( looks_like_number );
|
|
use Sys::Hostname;
|
|
use Sys::Syslog qw( :DEFAULT setlogsock );
|
|
use Socket;
|
|
use Text::Wrap;
|
|
use Time::HiRes qw( gettimeofday sleep );
|
|
use Unicode::UCD qw(charinfo);
|
|
|
|
use xCAT::zvmCPUtils;
|
|
use xCAT::zvmMsgs;
|
|
use xCAT::zvmUtils;
|
|
|
|
my $version = "1";
|
|
|
|
my $comments = ''; # Comments for the IVP run
|
|
my $dataFile = ''; # Input data file
|
|
my $decode = ''; # String specified by the --decode operand.
|
|
my $disable = ''; # Disable scheduled IVP from running.
|
|
my $displayHelp = 0; # Display help information
|
|
my $driver = ''; # Name of driver script
|
|
my $driverLoc = '/var/opt/xcat/ivp'; # Local directory containing the driver script, if it exists
|
|
my $encode = ''; # String specified by the --encode operand.
|
|
my $enable = ''; # Enable scheduled IVP to run.
|
|
my $endTime = ''; # Human readable end time
|
|
my %hosts; # Information on host nodes known by xCAT MN
|
|
my $id = ''; # Id of the IVP run
|
|
my %ignored; # List of ignored messages
|
|
my $ignoreCnt = 0; # Number of times we ignored a message
|
|
my $issueCmdOverIUCV = ''; # Issue a command to a node over IUCV
|
|
my $issueCmdToNode = ''; # Issue a command to a node using IUCV or SSH
|
|
my %localIPs; # Hash of the Local IP addresses for the xCAT MN system
|
|
my $logDir = '/var/log/xcat/ivp'; # Log location
|
|
my $logFile = ''; # Log file name and location
|
|
my $logFileHandle; # File handle for the log file.
|
|
my @ivpSummary; # Summary output lines for an IVP run.
|
|
my $locApplSystemRole = '/var/lib/sspmod/appliance_system_role';
|
|
my %mnInfo; # Managed node environment information
|
|
my %msgsToIgnore; # Hash of messages to ignore
|
|
my $needToFinishLogFile = 0; # If 1 then finishLogFile() needs to be called
|
|
my $nodeName = ''; # Name of node to be verified
|
|
my $notify = 0; # Notify a user
|
|
my $notifyOnErrorOrWarning = 0; # Indicates a log file should be sent if the need to notify
|
|
# a user was determined.
|
|
my $openstackIP = ''; # IP address of the OpenStack node
|
|
my $openstackUser = 'root'; # User on the OpenStack system that can access the
|
|
# OpenStack configuration files
|
|
my $orchParms = ''; # Orchestrator parms to be stored in the zvmivp table
|
|
my $prepParms = ''; # Parameters to pass to the preparation script
|
|
my $remove = 0; # Remove a scheduled IVP
|
|
my $runBasicIVP = 0; # Run the basic xCAT IVP
|
|
my $runCron = 0; # Cron job invocation to run a possible set of IVPs
|
|
# on a periodic basis.
|
|
my $runFullIVP = 0; # Run the full end to end IVP (compute node and xCAT)
|
|
my $schedule = ''; # Hours to schedule an automated IVP
|
|
my $scheduledType = ''; # Type of IVP to schedule ('fullivp' or 'basicivp')
|
|
my $startTime = ''; # Human readable start time
|
|
my $tgtOS = ''; # Target Node's distro & version
|
|
my $todayDate = ''; # Today's date in the form 'yyyy-mm-dd'
|
|
my $verifyAccess = 0; # Verify system can be accessed
|
|
my $verifyCaptureReqs = 0; # Verify systems meets capture requirements
|
|
my $verifyCloudInit = 0; # Verify cloud-init installed
|
|
my $verifyXcatconf4z = 0; # Verify xcatconf4z
|
|
my $versionOpt = 0; # Show version information flag
|
|
my $warnErrCnt = 0; # Count of warnings and error messages for a verify set
|
|
my $zhcpTarget = ''; # ZHCP server that is related to this verification
|
|
my $zvmHost = ''; # z/VM host related to this verification
|
|
my $zxcatParms = ''; # Parmeters to pass to zxcatIVP
|
|
|
|
# Routines to drive based on the selected test.
|
|
my %verifySets = (
|
|
'$decode ne \'\'' =>
|
|
[ '', # Do not show a function banner.
|
|
'encodeDecodeString()',
|
|
],
|
|
'$disable and ! $remove and $schedule eq \'\'' =>
|
|
[ 'Disabling an IVP run',
|
|
'enableDisableRun( $id )',
|
|
],
|
|
'$enable and ! $remove and $schedule eq \'\'' =>
|
|
[ 'Enabling an IVP run',
|
|
'enableDisableRun( $id )',
|
|
],
|
|
'$encode ne \'\'' =>
|
|
[ 'Encoding a string',
|
|
'encodeDecodeString()',
|
|
],
|
|
'$issueCmdOverIUCV ne \'\'' =>
|
|
[ 'Issue a command to a node using IUCV',
|
|
'cmdOnVM( \'onlyIUCV\' )',
|
|
],
|
|
'$issueCmdToNode ne \'\'' =>
|
|
[ 'Issue a command to a node using IUCV or SSH',
|
|
'cmdOnVM( \'either\' )',
|
|
],
|
|
'$remove' =>
|
|
[ 'Remove an automated periodic IVP',
|
|
'removeIVP( $id )',
|
|
],
|
|
'$runBasicIVP' =>
|
|
[ 'Verifying the basic general setup of xCAT',
|
|
'addLogHeader( \'basicivp\' )',
|
|
'verifyBasicXCAT()',
|
|
],
|
|
'$runCron' =>
|
|
[ 'Cron job for automated verification',
|
|
'driveCronList()',
|
|
],
|
|
'$runFullIVP' =>
|
|
[ 'Verifying the full end-to-end setup of the xCAT and the compute node',
|
|
'addLogHeader( \'fullivp\' )',
|
|
'checkForDefault( \'openstackIP\', \'--openstackIP was not specified\', \'xCAT_master\' )',
|
|
'verifyAccess( $openstackIP ); $rc = 0',
|
|
'getDistro( $openstackIP )',
|
|
'runPrepScript()',
|
|
'runDriverScript()',
|
|
],
|
|
'$schedule ne \'\'' =>
|
|
[ 'Scheduling an automated periodic IVP',
|
|
'scheduleIVP()',
|
|
],
|
|
'$verifyAccess' =>
|
|
[ 'Verifying access to the system',
|
|
'verifyPower( $nodeName )',
|
|
'verifyAccess( $nodeName )'
|
|
],
|
|
'$verifyCaptureReqs' =>
|
|
[ 'Verifying system meets the image requirements',
|
|
'verifyPower( $nodeName )',
|
|
'verifyAccess( $nodeName )',
|
|
'getDistro( $nodeName )',
|
|
'verifyDistro()',
|
|
],
|
|
'$verifyCloudInit' =>
|
|
[ 'Verifying cloud-init is installed and configured',
|
|
'verifyPower( $nodeName )',
|
|
'verifyAccess( $nodeName )',
|
|
'getDistro( $nodeName )',
|
|
'verifyService($nodeName, "cloud-config,2,3,4,5 cloud-final,2,3,4,5 cloud-init,2,3,4,5 cloud-init-local,2,3,4,5")',
|
|
'showCILevel()',
|
|
],
|
|
'$verifyXcatconf4z' =>
|
|
[ 'Verifying xcatconf4z is installed and configured',
|
|
'verifyPower( $nodeName )',
|
|
'verifyAccess( $nodeName )',
|
|
'getDistro( $nodeName )',
|
|
'verifyXcatconf4z( $nodeName )',
|
|
],
|
|
'$test' =>
|
|
[ 'Script test functon.',
|
|
'verifyLogResponse()',
|
|
],
|
|
);
|
|
|
|
|
|
# set the usage message
|
|
my $usage_string = "Usage:\n
|
|
$0 -n <node> <options>\n
|
|
or\n
|
|
$0 <options>\n
|
|
or\n
|
|
$0 -v\n
|
|
or\n
|
|
$0 [ -h | --help ]\n
|
|
The following options are supported:\n
|
|
-a | --access
|
|
Verify access from xCAT MN to node.
|
|
--basicivp
|
|
Run the basic xCAT IVP.
|
|
--capturereqs
|
|
Verify that node meets system capture
|
|
requirements for OpenStack.
|
|
--cmdoveriucv <cmd>
|
|
Issue a command to a node using IUCV.
|
|
Specify a period for <cmd> when combining this
|
|
option with the --file option.
|
|
--cmdtonode <cmd> Issue a command to a node using IUCV or SSH
|
|
if IUCV is not available.
|
|
Specify a period for <cmd> when combining this
|
|
option with the --file option.
|
|
-c | --cloudinit
|
|
Verify that node has cloud-init installed.
|
|
--comments
|
|
Comments to specify on the IVP run.
|
|
--cron
|
|
Driven as a cron job to run a set of periodic
|
|
IVPs.
|
|
--decode <parm>
|
|
Decode an encoded string. This is used to
|
|
view encoded values in the zvmivp table.
|
|
--disable Indicates that the IVP being scheduled should
|
|
be disabled from running.
|
|
--enable Indicates that the IVP being scheduled should
|
|
be enabled to run. This is the default when
|
|
neither --disable or --enable is specified.
|
|
--encode <parm>
|
|
Encode a string. This used for debug in order
|
|
to create a string to compare to an encoded
|
|
string from the zvmivp table. The <parm> must be
|
|
an ASCII string. This function does not support
|
|
encoding a UTF-8 character string.
|
|
--file File location containing additional input.
|
|
When used with the --cmdoveriucv or --cmdtonode
|
|
operands, this file contains the command to issue.
|
|
--fullivp
|
|
Run the full end-to-end IVP. This
|
|
verification checks the compute node and builds
|
|
a driver script which is then used to check
|
|
the xCAT environment based on the values
|
|
gathered in the driver script.
|
|
-h | --help
|
|
Display help information.
|
|
-i | --id <idToRun>
|
|
ID to associate with the IVP run.
|
|
--ignore <val>
|
|
Blank or comma separated list of message ids or
|
|
message severities to ignore. Ignored messages are
|
|
not counted as failures and do not produce messages.
|
|
Instead the number of ignored messages and their
|
|
message numbers are displayed at the end of
|
|
processing. Recognized message severities:
|
|
'bypass', 'info', 'warning', 'error'.
|
|
--notify
|
|
Notification of IVP failures should be sent
|
|
to a z/VM userid.
|
|
-n | --node <node>
|
|
Node name of system to verify.
|
|
--openstackip <ipAddr>
|
|
IP address of the OpenStack compute node.
|
|
--openstackuser <user>
|
|
User to use to access the compute node.
|
|
The user defaults to 'root'.
|
|
--orchparms <parms>
|
|
Parameters to be stored in the zvmivp table. These
|
|
parameters are passed to this routine when a
|
|
scheduled IVP is driven.
|
|
--prepparms <parms>
|
|
Parameters to pass to the preparation script as
|
|
part of a full IVP.
|
|
--remove
|
|
Indicates that an IVP in the zvmivp table should be
|
|
removed. The id is specified with the --id
|
|
operand.
|
|
--schedule <hours>
|
|
List of hours (in 24 hour format) to periodically
|
|
run the IVP. Multiple hours are separated by blanks.
|
|
For example, \'0 6 12 18 20\'.
|
|
--type < basicivp | fullivp >
|
|
Type of IVP to be scheduled.
|
|
-v
|
|
Display the version of this script.
|
|
-x | --xcatconf4z
|
|
Verify that node has xcatconf4z installed.
|
|
--zxcatparms <parms>
|
|
Parameters to pass to the zxcatIVP script.\n";
|
|
|
|
|
|
#-------------------------------------------------------
|
|
|
|
=head3 addLogHeader
|
|
|
|
Description : Add a header to the log file.
|
|
Arguments : None
|
|
Returns : 0 - No error
|
|
non-zero - Terminating error detected.
|
|
Example : $rc = addLogHeader();
|
|
|
|
=cut
|
|
|
|
#-------------------------------------------------------
|
|
sub addLogHeader {
|
|
my ( $hType ) = @_;
|
|
|
|
logResponse( 'DF', '*NONFORMATTED*', '*******************************************************************************' );
|
|
|
|
# Generate the header based on the header type.
|
|
if ( $hType eq 'basicivp' ) {
|
|
logResponse( 'DF', 'GENERIC_RESPONSE_NOINDENT', "This log file was created for a basic xCAT IVP. It consists of output from ".
|
|
"zxcatIVP that validates the xCAT management node (MN) and the ZHCP Agents that are used by ".
|
|
"the xCAT MN node to manage the z/VM hosts." );
|
|
} elsif ( $hType eq 'fullivp' ) {
|
|
logResponse( 'DF', 'GENERIC_RESPONSE_NOINDENT', "This log file was created for a full xCAT IVP which validates the z/VM OpenStack ".
|
|
"configuration files, the xCAT management node (MN) and the ZHCP Agents that are used by ".
|
|
"the xCAT MN node to manage the z/VM host. The log file contains the following output: output ".
|
|
"from the preparation script that validated the OpenStack properties, the contents of the ".
|
|
"driver script created by the preparation script to drive the second part of the IVP, output ".
|
|
"of the zxcatIVP run that is driven by the driver script to complete the full IVP." );
|
|
} else {
|
|
logResponse( 'DF', 'GENERIC_RESPONSE', "VerifyNode log file." );
|
|
}
|
|
|
|
logResponse( 'DF', '*NONFORMATTED*', '*******************************************************************************' );
|
|
|
|
return 0;
|
|
}
|
|
|
|
|
|
#-------------------------------------------------------
|
|
|
|
=head3 checkForDefault
|
|
|
|
Description : Determine if a default is needed for
|
|
the specified property and set it if
|
|
it is needed. Currently, this routine
|
|
is only used for the master IP address.
|
|
Arguments : Name of property
|
|
String to use in a DFLT01 message.
|
|
Default or function to perform.
|
|
Returns : 0 - No error
|
|
non-zero - Terminating error detected.
|
|
Example : $rc = checkForDefault( 'openStackIP', '--ip was not specified', 'xCAT_master' );
|
|
|
|
=cut
|
|
|
|
#-------------------------------------------------------
|
|
sub checkForDefault {
|
|
my ( $prop, $reasonString, $default ) = @_;
|
|
my $additionalInfo = '';
|
|
my $rc = 0;
|
|
my $val;
|
|
|
|
eval( "\$val = \$$prop" );
|
|
if ( $val eq '' ) {
|
|
if ( $default eq 'xCAT_master' ) {
|
|
$default = '';
|
|
if ( exists $mnInfo{'ip'} ) {
|
|
$default = $mnInfo{'ip'};
|
|
$additionalInfo = 'The value came from the IP address used as the xCAT MN\'s ' .
|
|
'master IP address that was specified in the site table.';
|
|
} else {
|
|
# Use one of the local IPv4 IPs. We don't support IPv6.
|
|
if ( keys %localIPs > 0 ) {
|
|
# Can't find the MN IP so use the first local IPv4 IP.
|
|
foreach my $ip ( keys %localIPs ) {
|
|
if ( $ip =~ /:/ ) {
|
|
next;
|
|
} else {
|
|
$default = $ip;
|
|
$additionalInfo = 'The function could not determine the master IP address for ' .
|
|
'the xCAT MN so the first local IPv4 address was used.';
|
|
last;
|
|
}
|
|
}
|
|
if ( $default eq '' ) {
|
|
$default = '127.0.0.1';
|
|
$additionalInfo = 'The function could not determine the IP address for the xCAT MN ' .
|
|
'and could not find any IPv4 addresses on the system so \'127.0.0.1\' will be used ' .
|
|
'in the hope that it will work.';
|
|
}
|
|
} else {
|
|
# Don't know the local IPs.
|
|
# Default to the IPv4 loopback and let it fail.
|
|
$default = '127.0.0.1';
|
|
$additionalInfo = 'The function cound not determine the IP addresses on the ' .
|
|
'xCAT MN system so \'127.0.0.1\' was used.';
|
|
}
|
|
}
|
|
}
|
|
|
|
if ( $rc == 0 ) {
|
|
# Found a default, set it in the property.
|
|
eval( "\$$prop = \'$default\'" );
|
|
logResponse( 'DF', 'DFLT01', $reasonString, $default, $additionalInfo );
|
|
}
|
|
}
|
|
|
|
return $rc;
|
|
}
|
|
|
|
|
|
#-------------------------------------------------------
|
|
|
|
=head3 cmdOnVM
|
|
|
|
Description : Send a command to a node and show the result.
|
|
Arguments : Target function, either 'onlyIUCV' or 'either'.
|
|
Returns : Return code:
|
|
0 - Normal Linux success
|
|
255 - Unable to SSH to system
|
|
non-zero - command error
|
|
Output from the command or a error string on an SSH failure.
|
|
Example : $rc = cmdOnVM( 'onlyIUCV' );
|
|
$rc = cmdOnVM( 'either' );
|
|
|
|
=cut
|
|
|
|
#-------------------------------------------------------
|
|
sub cmdOnVM {
|
|
my ( $targetFunc ) = @_;
|
|
my $cmd = '';
|
|
my $hcp = '';
|
|
my @lines;
|
|
my $rc = 0;
|
|
my $out = '';
|
|
my $sudo;
|
|
my $user;
|
|
my $userid = '';
|
|
|
|
# Get the command to issue.
|
|
if ( $targetFunc eq 'onlyIUCV' ) {
|
|
if ( $issueCmdOverIUCV ne '.' ) {
|
|
$cmd = $issueCmdOverIUCV;
|
|
}
|
|
} else {
|
|
if ( $issueCmdToNode ne '.' ) {
|
|
$cmd = $issueCmdToNode;
|
|
}
|
|
}
|
|
|
|
# Use the data file if it was specified.
|
|
if ( $dataFile ne '' ) {
|
|
if ( -e $dataFile ) {
|
|
open my $handle, '<', $dataFile;
|
|
chomp( @lines = <$handle> );
|
|
close $handle;
|
|
if ( $lines[0] ne '' ) {
|
|
$cmd = $lines[0];
|
|
} else {
|
|
logResponse( 'DF', 'GENERIC_RESPONSE', "$dataFile does not have anything in the first line.");
|
|
goto FINISH_cmdUsingIUCV;
|
|
}
|
|
} else {
|
|
logResponse( 'DF', 'GENERIC_RESPONSE', "$dataFile does not exist.");
|
|
goto FINISH_cmdUsingIUCV;
|
|
}
|
|
}
|
|
|
|
# Verify that we have a command to issue.
|
|
if ( $cmd eq '' ) {
|
|
logResponse( 'DF', 'GENERIC_RESPONSE', "A command to issue was not specified.");
|
|
goto FINISH_cmdUsingIUCV;
|
|
}
|
|
|
|
($user, $sudo) = xCAT::zvmUtils->getSudoer();
|
|
|
|
# Get the required properties (hcp, userid, user) from the node
|
|
$out = `/opt/xcat/bin/lsdef $nodeName -i hcp,userid`;
|
|
$rc = $?;
|
|
if ( $rc eq 0 ) {
|
|
my $fndNode = 0;
|
|
@lines = split( '\n', $out );
|
|
my $host = '';
|
|
foreach my $line ( @lines ) {
|
|
$line =~ s/^\s+|\s+$//g; # trim blanks from both ends of the string
|
|
if ( $line =~ /Object name:/ ) {
|
|
$fndNode = 1;
|
|
next;
|
|
} elsif ( $line =~ /hcp\=/ ) {
|
|
($hcp) = $line =~ m/hcp\=(.*)/;
|
|
next;
|
|
} elsif ( $line =~ /userid\=/ ) {
|
|
($userid) = $line =~ m/userid\=(.*)/;
|
|
next;
|
|
}
|
|
}
|
|
if ( $fndNode != 1 ) {
|
|
logResponse( 'DF', 'GENERIC_RESPONSE', "'$nodeName' is not an xCAT node." );
|
|
goto FINISH_cmdUsingIUCV;
|
|
}
|
|
} else {
|
|
$rc = logResponse( 'DFS', 'GNRL01', "$cmd", $rc, $out );
|
|
goto FINISH_cmdUsingIUCV;
|
|
}
|
|
|
|
# Send the command through IUCV
|
|
logResponse( 'DF', '*NONFORMATTED*', "\n**************************\nData for the function call\n**************************" );
|
|
logResponse( 'DF', '*NONFORMATTED*', " Node: $nodeName" );
|
|
if ( $targetFunc eq 'onlyIUCV' ) {
|
|
logResponse( 'DF', '*NONFORMATTED*', "z/VM userid: $userid" );
|
|
logResponse( 'DF', '*NONFORMATTED*', " HCP: $hcp" );
|
|
logResponse( 'DF', '*NONFORMATTED*', " User: $user" );
|
|
logResponse( 'DF', '*NONFORMATTED*', " Function: xCAT::zvmUtils->execcmdthroughIUCV" );
|
|
} else {
|
|
logResponse( 'DF', '*NONFORMATTED*', " User: $user" );
|
|
logResponse( 'DF', '*NONFORMATTED*', " Function: xCAT::zvmUtils->execcmdonVM" );
|
|
}
|
|
logResponse( 'DF', '*NONFORMATTED*', " Cmd: $cmd" );
|
|
|
|
logResponse( 'DF', '*NONFORMATTED*', "\n*********************\nInvoking the function\n*********************" );
|
|
if ( $targetFunc eq 'onlyIUCV' ) {
|
|
$out = xCAT::zvmUtils->execcmdthroughIUCV( $user, $hcp, $userid, $cmd );
|
|
} else {
|
|
$out = xCAT::zvmUtils->execcmdonVM( $user, $nodeName, $cmd );
|
|
}
|
|
logResponse( 'DF', '*NONFORMATTED*', "\n******\nResult\n******\n$out" );
|
|
|
|
FINISH_cmdUsingIUCV:
|
|
return $rc;
|
|
}
|
|
|
|
|
|
#-------------------------------------------------------
|
|
|
|
=head3 driveCronList
|
|
|
|
Description : Run thru the list of IVPs and run any that
|
|
are due to be run.
|
|
Arguments : None.
|
|
Returns : 0 - No error
|
|
non-zero - Terminating error detected.
|
|
Example : $rc = driveCronList();
|
|
|
|
=cut
|
|
|
|
#-------------------------------------------------------
|
|
sub driveCronList {
|
|
my $attemptedRuns = 0;
|
|
my $disabledRuns = 0;
|
|
my @ivps;
|
|
my $out = '';
|
|
my $rc = 0;
|
|
my $totalRunsInTable = 0;
|
|
|
|
# Determine the timestamp and other time related information.
|
|
my ($sec, $min, $hour, $mday, $mon, $year, $wday, $yday, $isdst) = localtime(time);
|
|
$year = $year + 1900;
|
|
$todayDate = sprintf( "%04d-%02d-%02d", $year, $mon + 1, $mday );
|
|
$startTime = sprintf( "%02d:%02d:%02d on %04d-%02d-%02d",
|
|
$hour, $min, $sec, $year, $mon + 1, $mday );
|
|
my $currTime = sprintf( "%04d-%02d-%02d_%02d:%02d:%02d",
|
|
$year, $mon + 1, $mday, $hour, $min, $sec);
|
|
my $newLastRun = "$year $yday $hour";
|
|
|
|
logResponse( 'DFS', '*NONFORMATTED*', "CRON job run started at $startTime." );
|
|
|
|
# Read the z/VM IVP table to get the list of known IVP runs.
|
|
$out = `/opt/xcat/sbin/tabdump zvmivp | grep -v "^#"`;
|
|
$rc = $?;
|
|
my @lines = split( '\n', $out );
|
|
$totalRunsInTable = scalar( @lines );
|
|
if ( $totalRunsInTable != 0 ) {
|
|
my $row = 0;
|
|
my $nowTotalHours = ( ( $year ) * 8784 ) + ( $yday * 24 ) + $hour;
|
|
foreach my $line ( @lines ) {
|
|
$row++;
|
|
chomp( $line );
|
|
my @field = split( ',', $line );
|
|
for ( my $i=0; $i < scalar( @field ); $i++ ) {
|
|
$field[$i] =~ s/^\"|\"$//g; # trim quotes from both ends of the string
|
|
}
|
|
|
|
# Process the fields from the zvmIVP table. Each row is related to a
|
|
# scheduled run.
|
|
# Field 0 - Unique IVP ID
|
|
# 1 - IP property for the OpenStack system
|
|
# 2 - Schedule, hours to run the IVP
|
|
# 3 - Last time IVP was run, ( year day hour )
|
|
# 4 - Type of run, e.g. basicivp, fullivp
|
|
# 5 - OpenStack user to access the config files
|
|
# 6 - Parameters to pass to the orchestrator, verifynode
|
|
# 7 - Parameters to pass to the preparator script, prep_zxcatIVP
|
|
# 8 - Parameters to pass to the main IVP script, zxcatIVP.pl
|
|
# 9 - Comments related to the IVP run
|
|
# 10 - Disabled, 1 or YES equals disabled
|
|
my %ivpRun;
|
|
if ( defined $field[0] and $field[0] ne '' ) {
|
|
$ivpRun{'id'} = $field[0];
|
|
} else {
|
|
logResponse( 'DS', 'GENERIC_RESPONSE', "Row $row in the ivp table missing a ".
|
|
"required property, \'id\'. The row will be treated as disabled." );
|
|
$disabledRuns++;
|
|
next;
|
|
}
|
|
|
|
if ( defined $field[10] and ($field[10] eq '1' or uc( $field[10] ) eq 'YES') ) {
|
|
# IVP run is disabled
|
|
logResponse( 'DS', 'GENERIC_RESPONSE', "Row $row, ID($ivpRun{'id'}), in the ivp table is ".
|
|
"disabled and will not be processed." );
|
|
$disabledRuns++;
|
|
next;
|
|
}
|
|
|
|
if ( defined $field[1] and $field[1] ne '' ) {
|
|
$ivpRun{'ip'} = $field[1];
|
|
} else {
|
|
# Assume this is a local xCAT IVP run
|
|
$ivpRun{'ip'} = 'xcat';
|
|
logResponse( 'DS', 'GENERIC_RESPONSE', "Row $row, ID($ivpRun{'id'}), in the ivp table " .
|
|
"has no value specified for the \'ip\' property. " .
|
|
"The IVP run will be against the system which is " .
|
|
"running the xCAT management node." );
|
|
}
|
|
|
|
if ( defined $field[2] and $field[2] ne '' ) {
|
|
$ivpRun{'schedule'} = hexDecode( $field[2] );
|
|
} else {
|
|
# Default run time is 1 AM
|
|
$ivpRun{'schedule'} = '1';
|
|
}
|
|
|
|
my $lRYear;
|
|
my $lRYDay;
|
|
my $lRHour;
|
|
|
|
if ( defined $field[3] and $field[3] ne '' ) {
|
|
$ivpRun{'last_run'} = $field[3];
|
|
my @parts = split ( '\s', $ivpRun{'last_run'} );
|
|
if ( defined $parts[0] and $parts[0] ne '' ) {
|
|
$lRYear = $parts[0];
|
|
} else {
|
|
# Missing year, set default to immediately, 0.
|
|
$lRYear = 0;
|
|
}
|
|
if ( defined $parts[1] and $parts[1] ne '' ) {
|
|
$lRYDay = $parts[1];
|
|
} else {
|
|
# Missing day number, set default to immediately, 0.
|
|
$lRYDay = 0;
|
|
}
|
|
if ( defined $parts[2] and $parts[2] ne '' ) {
|
|
$lRHour = $parts[2];
|
|
} else {
|
|
# Missing hour, set default to immediately, -1.
|
|
$lRHour = 0;
|
|
}
|
|
} else {
|
|
# Force a schedule run
|
|
$lRYear = 0;
|
|
$lRYDay = 0;
|
|
$lRHour = -1;
|
|
}
|
|
$ivpRun{'last_run'} = "$lRYear $lRYDay $lRHour";
|
|
|
|
# Weed out any runs that are not due to be run.
|
|
# Calculate the difference (since 2000) between the last run hour and the current hour,
|
|
# ignoring leap years by saying every year is a leap year.
|
|
my $totalHourDiff = $nowTotalHours - (( ( $lRYear ) * 8784 ) + ( $lRYDay * 24 ) + $lRHour);
|
|
|
|
if ( $totalHourDiff == 0 ) {
|
|
# Same hour of the same day of the same year for some reason so ignore this IVP row.
|
|
logResponse( 'DS', 'GENERIC_RESPONSE', "Row $row, ID($ivpRun{'id'}), in the ivp table ".
|
|
"is being ignored because it was already run this hour." );
|
|
next;
|
|
} else {
|
|
# Determine the range of hours that are of interest within a 24 hour time period. Anything
|
|
# that is more than 24 hours old is automatically overdue and should be run.
|
|
if ( $totalHourDiff < 24) {
|
|
my $todayStartHour;
|
|
my $yesterdayStartHour;
|
|
if ( $yday > $lRYDay ) {
|
|
# Last run yesterday
|
|
$todayStartHour = 0;
|
|
$yesterdayStartHour = $lRHour + 1;
|
|
} else {
|
|
# Last run today
|
|
$todayStartHour = $hour - $totalHourDiff;
|
|
$yesterdayStartHour = 24;
|
|
}
|
|
|
|
# Less than a day so we need to worry about which scheduled hours were missed.
|
|
my $overdue = 0;
|
|
my @schedHours = split ( '\s', $ivpRun{'schedule'} );
|
|
foreach my $sHour ( @schedHours ) {
|
|
if ( $sHour == 0 and $sHour ne '0' ) {
|
|
# not numeric, ignore
|
|
logResponse( 'DS', 'GENERIC_RESPONSE', "Row $row, ID($ivpRun{'id'}), in the ivp table ".
|
|
"contains as schedule hour that is non-numeric: $sHour. ".
|
|
"The value will be ignored." );
|
|
next;
|
|
}
|
|
if (( $sHour >= $yesterdayStartHour ) or
|
|
( $sHour > $todayStartHour and $sHour <= $hour ) ) {
|
|
logResponse( 'DS', 'GENERIC_RESPONSE', "Row $row, ID($ivpRun{'id'}), in the ivp table ".
|
|
"is due to be run. Scheduled hour: $sHour, last run: $lRHour." );
|
|
$overdue = 1;
|
|
last;
|
|
}
|
|
}
|
|
if ( $overdue == 0 ) {
|
|
# did not find a scheduled hour that is due
|
|
logResponse( 'DS', 'GENERIC_RESPONSE', "Row $row, ID($ivpRun{'id'}), in the ivp table is not ".
|
|
"due to be run this hour." );
|
|
next;
|
|
}
|
|
} else {
|
|
logResponse( 'DS', 'GENERIC_RESPONSE', "Row $row, ID($ivpRun{'id'}), in the ivp table ".
|
|
"is due to be run. Last run over 24 hours ago." );
|
|
}
|
|
}
|
|
|
|
if ( defined $field[4] and $field[4] ne '' ) {
|
|
$ivpRun{'type_of_run'} = lc( $field[4] );
|
|
} else {
|
|
$ivpRun{'type_of_run'} = 'not specified';
|
|
}
|
|
if ( $ivpRun{'type_of_run'} ne 'basicivp' and $ivpRun{'type_of_run'} ne 'fullivp' ) {
|
|
my $badValue = $ivpRun{'type_of_run'};
|
|
if ( exists $mnInfo{'ip'} and ( $ivpRun{'ip'} eq $mnInfo{'ip'} )) {
|
|
if (( $mnInfo{'role'} eq 'controller' ) or
|
|
( $mnInfo{'role'} eq 'compute' ) or
|
|
( $mnInfo{'role'} eq 'compute_mn' )) {
|
|
$ivpRun{'type_of_run'} = 'fullivp';
|
|
} else {
|
|
$ivpRun{'type_of_run'} = 'basicivp';
|
|
}
|
|
} else {
|
|
$ivpRun{'type_of_run'} = 'basicivp';
|
|
}
|
|
logResponse( 'DS', 'GENERIC_RESPONSE', "Row $row, ID($ivpRun{'id'}), in the ivp table contains ".
|
|
"an unknown run type value of \'$badValue\'. Instead ".
|
|
"a run type of \'$ivpRun{'type_of_run'}\' will be performed." );
|
|
}
|
|
|
|
if ( defined $field[5] and $field[5] ne '' ) {
|
|
$ivpRun{'access_user'} = hexDecode( $field[5] );
|
|
} else {
|
|
$ivpRun{'access_user'} = '';
|
|
}
|
|
|
|
if ( defined $field[6] and $field[6] ne '' ) {
|
|
$ivpRun{'orch_parms'} = hexDecode( $field[6] );
|
|
} else {
|
|
$ivpRun{'orch_parms'} = '';
|
|
}
|
|
|
|
if ( defined $field[7] and $field[7] ne '' ) {
|
|
$ivpRun{'prep_parms'} = hexDecode( $field[7] );
|
|
} else {
|
|
$ivpRun{'prep_parms'} = '';
|
|
}
|
|
|
|
if ( defined $field[8] and $field[8] ne '' ) {
|
|
# Ensure that it is reencoded to pass along to do the actual work.
|
|
$ivpRun{'main_ivp_parms'} = hexEncode( $field[8] );
|
|
} else {
|
|
$ivpRun{'main_ivp_parms'} = '';
|
|
}
|
|
|
|
if ( defined $field[9] and $field[9] ne '' ) {
|
|
$ivpRun{'comments'} = hexDecode( $field[9] );
|
|
} else {
|
|
$ivpRun{'comments'} = '';
|
|
}
|
|
|
|
push @ivps, \%ivpRun;
|
|
}
|
|
} else {
|
|
# Table is empty. Create a single run for the z/VM related to the
|
|
# xCAT MN.
|
|
logResponse( 'DS', 'GENERIC_RESPONSE', "The ivp table is empty. Default IVPs will be enabled." );
|
|
if ( exists $mnInfo{'node'} ) {
|
|
# First IVP is a full IVP for an OpenStack related xCAT at 1 am.
|
|
# Otherwise it is a basic IVP.
|
|
my %ivpRun;
|
|
$ivpRun{'id'} = 1;
|
|
if ( exists $mnInfo{'ip'} ) {
|
|
$ivpRun{'ip'} = $mnInfo{'ip'};
|
|
} else {
|
|
$ivpRun{'ip'} = 'xcat';
|
|
}
|
|
$ivpRun{'schedule'} = '1';
|
|
if ( $mnInfo{'role'} eq 'controller' or $mnInfo{'role'} eq 'compute' or $mnInfo{'role'} eq 'compute_mn' ) {
|
|
$ivpRun{'type_of_run'} = 'fullivp';
|
|
$ivpRun{'access_user'} = 'root';
|
|
} else {
|
|
$ivpRun{'type_of_run'} = 'basicivp';
|
|
$ivpRun{'access_user'} = '';
|
|
}
|
|
$ivpRun{'orch_parms'} = '';
|
|
$ivpRun{'prep_parms'} = '';
|
|
$ivpRun{'main_ivp_parms'} = '';
|
|
$ivpRun{'comments'} = 'Default Full IVP run';
|
|
$ivpRun{'defaultRun'} = 'run';
|
|
push @ivps, \%ivpRun;
|
|
$totalRunsInTable++;
|
|
|
|
# Second IVP is a basic IVP run each hour except for 1 am when a full IVP is run.
|
|
my %ivpRun2;
|
|
$ivpRun2{'id'} = 2;
|
|
if ( exists $mnInfo{'ip'} ) {
|
|
$ivpRun2{'ip'} = $mnInfo{'ip'};
|
|
} else {
|
|
$ivpRun2{'ip'} = 'xcat';
|
|
}
|
|
$ivpRun2{'schedule'} = '0 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23';
|
|
$ivpRun2{'type_of_run'} = 'basicivp';
|
|
$ivpRun2{'access_user'} = '';
|
|
$ivpRun2{'orch_parms'} = '';
|
|
$ivpRun2{'prep_parms'} = '';
|
|
$ivpRun2{'main_ivp_parms'} = '';
|
|
$ivpRun2{'comments'} = 'Default Basic IVP run';
|
|
$ivpRun2{'defaultRun'} = 'runLater';
|
|
push @ivps, \%ivpRun2;
|
|
$totalRunsInTable++;
|
|
} else {
|
|
# xCAT MN does not exist as a node so we will only do a basic IVP the system.
|
|
# This is a very abnormal case. We deal in the abnormal so let's run some type
|
|
# of IVP but not define it in the table.
|
|
logResponse( 'DFS', 'MN03' );
|
|
my %ivpRun;
|
|
$ivpRun{'id'} = 1;
|
|
if ( exists $mnInfo{'ip'} ) {
|
|
$ivpRun{'ip'} = $mnInfo{'ip'};
|
|
} else {
|
|
$ivpRun{'ip'} = 'xcat';
|
|
}
|
|
$ivpRun{'schedule'} = '0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23';
|
|
$ivpRun{'type_of_run'} = 'basicivp';
|
|
$ivpRun{'access_user'} = '';
|
|
$ivpRun{'orch_parms'} = '';
|
|
$ivpRun{'prep_parms'} = '';
|
|
$ivpRun{'main_ivp_parms'} = '';
|
|
$ivpRun{'comments'} = 'Default Basic IVP run';
|
|
$ivpRun{'defaultRun'} = 'noTableUpdate';
|
|
push @ivps, \%ivpRun;
|
|
$totalRunsInTable++;
|
|
}
|
|
}
|
|
|
|
# Handle pruning, if necessary.
|
|
if ( $hour == 0 ) {
|
|
pruneLogs( $year, $mon + 1, $mday );
|
|
}
|
|
|
|
# Go thru the list of IVPs and drive a run for each IVP
|
|
# that is due to be run.
|
|
foreach my $ivp ( @ivps ) {
|
|
my %ivpRun = %$ivp;
|
|
|
|
# Build the verify node command.
|
|
my $cmd = '/opt/xcat/bin/verifynode -n xcat --notify';
|
|
|
|
if ( $ivpRun{'type_of_run'} eq 'fullivp' ) {
|
|
$cmd = "$cmd --fullivp --openstackip \'$ivpRun{'ip'}\' --openstackuser \'" . hexEncode( $ivpRun{'access_user'} ) . "\'";
|
|
} else {
|
|
$cmd = "$cmd --basicivp";
|
|
}
|
|
|
|
if ( $ivpRun{'orch_parms'} ne '' ) {
|
|
$cmd = "$cmd $ivpRun{'orch_parms'}"; # Don't hex encode orchestrator parms
|
|
}
|
|
|
|
if ( $ivpRun{'prep_parms'} ne '' ) {
|
|
$cmd = "$cmd --prepparms \'" . hexEncode( $ivpRun{'prep_parms'} ) . "\'";
|
|
}
|
|
|
|
if ( $ivpRun{'main_ivp_parms'} ne '' ) {
|
|
$cmd = "$cmd --zxcatparms \'" . hexEncode( $ivpRun{'main_ivp_parms'} ) . "\'";
|
|
}
|
|
|
|
# Update the table to indicate the time we start the IVP run.
|
|
if ( ! exists $ivpRun{'defaultRun'} ) {
|
|
my $chtabCmd = "/opt/xcat/sbin/chtab id=\'$ivpRun{'id'}\' zvmivp.last_run=\'$newLastRun\'";
|
|
$out = `$chtabCmd`;
|
|
$rc = $?;
|
|
if ( $rc != 0 ) {
|
|
logResponse( 'DS', '*NONFORMATTED*', "\'$chtabCmd\' failed for IVP ID($ivpRun{'id'}), ip: $ivpRun{'ip'}, rc: $rc, out: $out\n" );
|
|
}
|
|
} elsif ( $ivpRun{'defaultRun'} ne 'noTableUpdate' ) {
|
|
logResponse( 'DS', '*NONFORMATTED*', "Adding IVP ID($ivpRun{'id'}) to the z/VM IVP table: $ivpRun{'comments'}\n" );
|
|
my $chtabCmd = "/opt/xcat/sbin/chtab id=\'$ivpRun{'id'}\' ".
|
|
"zvmivp.ip=\'$ivpRun{'ip'}\' ".
|
|
"zvmivp.schedule=\'$ivpRun{'schedule'}\' ".
|
|
"zvmivp.last_run=\'$newLastRun\' ".
|
|
"zvmivp.type_of_run=\'$ivpRun{'type_of_run'}\' ".
|
|
"zvmivp.access_user=\'" . hexEncode( $ivpRun{'access_user'} ) . "\' ".
|
|
"zvmivp.comments=\'" . hexEncode( $ivpRun{'comments'} ) . "\' ";
|
|
$out = `$chtabCmd`;
|
|
$rc = $?;
|
|
if ( $rc != 0 ) {
|
|
logResponse( 'DS', '*NONFORMATTED*', "\'$chtabCmd\' failed for ID($ivpRun{'id'}), ip: $ivpRun{'ip'}, rc: $rc, out: $out\n" );
|
|
}
|
|
|
|
# Skip running this one if it is marked 'runLater'.
|
|
# A basic IVP is not needed if a full IVP is going to be run.
|
|
if ( $ivpRun{'defaultRun'} eq 'runLater' ) {
|
|
logResponse( 'DS', '*NONFORMATTED*', "Ignoring IVP ID($ivpRun{'id'}) for this hour. It will be run next hour. IVP comment: $ivpRun{'comments'}\n" );
|
|
next;
|
|
}
|
|
}
|
|
|
|
# Drive the run.
|
|
$attemptedRuns++;
|
|
my $logComments = '';
|
|
if ( $ivpRun{'comments'} eq '' ) {
|
|
$logComments = "Running IVP ID($ivpRun{'id'})";
|
|
} else {
|
|
$logComments = "Running IVP ID($ivpRun{'id'}) - $ivpRun{'comments'}";
|
|
}
|
|
logResponse( 'DS', '*NONFORMATTED*', "$logComments" );
|
|
$out = `$cmd`;
|
|
$rc = $?;
|
|
if ( $rc != 0 ) {
|
|
logResponse( 'DS', '*NONFORMATTED*', "The IVP run for ID($ivpRun{'id'}) returned a ".
|
|
"non-zero return code, rc: $rc\n" );
|
|
}
|
|
}
|
|
|
|
logResponse( 'DFS', '*NONFORMATTED*', "IVP CRON results: $totalRunsInTable runs considered, $disabledRuns runs disabled, $attemptedRuns runs attempted" );
|
|
$rc = 0;
|
|
return $rc;
|
|
}
|
|
|
|
|
|
#-------------------------------------------------------
|
|
|
|
=head3 enableDisableRun
|
|
|
|
Description : Enable or Disable command line function.
|
|
Arguments : ID of the IVP to be disabled
|
|
Global input set by command operands are used
|
|
by this routine:
|
|
$disable - Disable a run.
|
|
$enable - Enable a run.
|
|
Returns : 0 - No error
|
|
non-zero - Terminating error detected.
|
|
Example : $rc = enableDisableRun();
|
|
|
|
=cut
|
|
|
|
#-------------------------------------------------------
|
|
sub enableDisableRun {
|
|
my ( $id ) = @_;
|
|
my %ids;
|
|
my $rc = 0;
|
|
|
|
if ( $id eq '' ) {
|
|
logResponse( 'DF', 'OPER01', '--id' );
|
|
goto FINISH_enableDisableRun;
|
|
}
|
|
|
|
# Get the list of IVP ids in the zvmivp table.
|
|
my $cmd = '/opt/xcat/sbin/tabdump zvmivp | grep -v "^#"';
|
|
my $out = `$cmd`;
|
|
$rc = $?;
|
|
if ( $rc ne 0 ) {
|
|
logResponse( 'DF', 'GNRL04', $cmd, $rc, $out );
|
|
goto FINISH_enableDisableRun;
|
|
}
|
|
|
|
my @lines = split( '\n', $out );
|
|
foreach my $line ( @lines ) {
|
|
chomp( $line );
|
|
my ($ivpId, $junk) = split( ',', $line, 2 );
|
|
$ivpId =~ s/^\"|\"$//g; # trim quotes from both ends of the string
|
|
$ivpId =~ s/^\s+|\s+$//g; # trim spaces from both ends of the string
|
|
if ( $ivpId ne '' ) {
|
|
$ids{$ivpId} = 1;
|
|
}
|
|
}
|
|
|
|
# Validate the specified id.
|
|
if ( ! exists $ids{$id} ) {
|
|
logResponse( 'DF', 'ID03', $id );
|
|
}
|
|
|
|
my $action;
|
|
my $disableFlag = '';
|
|
if ( $disable ) {
|
|
$disableFlag = 'YES';
|
|
$action = 'disable';
|
|
} else {
|
|
$action = 'enable';
|
|
}
|
|
|
|
# Update the table.
|
|
logResponse( 'DS', '*NONFORMATTED*', "Updating the z/VM IVP table (zvmivp) to $action $id" );
|
|
my $chtabCmd = "/opt/xcat/sbin/chtab id='$id' zvmivp.disable=\'$disableFlag\'";
|
|
$out = `$chtabCmd`;
|
|
$rc = $?;
|
|
if ( $rc != 0 ) {
|
|
logResponse( 'DS', '*NONFORMATTED*', "\'$chtabCmd\' failed for ID($id), rc: $rc, out: $out\n" );
|
|
$rc = 1;
|
|
}
|
|
|
|
FINISH_enableDisableRun:
|
|
return $rc;
|
|
}
|
|
|
|
|
|
#-------------------------------------------------------
|
|
|
|
=head3 encodeDecodeString
|
|
|
|
Description : Encode and decode command line function.
|
|
Arguments : Global input set by command operands are used
|
|
by this routine:
|
|
$decode - String to be decoded if not ''.
|
|
$encode - String to be encoded if not ''.
|
|
Returns : 0 - No error
|
|
non-zero - Terminating error detected.
|
|
Example : $rc = encodeDecodeString();
|
|
|
|
=cut
|
|
|
|
#-------------------------------------------------------
|
|
sub encodeDecodeString {
|
|
my $newVal = '';
|
|
if ( $decode ne '' ) {
|
|
binmode(STDOUT, ":utf8");
|
|
print( "Value is: '" . hexDecode( $decode ) . "'\n" );
|
|
}
|
|
|
|
if ( $encode ne '' ) {
|
|
print( "Value is: '" . hexEncode( $encode ) . "'\n" );
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
|
|
#-------------------------------------------------------
|
|
|
|
=head3 finishLogFile
|
|
|
|
Description : Complete the log file for the run and
|
|
send it to the notify user if necessary.
|
|
Arguments : None
|
|
Returns : 0 - No error
|
|
non-zero - Terminating error detected.
|
|
Example : $rc = finishLogFile();
|
|
|
|
=cut
|
|
|
|
#-------------------------------------------------------
|
|
sub finishLogFile {
|
|
my $rc = 0;
|
|
$needToFinishLogFile = 0;
|
|
|
|
# Add the summary information to the log file.
|
|
my ($sec, $min, $hour, $mday, $mon, $year, $wday, $yday, $isdst) = localtime(time);
|
|
$endTime = sprintf("%02d:%02d:%02d on %04d-%02d-%02d",
|
|
$hour, $min, $sec, $year + 1900, $mon + 1, $mday );
|
|
|
|
logResponse( 'DF', '*NONFORMATTED*', '*******************************************************************************' );
|
|
logResponse( 'DF', 'GENERIC_RESPONSE', 'Results of the IVP run:\n' );
|
|
logResponse( 'DF', 'GENERIC_RESPONSE', "Run started at $startTime and ended at $endTime." );
|
|
foreach my $line ( @ivpSummary ) {
|
|
logResponse( 'DF', 'GENERIC_RESPONSE', $line );
|
|
}
|
|
if ( $ignoreCnt != 0 ){
|
|
logResponse( 'DFS', 'GENERIC_RESPONSE', "The orchestrator script (verifynode) ignored messages $ignoreCnt times." );
|
|
my @ignoreArray = sort keys %ignored;
|
|
my $ignoreList = join ( ', ', @ignoreArray );
|
|
logResponse( 'DFS', 'GENERIC_RESPONSE', "Message Ids of ignored messages by verifynode: $ignoreList" );
|
|
}
|
|
logResponse( 'DF', '*NONFORMATTED*', '*******************************************************************************' );
|
|
|
|
# Notify the z/VM userid, if necessary.
|
|
if ( $notify and $notifyOnErrorOrWarning ) {
|
|
notifyUser();
|
|
}
|
|
|
|
FINISH_finishLogFile:
|
|
return $rc;
|
|
}
|
|
|
|
|
|
#-------------------------------------------------------
|
|
|
|
=head3 getDistro
|
|
|
|
Description : Create the dummy image file.
|
|
Arguments : Node name or IP address
|
|
Returns : 0 - No error
|
|
non-zero - Terminating error detected.
|
|
Example : $rc = getDistro();
|
|
|
|
=cut
|
|
|
|
#-------------------------------------------------------
|
|
sub getDistro {
|
|
my ( $nodeOrIP ) = @_;
|
|
my $rc = 0;
|
|
|
|
$tgtOS = xCAT::zvmUtils->getOsVersion( $::SUDOER, $nodeOrIP );
|
|
|
|
FINISH_getDistro:
|
|
return $rc;
|
|
}
|
|
|
|
|
|
#-------------------------------------------------------
|
|
|
|
=head3 getLocalIPs
|
|
|
|
Description : Get the IP addresses from ifconfig.
|
|
Arguments : Node name or IP address
|
|
Returns : return code (Currently always 0)
|
|
0 - No error
|
|
non-zero - Terminating error detected.
|
|
Hash of local IP addresses
|
|
Example : $rc = getLocalIPs();
|
|
|
|
=cut
|
|
|
|
#-------------------------------------------------------
|
|
sub getLocalIPs {
|
|
my $ip;
|
|
my $junk;
|
|
my $rc = 0;
|
|
|
|
my $out = `/sbin/ip addr | grep -e '^\\s*inet' -e '^\\s*inet6'`;
|
|
my @lines = split( '\n', $out );
|
|
foreach my $line ( @lines ) {
|
|
my @parts = split( ' ', $line );
|
|
($ip) = split( '/', $parts[1], 2 );
|
|
$localIPs{$ip} = 1;
|
|
}
|
|
|
|
FINISH_getLocalIPs:
|
|
return $rc;
|
|
}
|
|
|
|
|
|
#-------------------------------------------------------
|
|
|
|
=head3 getMNInfo
|
|
|
|
Description : Get information related to the xCAT MN
|
|
and the z/VM system where it is running.
|
|
Arguments : None
|
|
Returns : 0 - No error
|
|
non-zero - Terminating error detected.
|
|
Example : $rc = getMNInfo();
|
|
|
|
=cut
|
|
|
|
#-------------------------------------------------------
|
|
sub getMNInfo {
|
|
my $cmd;
|
|
my $hostNode;
|
|
my @lines;
|
|
my $masterIP = '';
|
|
my $out = '';
|
|
my @parts;
|
|
my $rc = 0;
|
|
my $xcatMN;
|
|
|
|
# Get the list of Local IP addresses
|
|
getLocalIPs();
|
|
|
|
# Wait up to 10 minutes for xcatd to come up.
|
|
my $sleepTime = 60;
|
|
my $maxWaits = 10;
|
|
for ( my $i=1; $i<=$maxWaits; $i++ ) {
|
|
# lsdef uses the xCAT daemon. We use that to see if it is started.
|
|
`/opt/xcat/bin/lsdef`;
|
|
if ( $? == 0 ) {
|
|
last;
|
|
} else {
|
|
logResponse( 'DFS', 'WAIT01', $sleepTime, $i, $maxWaits );
|
|
sleep( $sleepTime );
|
|
}
|
|
}
|
|
|
|
# Get information related to all of the host nodes
|
|
$cmd = '/opt/xcat/bin/lsdef -w mgt=zvm -w hosttype=zvm -i hcp';
|
|
$out = `$cmd`;
|
|
$rc = $?;
|
|
if ( $rc == 0 ) {
|
|
@lines = split( '\n', $out );
|
|
my $host = '';
|
|
foreach my $line ( @lines ) {
|
|
$line =~ s/^\s+|\s+$//g; # trim blanks from both ends of the string
|
|
if ( $line =~ /Object name: / ) {
|
|
($host) = $line =~ m/Object name: (.*)/;
|
|
next;
|
|
} elsif ( $line =~ /hcp\=/ ) {
|
|
($hosts{$host}{'hcp'}) = $line =~ m/hcp\=(.*)/;
|
|
next;
|
|
}
|
|
}
|
|
} else {
|
|
$rc = logResponse( 'DFS', 'GNRL01', "$cmd", $rc, $out );
|
|
$notify = 1;
|
|
}
|
|
|
|
# Get key info related to the xCAT MN and the notify targets from the site table.
|
|
$mnInfo{'PruneIVP'} = 7;
|
|
$cmd = "/opt/xcat/sbin/tabdump site";
|
|
$out = `$cmd`;
|
|
$rc = $?;
|
|
if ( $rc == 0 ) {
|
|
@lines = split( '\n', $out );
|
|
@lines = grep( /^\"master\"|^\"zvmnotify\"|^\"zvmpruneivp\"/, @lines );
|
|
foreach my $line ( @lines ) {
|
|
$line =~ s/^\s+|\s+$//g; # trim blanks from both ends of the string
|
|
@parts = split( ',', $line );
|
|
$parts[1] =~ s/\"//g;
|
|
if ( $parts[0] =~ /^\"master\"/ ) {
|
|
$masterIP = $parts[1];
|
|
} elsif ( $parts[0] =~ /^\"zvmnotify\"/ ) {
|
|
my @notifyInfo = split( ';', $parts[1] );
|
|
my $notifyHost = '';
|
|
my $notifyUser = '';
|
|
foreach my $hostUser ( @notifyInfo ) {
|
|
($notifyHost) = $hostUser =~ m/(.*)\(/;
|
|
($notifyUser) = $hostUser =~ m/\((.*)\)/;
|
|
if ( defined $notifyHost and $notifyHost ne '' and defined $notifyUser and $notifyUser ne '' ) {
|
|
$hosts{$notifyHost}{'notifyUser'} = uc( $notifyUser );
|
|
} else {
|
|
$rc = logResponse( 'DFS', 'SITE01', 'zvmnotify', $parts[1], $hostUser );
|
|
$notify = 1;
|
|
}
|
|
}
|
|
} elsif ( $parts[0] =~ /^\"zvmpruneivp\"/ ) {
|
|
$mnInfo{'PruneIVP'} = $parts[1];
|
|
}
|
|
}
|
|
} else {
|
|
# Unable to read the site table
|
|
$rc = logResponse( 'DFS', 'GNRL01', "$cmd", $rc, $out );
|
|
$notify = 1;
|
|
}
|
|
|
|
# Find a node that has the same address as the master IP and then find
|
|
# the host node that relates to that node.
|
|
if ( $masterIP eq '' ) {
|
|
# Unable to get the master value
|
|
$rc = logResponse( 'DFS', 'SITE02', 'master' );
|
|
} else {
|
|
$cmd = "/opt/xcat/bin/lsdef -w ip=$masterIP -i hcp";
|
|
$out = `$cmd`;
|
|
$rc = $?;
|
|
if ( $rc == 0 ) {
|
|
@lines = split( '\n', $out );
|
|
foreach my $line ( @lines ) {
|
|
$line =~ s/^\s+|\s+$//g; # trim blanks from both ends of the string
|
|
if ( $line =~ /Object name: / ) {
|
|
($xcatMN) = $line =~ m/Object name: (.*)/;
|
|
$mnInfo{'node'} = $xcatMN;
|
|
$mnInfo{'ip'} = $masterIP;
|
|
next;
|
|
} elsif ( $line =~ /hcp\=/ ) {
|
|
($mnInfo{'hcp'}) = $line =~ m/hcp\=(.*)/;
|
|
next;
|
|
}
|
|
}
|
|
} else {
|
|
$rc = logResponse( 'DFS', 'MN01', $masterIP, $cmd, $rc, $out );
|
|
}
|
|
|
|
# Find the host node which uses the hcp.
|
|
foreach $hostNode ( keys %hosts ) {
|
|
if ( exists $hosts{$hostNode}{'hcp'} and $hosts{$hostNode}{'hcp'} eq $mnInfo{'hcp'} ) {
|
|
if ( exists $hosts{$hostNode}{'notify'} ) {
|
|
$hosts{$hostNode}{'notify'} = $hosts{$hostNode}{'notify'} . 'm';
|
|
} else {
|
|
$hosts{$hostNode}{'notify'} = 'm';
|
|
}
|
|
$mnInfo{'host'} = $hostNode;
|
|
}
|
|
}
|
|
}
|
|
|
|
# The following is useful in a z/VM CMA system only.
|
|
# As a safety measure, in case the site table is lost or does not have zvmnotify property,
|
|
# read the OPNCLOUD application system role file to find the value of the 'notify' property
|
|
# and remember it with the xCAT MN information.
|
|
# Also, determine the system role.
|
|
if ( -e $locApplSystemRole ) {
|
|
$out = `cat /var/lib/sspmod/appliance_system_role | grep -e '^notify\=' -e '^role\='`;
|
|
@lines = split( '\n', $out );
|
|
foreach my $line ( @lines ) {
|
|
$line =~ s/^\s+|\s+$//g; # trim blanks from both ends of the string
|
|
my ($key, $value) = split( '=', $line, 2 );
|
|
#($mnInfo{'notifyUser'}) = $line =~ m/^notify\=(.*)/;
|
|
if ( $key eq 'role' ) {
|
|
$mnInfo{'role'} = lc( $value );
|
|
}
|
|
if ( $key eq 'notify' ) {
|
|
$mnInfo{'notifyUser'} = uc( $value );
|
|
}
|
|
}
|
|
} else {
|
|
# Not CMA. Indicate role is not set.
|
|
$mnInfo{'role'} = '';
|
|
}
|
|
}
|
|
|
|
|
|
#-------------------------------------------------------
|
|
|
|
=head3 getOpenStackLevel
|
|
|
|
Description : Get the OpenStack version level as a
|
|
name.
|
|
Arguments : IP address of OpenStack compute node.
|
|
Returns : string - OpenStack version name
|
|
(e.g. NEWTON), or
|
|
null string - error
|
|
Example : $level = getOpenStackLevel();
|
|
|
|
=cut
|
|
|
|
#-------------------------------------------------------
|
|
sub getOpenStackLevel {
|
|
my ( $nodeOrIP, $openstackUser ) = @_;
|
|
my $cmd;
|
|
my $level = '';
|
|
my $numWords = 0;
|
|
my $out = '';
|
|
my @parts;
|
|
my $rc;
|
|
my %openStackVersion;
|
|
|
|
# Get the list of known versions
|
|
$cmd = 'cat /opt/xcat/openstack.versions';
|
|
$out = `$cmd`;
|
|
$rc = $?;
|
|
if ( $rc != 0 ) {
|
|
logResponse( 'DFS', 'GNRL01', $cmd, $rc, $out );
|
|
$level = '';
|
|
goto FINISH_getOpenStackLevel;
|
|
}
|
|
|
|
my @versionLines = split ( /\n/, $out );
|
|
foreach my $versionLine ( @versionLines ) {
|
|
if ( $versionLine =~ /^#/ ) { next; }
|
|
@parts = split( ' ', $versionLine );
|
|
$openStackVersion{$parts[0]} = $parts[1];
|
|
}
|
|
|
|
# Get the version information from the OpenStack node
|
|
$cmd = "nova-manage --version 2>&1";
|
|
($rc, $out) = sshToNode( $openstackIP, $openstackUser, $cmd );
|
|
if ( $rc != 0 ) {
|
|
# SSH failed, message already sent.
|
|
goto FINISH_getOpenStackLevel;
|
|
}
|
|
$out =~ s/^\s+|\s+$//g; # trim blanks from both ends of the string
|
|
++$numWords while $out =~ /\S+/g;
|
|
if ( $numWords != 1 ) {
|
|
logResponse( 'DFS', 'GOSL01', $cmd, $openstackIP, $rc, $out );
|
|
goto FINISH_getOpenStackLevel;
|
|
}
|
|
|
|
@parts = split( '\.', $out );
|
|
if ( !exists $parts[0] or !exists $parts[1] ) {
|
|
logResponse( 'DFS', 'GOSL01', $cmd, $openstackIP, $rc, $out );
|
|
goto FINISH_getOpenStackLevel;
|
|
} elsif ( exists $openStackVersion{"$parts[0]."} ) {
|
|
$level = $openStackVersion{"$parts[0]."}
|
|
} elsif ( exists $openStackVersion{"$parts[0].$parts[1]."} ) {
|
|
$level = $openStackVersion{"$parts[0].$parts[1]."}
|
|
} else {
|
|
logResponse( 'DFS', 'GOSL02', $cmd, $openstackIP, $rc, $out );
|
|
goto FINISH_getOpenStackLevel;
|
|
}
|
|
|
|
FINISH_getOpenStackLevel:
|
|
return $level;
|
|
}
|
|
|
|
|
|
#-------------------------------------------------------
|
|
|
|
=head3 hexDecode
|
|
|
|
Description : Convert a string of printable hex
|
|
characters (4 hex characters per actual
|
|
character) into the actual string that
|
|
it represents. The string should
|
|
begin with 'HexEncoded:' which indicates
|
|
that it is encoded. The routine tolerates
|
|
getting called with a string that is not
|
|
not encoded and will return the string that
|
|
was passed to it instead of trying to decode
|
|
it.
|
|
Arguments : printable hex value
|
|
Returns : Perl string
|
|
Example : $rc = hexDecode();
|
|
|
|
=cut
|
|
|
|
#-------------------------------------------------------
|
|
sub hexDecode {
|
|
my ( $hexVal ) = @_;
|
|
my $result = '';
|
|
|
|
if ( $hexVal =~ /^HexEncoded:/ ) {
|
|
($hexVal) = $hexVal =~ m/HexEncoded:(.*)/;
|
|
my @hexes = unpack( "(a4)*", $hexVal);
|
|
for ( my $i = 0; $i < scalar(@hexes); $i++ ) {
|
|
$result .= chr( hex( $hexes[$i] ) );
|
|
}
|
|
} else {
|
|
$result = $hexVal;
|
|
}
|
|
|
|
return $result;
|
|
}
|
|
|
|
|
|
#-------------------------------------------------------
|
|
|
|
=head3 hexEncode
|
|
|
|
Description : Convert a string into a string of
|
|
printable hex characters in which each
|
|
character in the original string is
|
|
represented by the 2 byte (4 char) hex
|
|
code value. The string is preceded by
|
|
'HexEncoded:' to indicate that it has
|
|
been encoded. The routine will tolerate
|
|
getting called with a string that has
|
|
already been encoded and not reencode it.
|
|
Arguments : ASCII or Unicode string
|
|
Returns : Hex string
|
|
Example : $rc = hexEncode();
|
|
|
|
=cut
|
|
|
|
#-------------------------------------------------------
|
|
sub hexEncode {
|
|
my ( $str ) = @_;
|
|
my $hex;
|
|
my $result = $str; # All work done within the result variable.
|
|
|
|
if ( $result ne '' ) {
|
|
# Encode the string if is not already encoded. Otherwise, leave it alone.
|
|
if ( $result !~ /^HexEncoded:/ ) {
|
|
$result =~ s/(.)/sprintf("%04x",ord($1))/eg;
|
|
$result = "HexEncoded:$result";
|
|
}
|
|
}
|
|
|
|
return $result;
|
|
}
|
|
|
|
|
|
#-------------------------------------------------------
|
|
|
|
=head3 logResponse
|
|
|
|
Description : Build and log the response.
|
|
Arguments : Target destination for the response. This is
|
|
string which can contain a charater for each
|
|
possible destination, e.g 'DSF'. The values are:
|
|
D - send to STDOUT
|
|
S - send to syslog
|
|
F - send to the log file
|
|
The default is 'DF' so that we send the output
|
|
to STDOUT and put it in the log file.
|
|
message ID or special flag:
|
|
*NONFORMATTED* indicates that the message build should not
|
|
be invoked but instead the message substitutions are
|
|
lines for the message to be produced.
|
|
*RESET* indicates that the message counter should be reset.
|
|
Array of message substitutions
|
|
Returns : 0 - No error, general response or info message detected.
|
|
1 - Non-terminating message detected.
|
|
2 - Terminating message detected.
|
|
Example : $rc = logResponse( 'D', 'VX01' );
|
|
$rc = logResponse( '', 'VX03', $nodeName, $sub2);
|
|
$rc = logResponse( 'DFS', 'VX03', $nodeName, 'sub2a');
|
|
|
|
=cut
|
|
|
|
#-------------------------------------------------------
|
|
sub logResponse {
|
|
my ( $dest, $msgId, @msgSubs ) = @_;
|
|
my $rc = 0;
|
|
my $extraInfo = '';
|
|
my $msg = '';
|
|
my @msgLines;
|
|
my $sev;
|
|
my $line;
|
|
|
|
if ( $msgId eq '*RESET*' ) {
|
|
$warnErrCnt = 0;
|
|
goto FINISH_logResponse;
|
|
} elsif ( $msgId eq '*NONFORMATTED*' ) {
|
|
$rc = 0;
|
|
if ( @msgSubs ) {
|
|
foreach my $line ( @msgSubs ) {
|
|
$msg = "$msg$line\n";
|
|
}
|
|
} else {
|
|
$msg = '\n';
|
|
}
|
|
$sev = 0;
|
|
} else {
|
|
( $rc, $sev, $msg, $extraInfo ) = xCAT::zvmMsgs->buildMsg('VERIFYNODE', $msgId, \@msgSubs);
|
|
if ( defined $msgsToIgnore{$msgId} ) {
|
|
# Ignore this message id
|
|
$ignored{$msgId} = 1;
|
|
$ignoreCnt += 1;
|
|
print( "Message $msgId is being ignored but would have occurred here.\n" );
|
|
goto FINISH_logResponse;
|
|
} elsif ( defined $msgsToIgnore{$sev} ) {
|
|
# Ignoring all messages of this severity.
|
|
$ignored{$msgId} = 1;
|
|
$ignoreCnt += 1;
|
|
print( "Message $msgId is being ignored but would have occurred here.\n" );
|
|
goto FINISH_logResponse;
|
|
}
|
|
}
|
|
|
|
if ( $sev >= 4 ) {
|
|
$warnErrCnt += 1;
|
|
}
|
|
|
|
# Send the message to the requested destination.
|
|
if ( $dest =~ 'D' ) {
|
|
print "$msg"; # Send message to STDOUT
|
|
}
|
|
if ( $dest =~ 'F' and defined $logFileHandle ) {
|
|
print $logFileHandle $msg;
|
|
}
|
|
if ( $dest =~ 'S' ) {
|
|
my $logMsg = $msg;
|
|
$logMsg =~ s/\t//g;
|
|
$logMsg =~ s/\n/ /g;
|
|
syslog( 'err', $logMsg );
|
|
}
|
|
|
|
# Send the extra info to the requested destination (never send it to syslog).
|
|
if ( $extraInfo ne '' ) {
|
|
if ( $dest =~ 'D' ) {
|
|
print "$extraInfo";
|
|
}
|
|
if ( $dest =~ 'F' and defined $logFileHandle ) {
|
|
print $logFileHandle $msg;
|
|
}
|
|
}
|
|
|
|
FINISH_logResponse:
|
|
return $rc;
|
|
}
|
|
|
|
|
|
#-------------------------------------------------------
|
|
|
|
=head3 notifyUser
|
|
|
|
Description : Notify a z/VM user. Send a message
|
|
and the log as a SPOOL file.
|
|
Arguments : Node name or IP address
|
|
Returns : 0 - Always ok.
|
|
Example : $rc = notifyUser();
|
|
|
|
=cut
|
|
|
|
#-------------------------------------------------------
|
|
sub notifyUser {
|
|
my $cmd;
|
|
my $host;
|
|
my @hostOrder;
|
|
my $msg = '';
|
|
my $out = '';
|
|
my $rc = 0;
|
|
my $tempFile;
|
|
my $th;
|
|
|
|
# Determine who we should notify based on the host related to the verification.
|
|
# The driver script would have set the host related to an full IVP (if we can trust it).
|
|
# If we cannot get to the host related to the run then we fall back to sending
|
|
# the notification to the user on the host in which the xCAT MN is running.
|
|
# If that fails then "tough luck, Chuck".
|
|
my $mnAdded = 0;
|
|
foreach $host ( keys %hosts ) {
|
|
if ( exists $hosts{$host}{'notify'} and exists $hosts{$host}{'hcp'} and exists $hosts{$host}{'notifyUser'} ) {
|
|
if ( $hosts{$host}{'notify'} =~ /d/ ) {
|
|
# Driver script specified hosts go on the top of the stack.
|
|
push @hostOrder, $host;
|
|
if ( $hosts{$host}{'notify'} =~ /m/ ) {
|
|
# The host related to the MN is already in the list
|
|
$mnAdded = 1;
|
|
}
|
|
} elsif ( $hosts{$host}{'notify'} =~ /m/ ) {
|
|
# Management node systems go on the bottom as a fall back plan.
|
|
unshift @hostOrder, $host;
|
|
$mnAdded = 1;
|
|
}
|
|
}
|
|
}
|
|
if ( $mnAdded == 0 and exists $mnInfo{'hcp'} and exists $mnInfo{'notifyUser'} ) {
|
|
# Did not find a host related to this management node but we have the
|
|
# necessary information to send the notification so add a dummy host
|
|
# to the hosts hash and add the dummy host to the end of the hostOrder
|
|
# stack to be used when all other notification attemps fail.
|
|
$host = '*DEFAULT_MN_HOST*';
|
|
$hosts{$host}{'notify'} = 'm';
|
|
$hosts{$host}{'notifyUser'} = $mnInfo{'notifyUser'};
|
|
$hosts{$host}{'hcp'} = $mnInfo{'hcp'};
|
|
unshift @hostOrder, $host;
|
|
}
|
|
|
|
# Prepare the message.
|
|
$msg = "The xCAT IVP detected some possible issues. A log file is being sent to your reader.";
|
|
|
|
# Prepare the log file by removing tabs and splitting the log file at 80 character lines.
|
|
$tempFile = mktemp( "$logDir/$logFile.XXXXXX" );
|
|
`expand $logDir/$logFile | fold -w 80 1>$tempFile`;
|
|
|
|
# Send the message and log file to the first entity in the host order list that lets us
|
|
# successfully do so.
|
|
my $done = 0;
|
|
my $failed = 0;
|
|
foreach $host ( @hostOrder ) {
|
|
# Check if zHCP's punch is online and online it if it is not.
|
|
($rc, $out) = sshToNode( $hosts{$host}{'hcp'}, '', 'cat /sys/bus/ccw/drivers/vmur/0.0.000d/online' );
|
|
chomp( $out );
|
|
if ($rc != 0 or $out != 1) {
|
|
$cmd = '/sbin/cio_ignore -r 000d;/sbin/chccwdev -e 000d';
|
|
($rc, $out) = sshToNode( $hosts{$host}{'hcp'}, '', $cmd );
|
|
chomp( $out );
|
|
if ( $rc != 0 or !( $out =~ m/Done$/i ) ) {
|
|
$rc = logResponse( 'DFS', 'GNRL03', $hosts{$host}{'hcp'}, $cmd, $rc, $out );
|
|
next;
|
|
}
|
|
($rc, $out) = sshToNode( $hosts{$host}{'hcp'}, '', 'which udevadm &> /dev/null && udevadm settle || udevsettle' );
|
|
# Don't worry about the udevadm settle. If it remains a problem then we will see it on the next command to ZHCP.
|
|
}
|
|
|
|
# Send log file. It will exist temporarily on the ZHCP agent in the /tmp directory
|
|
# under its original log file name.
|
|
$rc = xCAT::zvmUtils->sendFile( 'root', $hosts{$host}{'hcp'}, "$tempFile", "/tmp/$logFile" );
|
|
if ( $rc != 0 ) {
|
|
# An error is not a problem because the zhcp node could be logged off.
|
|
next;
|
|
}
|
|
|
|
# Punch the file from the ZHCP agent to the target user.
|
|
$out = xCAT::zvmCPUtils->punch2Reader( 'root', $hosts{$host}{'hcp'}, $hosts{$host}{'notifyUser'}, "/tmp/$logFile", 'XCAT_IVP.RESULTS', '-t', 'A' );
|
|
if ( $out ne 'Done' ) {
|
|
$rc = logResponse( 'DFS', 'GNRL02', "/tmp/$logFile", $hosts{$host}{'hcp'}, $hosts{$host}{'notifyUser'}, $host, "$logDir/$logFile", $out );
|
|
$failed = 1;
|
|
}
|
|
|
|
# Clean up the ZHCP node.
|
|
($rc, $out) = sshToNode( $hosts{$host}{'hcp'}, '', "rm -f /tmp/$logFile" );
|
|
if ( $rc != 0 ) {
|
|
$rc = logResponse( 'DFS', 'GNRL03', $hosts{$host}{'hcp'}, "rm -f /tmp/$logFile", $rc, $out );
|
|
next;
|
|
}
|
|
|
|
# Try another host if we failed to send the file.
|
|
if ( $failed ) {
|
|
$failed = 0;
|
|
next;
|
|
}
|
|
|
|
# Send a message if the user is logged on the system.
|
|
($rc, $out) = sshToNode( $hosts{$host}{'hcp'}, '', "vmcp query user $hosts{$host}{'notifyUser'}" );
|
|
if ( $rc != 0 ) {
|
|
# User is either not logged on or we had a problem issuing the command. Not important, leave.
|
|
last;
|
|
}
|
|
$out = uc( $out );
|
|
if ( $out =~ /^$hosts{$host}{'notifyUser'}/ ) {
|
|
my ($userStatus, $junk) = split '\s', $out, 2;
|
|
chomp( $userStatus );
|
|
$userStatus =~ s/^\s+|\s+$//g; # trim both ends of the string
|
|
if ( $userStatus eq 'SSI' or $userStatus eq 'DSC' ) {
|
|
logResponse( 'DFS', 'MSG01', $hosts{$host}{'notifyUser'}, $host, $userStatus );
|
|
last;
|
|
}
|
|
}
|
|
|
|
# Don't worry if message still does not work. It is only a message.
|
|
($rc, $out) = sshToNode( $hosts{$host}{'hcp'}, '', "vmcp message $hosts{$host}{'notifyUser'} $msg" );
|
|
last;
|
|
}
|
|
|
|
# Remove the temporary log file from the xCAT MN system.
|
|
$out = `rm -f $tempFile`;
|
|
$rc = $?;
|
|
if ( $rc != 0 ) {
|
|
logResponse( 'DFS', 'GNRL01', "rm -f $tempFile", $rc, $out );
|
|
}
|
|
|
|
$rc = 0;
|
|
return $rc;
|
|
}
|
|
|
|
|
|
#-------------------------------------------------------
|
|
|
|
|
|
=head3 pruneLogs
|
|
|
|
Description : Compress and prune log files. The number of days to wait
|
|
before pruning is specified in the site table with the
|
|
zvmpruneivp property. If the property does not exist
|
|
or Management node information does not exist because of
|
|
some reason then the default is to prune logs that are
|
|
7 days or older.
|
|
Arguments : Current year
|
|
Current month (numeric)
|
|
Current day (Julian day, numeric)
|
|
Returns : None.
|
|
Example : pruneLogs( $year, $mon + 1, $mday );
|
|
|
|
=cut
|
|
|
|
#-------------------------------------------------------
|
|
sub pruneLogs{
|
|
my ( $year, $month, $day ) = @_;
|
|
|
|
my @lines;
|
|
my $pruneOffset = 0;
|
|
my $out;
|
|
my $pruneDate = '';
|
|
my $rc;
|
|
my $removed = 0;
|
|
my $zipOut;
|
|
my $zipped = 0;
|
|
logResponse( 'DFS', '*NONFORMATTED*', "CRON job is pruning log files." );
|
|
|
|
# Determine the prune date value.
|
|
if ( ! exists $mnInfo{'PruneIVP'} or $mnInfo{'PruneIVP'} eq '' ) {
|
|
$pruneOffset = 7;
|
|
} else {
|
|
if ( looks_like_number( $mnInfo{'PruneIVP'} ) ) {
|
|
$pruneOffset = $mnInfo{'PruneIVP'};
|
|
} else {
|
|
$pruneOffset = 7;
|
|
logResponse( 'DFS', 'TP01', 'zvmpruneivp', 'site', 'not numeric', $mnInfo{'PruneIVP'}, $pruneOffset );
|
|
}
|
|
}
|
|
$pruneDate = strftime "%Y-%m-%d", 0, 0, 0, $day - $pruneOffset, $month-1, $year-1900;
|
|
logResponse( 'DFS', '*NONFORMATTED*', "Prune date: $pruneDate." );
|
|
|
|
# Zip any log files that are not for today. Current log files are similar to
|
|
# /var/log/xcat/ivp/IVP_XCAT_2016-09-19_01:01:03.log and will have .gz appended to the name
|
|
# after they have been zipped.
|
|
$out = `ls -1 /var/log/xcat/ivp/*.log`;
|
|
$rc = $?;
|
|
if ( $rc == 0 ) {
|
|
@lines = split( '\n', $out );
|
|
foreach my $fqFile ( @lines ) {
|
|
chomp( $fqFile );
|
|
my ($file) = $fqFile =~ /\/var\/log\/xcat\/ivp\/(.*)/;
|
|
my @parts = split( '_', $file );
|
|
if ( ! defined $parts[0] or $parts[0] ne 'IVP' or
|
|
! defined $parts[1] or ! defined $parts[2] or
|
|
$parts[2] eq $todayDate ) {
|
|
next;
|
|
}
|
|
#logResponse( 'DFS', 'GENERIC_RESPONSE', "ZIPPING: $file\n" );
|
|
$zipOut = `gzip $fqFile 2>&1`;
|
|
$rc = $?;
|
|
if ( $rc != 0 ) {
|
|
logResponse( 'DF', 'GENERIC_RESPONSE', "Unable to gzip $fqFile, rc: $rc, out: $out" );
|
|
} else {
|
|
$zipped++;
|
|
}
|
|
}
|
|
}
|
|
|
|
# Prune any gzipped files that are too old.
|
|
$out = `ls -1 /var/log/xcat/ivp/*.log.gz`;
|
|
$rc = $?;
|
|
if ( $rc == 0 ) {
|
|
@lines = split( '\n', $out );
|
|
foreach my $fqFile ( @lines ) {
|
|
chomp( $fqFile );
|
|
my ($file) = $fqFile =~ /\/var\/log\/xcat\/ivp\/(.*)/;
|
|
my @parts = split( '_', $file );
|
|
if ( ! defined $parts[0] or $parts[0] ne 'IVP' or
|
|
! defined $parts[1] or ! defined $parts[2] or
|
|
$parts[2] ge $todayDate ) {
|
|
next;
|
|
}
|
|
if ( $parts[2] ge $pruneDate ) {
|
|
next;
|
|
}
|
|
|
|
#logResponse( 'DFS', 'GENERIC_RESPONSE', "REMOVING: $file\n" );
|
|
$zipOut = `rm $fqFile 2>&1`;
|
|
$rc = $?;
|
|
if ( $rc != 0 ) {
|
|
logResponse( 'DF', 'GENERIC_RESPONSE', "Unable to remove \'rm $fqFile\', rc: $rc, out: $out" );
|
|
} else {
|
|
$removed++;
|
|
}
|
|
}
|
|
}
|
|
|
|
logResponse( 'DFS', '*NONFORMATTED*', "IVP log pruning completed, zipped: $zipped, pruned: $removed." );
|
|
return;
|
|
}
|
|
|
|
|
|
#-------------------------------------------------------
|
|
|
|
=head3 removeIVP
|
|
|
|
Description : Remove an automated IVP.
|
|
Arguments : ID of the IVP to be removed
|
|
Returns : 0 - No error
|
|
non-zero - Terminating error detected.
|
|
Example : $rc = removeIVP();
|
|
|
|
=cut
|
|
|
|
#-------------------------------------------------------
|
|
sub removeIVP {
|
|
my ( $id ) = @_;
|
|
my $cmd;
|
|
my $out;
|
|
my $rc;
|
|
my $ivpId;
|
|
my %ids;
|
|
my $junk;
|
|
|
|
# Get the list of IVP ids in the zvmivp table.
|
|
$cmd = '/opt/xcat/sbin/tabdump zvmivp | grep -v "^#"';
|
|
$out = `$cmd`;
|
|
$rc = $?;
|
|
if ( $rc ne 0 ) {
|
|
logResponse( 'DF', 'GNRL04', $cmd, $rc, $out );
|
|
goto FINISH_removeIVP;
|
|
}
|
|
my @lines = split( '\n', $out );
|
|
foreach my $line ( @lines ) {
|
|
chomp( $line );
|
|
my ($ivpId, $junk) = split( ',', $line, 2 );
|
|
$ivpId =~ s/^\"|\"$//g; # trim quotes from both ends of the string
|
|
$ivpId =~ s/^\s+|\s+$//g; # trim spaces from both ends of the string
|
|
if ( $ivpId ne '' ) {
|
|
$ids{$ivpId} = 1;
|
|
}
|
|
}
|
|
|
|
# Validate the specified id.
|
|
if ( $id eq '' or ! exists $ids{$id} ) {
|
|
logResponse( 'DF', 'ID03', $id );
|
|
}
|
|
|
|
# Update the table.
|
|
logResponse( 'DS', '*NONFORMATTED*', "Updating the z/VM IVP table (zvmivp) to remove $id" );
|
|
my $chtabCmd = "/opt/xcat/sbin/chtab -d id=\'$id\' zvmivp";
|
|
$out = `$chtabCmd`;
|
|
$rc = $?;
|
|
if ( $rc != 0 ) {
|
|
logResponse( 'DS', '*NONFORMATTED*', "\'$chtabCmd\' failed for ID($id), rc: $rc, out: $out\n" );
|
|
$rc = 1;
|
|
}
|
|
|
|
FINISH_removeIVP:
|
|
return $rc;
|
|
}
|
|
|
|
|
|
#-------------------------------------------------------
|
|
|
|
=head3 runDriverScript
|
|
|
|
Description : Run a downloaded driver script.
|
|
Arguments : This routine is driven from the action
|
|
table, verifySets. It takes global
|
|
input that can be set as command
|
|
input or by precursor routines.
|
|
The input variables include:
|
|
$driver - Driver script name
|
|
$driverLoc - Location of the driver script
|
|
$hosts - Infor for defined host nodes
|
|
$infoCnt - Count of warnings that were generated
|
|
$warningCnt - Count of warnings that were generated
|
|
$zxcatParms - Command line parms to
|
|
pass to zxcatIVP
|
|
Returns : 0 - No error
|
|
non-zero - Terminating error detected.
|
|
Example : $rc = runDriverScript();
|
|
|
|
=cut
|
|
|
|
#-------------------------------------------------------
|
|
sub runDriverScript{
|
|
my $infoCnt = 0;
|
|
my @lines;
|
|
my $out;
|
|
my $rc = 0;
|
|
my $warningCnt = 0;
|
|
|
|
# Analyze the driver script to determine the z/VM host that it intends to support.
|
|
my $hostNodeZhcp = '';
|
|
my $hostNode = `cat $driverLoc/$driver | grep "^export zxcatIVP_hostNode\=" | sed '/^export zxcatIVP_hostNode\=*/!d; s///; s/\"//g;q'`;
|
|
chomp( $hostNode );
|
|
|
|
if ( $hostNode ne '' ) {
|
|
if ( exists $hosts{$hostNode}{'hcp'} ) {
|
|
if ( exists $hosts{$hostNode}{'notify'} ) {
|
|
$hosts{$hostNode}{'notify'} = $hosts{$hostNode}{'notify'} . 'd';
|
|
} else {
|
|
$hosts{$hostNode}{'notify'} = 'd';
|
|
}
|
|
$zhcpTarget = $hosts{$hostNode}{'hcp'};
|
|
$zvmHost = $hostNode;
|
|
push @ivpSummary, "The ZHCP related to the host, \'$hostNode\', identified in the driver script has been identified ".
|
|
"as \'$zhcpTarget\'.";
|
|
} else {
|
|
$rc = logResponse( 'DFS', 'DRIV03', $zvmHost );
|
|
$notify = 1;
|
|
}
|
|
} else {
|
|
# Driver script did not specify the host node.
|
|
$rc = logResponse( 'DFS', 'DRIV02', "$driverLoc/$driver", 'zxcatIVP_hostNode' );
|
|
$notify = 1;
|
|
}
|
|
|
|
# Analyze the driver script to determine the ZHCP node that it intends to support.
|
|
my $zhcpNode = `cat $driverLoc/$driver | grep "^export zxcatIVP_zhcpNode\=" | sed '/^export zxcatIVP_zhcpNode\=*/!d; s///; s/\"//g;q'`;
|
|
chomp( $zhcpNode );
|
|
my $zhcpHcp = `/opt/xcat/bin/lsdef $zhcpNode | grep "hcp\=" | sed '/hcp\=*/!d; s///; s/\"//g;q'`;
|
|
if ( $zhcpHcp ne '' ) {
|
|
$zhcpHcp =~ s/^\s+|\s+$//g; # trim both ends of the string
|
|
$zhcpTarget = $zhcpHcp;
|
|
foreach $hostNode ( keys %hosts ) {
|
|
if ( exists $hosts{$hostNode}{'hcp'} and $hosts{$hostNode}{'hcp'} eq $zhcpHcp ) {
|
|
if ( exists $hosts{$hostNode}{'notify'} ) {
|
|
$hosts{$hostNode}{'notify'} = $hosts{$hostNode}{'notify'} . 'd';
|
|
} else {
|
|
$hosts{$hostNode}{'notify'} = 'd';
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
my $exportCmd = '';
|
|
if ( $zxcatParms ne '' ) {
|
|
$ENV{zxcatIVP_moreCmdOps}=$zxcatParms;
|
|
} else {
|
|
delete $ENV{zxcatIVP_moreCmdOps};
|
|
}
|
|
|
|
logResponse( 'DF', '*NONFORMATTED*', '*******************************************************************************' );
|
|
logResponse( 'DF', 'GENERIC_RESPONSE_NOINDENT', "Using the driver script to drive the IVP. The output from the run follows." );
|
|
logResponse( 'DF', '*NONFORMATTED*', '*******************************************************************************' );
|
|
$out = `chmod +x $driverLoc/$driver`;
|
|
$out = `$driverLoc/$driver`;
|
|
$rc = $?;
|
|
if ( $rc != 0 ) {
|
|
$rc = logResponse( 'DFS', 'DRIV01', "$driverLoc/$driver", $rc, $out );
|
|
$notify = 1;
|
|
goto FINISH_runDriverScript;
|
|
}
|
|
|
|
# Determine how many warnings and information messages were generated so that we can
|
|
# produce a summary message upon completion of the IVP.
|
|
@lines = split( '\n', $out );
|
|
foreach my $line ( @lines ) {
|
|
if ( $line =~ /^Warning \(/ ) { $warningCnt++; }
|
|
if ( $line =~ /^Info \(/ ) { $infoCnt++; }
|
|
}
|
|
push @ivpSummary, "The driver script generated $warningCnt warnings and $infoCnt information messages.";
|
|
logResponse( 'DF', '*NONFORMATTED*', $out );
|
|
|
|
FINISH_runDriverScript:
|
|
if ( $warningCnt != 0 ) {
|
|
$notify = 1;
|
|
}
|
|
return $rc;
|
|
}
|
|
|
|
|
|
#-------------------------------------------------------
|
|
|
|
=head3 runPrepScript
|
|
|
|
Description : Push the appropriate level of preparation
|
|
script to a compute node and run it to
|
|
validate the system and build the driver
|
|
script.
|
|
Arguments : Global defaults set as command line parameters provide input
|
|
to this routine:
|
|
$driver - Name of driver script
|
|
$driverLoc - Location of the driver script
|
|
$infoCnt - Count of warnings that were generated
|
|
$openstackIP - IP for OpenStack compute node
|
|
$openstackUser - User for OpenStack compute node
|
|
$prepParms - Parms for preparation script
|
|
$warnErrCnt - Count of warnings and errors that were generated
|
|
Returns : 0 - No error
|
|
non-zero - Terminating error detected.
|
|
Example : $rc = runPrepScript();
|
|
|
|
=cut
|
|
|
|
#-------------------------------------------------------
|
|
sub runPrepScript {
|
|
my $cmd = '';
|
|
my $infoCnt = 0;
|
|
my $line;
|
|
my @lines;
|
|
my $out = '';
|
|
my $rc = 0;
|
|
my $retRC = 0;
|
|
my $tmpDir = '';
|
|
my $warningCnt = 0;
|
|
|
|
# Determine the name of the driver script based on the IP address for the compute node.
|
|
$driver = "zxcatIVPDriver_$openstackIP.sh";
|
|
|
|
# Create the local IVP directory.
|
|
if ( !-d $driverLoc ) {
|
|
$out = `mkdir -m a=rwx,g=rx,o= -p $driverLoc`;
|
|
$rc = $?;
|
|
if ( $rc != 0 ) {
|
|
$rc = logResponse( 'DFS', 'PREP01', $driverLoc, $rc, $out );
|
|
$notify = 1;
|
|
goto FINISH_runPrepScript;
|
|
}
|
|
}
|
|
|
|
# Determine the OpenStack level.
|
|
my $level = getOpenStackLevel( $openstackIP, $openstackUser );
|
|
if ( $level eq '' ) {
|
|
$notify = 1;
|
|
$rc = logResponse( 'DFS', 'PREP06', $openstackIP );
|
|
goto FINISH_runPrepScript;
|
|
}
|
|
my $levelLetter = substr( $level, 0, 1);
|
|
push @ivpSummary, "OpenStack system at $openstackIP is running the Nova $level release.";
|
|
|
|
# Determine the appropriate preparation script to send to the compute node.
|
|
my $prepDir = '/opt/xcat/share/xcat/tools/zvm';
|
|
my $prepScript = "prep_zxcatIVP_$level.pl";
|
|
logResponse( 'DF', 'GENERIC_RESPONSE', "Attempting to push $prepScript to the OpenStack system ".
|
|
"at $openstackIP. It will be used to validate the OpenStack environment and create ".
|
|
"a driver script to be used for the validation on the xCAT management node." );
|
|
my $runPrepScript = 1;
|
|
|
|
# Create a directory on the compute node to hold the preparation script and driver script.
|
|
$cmd = 'mktemp -d /tmp/xCAT.XXXXXXXXXX';
|
|
($rc, $tmpDir) = sshToNode( $openstackIP, $openstackUser, $cmd );
|
|
if ( $rc != 0 ) {
|
|
# Unable to create the directory. Let's see if we have an old driver script that we can use.
|
|
$rc = logResponse( 'DFS', 'PREP02', $openstackIP, "$driverLoc/$driver", $cmd, $rc, $out );
|
|
$notify = 1;
|
|
goto FINISH_runPrepScript;
|
|
} else {
|
|
# Push the preparation script to the compute node and run it.
|
|
chomp($tmpDir);
|
|
$cmd = "/usr/bin/scp $prepDir/$prepScript $openstackUser\@$openstackIP:$tmpDir";
|
|
$out = `/usr/bin/scp "$prepDir/$prepScript" "$openstackUser\@$openstackIP:$tmpDir"`;
|
|
$rc = $?;
|
|
if ( $rc != 0 ) {
|
|
# Unable to push the preparation script. Let's see if we have an old driver script that we can use.
|
|
$rc = logResponse( 'DFS', 'PREP02', $openstackIP, "$driverLoc/$driver", $cmd, $rc, $out );
|
|
$notify = 1;
|
|
goto FINISH_runPrepScript;
|
|
}
|
|
}
|
|
|
|
if ( $runPrepScript ) {
|
|
# Run the preparation script.
|
|
($rc, $out) = sshToNode( $openstackIP, $openstackUser, "chmod +x $tmpDir/$prepScript");
|
|
($rc, $out) = sshToNode( $openstackIP, $openstackUser, "$tmpDir/$prepScript '--driver' $tmpDir/$driver $prepParms" );
|
|
logResponse( 'DF', '*NONFORMATTED*', '*******************************************************************************' );
|
|
logResponse( 'DF', 'GENERIC_RESPONSE_NOINDENT', "The output from the preparation script that is run on the compute node follows. ".
|
|
"The preparation script validates the z/VM related OpenStack configuration file properties." );
|
|
logResponse( 'DF', '*NONFORMATTED*', '*******************************************************************************' );
|
|
logResponse( 'DF', '*NONFORMATTED*', $out );
|
|
|
|
# Determine how many warnings were generated so that we can produce a summary message upon completion of the IVP.
|
|
if ( $levelLetter gt 'L' ) {
|
|
@lines = split( '\n', $out );
|
|
foreach $line ( @lines ) {
|
|
if ( $line =~ /^Warning \(/ ) { $warningCnt++; }
|
|
if ( $line =~ /^Info \(/ ) { $infoCnt++; }
|
|
}
|
|
} else {
|
|
@lines = split( '\n', $out );
|
|
foreach $line ( @lines ) {
|
|
if ( $line =~ /^Warning:/ ) { $warningCnt++; }
|
|
if ( $line =~ /^Info:/ ) { $infoCnt++; }
|
|
}
|
|
}
|
|
push @ivpSummary, "The preparation script generated $warningCnt warnings and $infoCnt information messages.";
|
|
|
|
# Pull back the driver script.
|
|
if ( -e "$driverLoc/$driver" ) {
|
|
$out = `mv -f $driverLoc/$driver $driverLoc/$driver.old`;
|
|
$rc = $?;
|
|
if ( $rc != 0 ) {
|
|
logResponse( 'S', 'GENERIC_RESPONSE', "Unable to move $driverLoc/$driver to $driverLoc/$driver.old, rc: $rc" );
|
|
$notify = 1;
|
|
}
|
|
}
|
|
|
|
$cmd = "/usr/bin/scp $openstackUser\@$openstackIP:$tmpDir/$driver $driverLoc/$driver";
|
|
$out = `/usr/bin/scp "$openstackUser\@$openstackIP:$tmpDir/$driver" "$driverLoc/$driver"`;
|
|
$rc = $?;
|
|
if ( $rc != 0 ) {
|
|
$rc = logResponse( 'DFS', 'PREP04', "$driverLoc/$driver", $openstackIP, $cmd, $rc, $out );
|
|
$notify = 1;
|
|
goto FINISH_runPrepScript;
|
|
}
|
|
|
|
$cmd = "chmod 660 $driverLoc/$driver 2>&1";
|
|
$out = `$cmd`;
|
|
$rc = $?;
|
|
if ( $rc != 0 ) {
|
|
logResponse( 'FS', 'PREP07', "$driverLoc/$driver", $cmd, $rc, $out );
|
|
$notify = 1;
|
|
}
|
|
}
|
|
|
|
FINISH_runPrepScript:
|
|
if ( !-s "$driverLoc/$driver" && -s "$driverLoc/$driver.old" ) {
|
|
# New driver does not exist but old driver exists, use the old one.
|
|
$out = `mv -f $driverLoc/$driver.old $driverLoc/$driver`;
|
|
$rc = $?;
|
|
if ( $rc != 0 ) {
|
|
logResponse( 'S', 'GENERIC_RESPONSE', "Unable to move $driverLoc/$driver.old to $driverLoc/$driver, rc: $rc" );
|
|
}
|
|
}
|
|
|
|
if ( -s "$driverLoc/$driver" ) {
|
|
$retRC = 0;
|
|
# Add the driver script to the log file.
|
|
$out = `cat $driverLoc/$driver`;
|
|
logResponse( 'F', '*NONFORMATTED*', '*******************************************************************************' );
|
|
logResponse( 'F', 'GENERIC_RESPONSE_NOINDENT', "The contents of the driver script used for the rest of this IVP follows. ".
|
|
"The driver script is used to run the rest of the IVP on the xCAT Management Node." );
|
|
logResponse( 'F', 'GENERIC_RESPONSE_NOINDENT', 'Note: Any line which set a password related property has been removed.' );
|
|
logResponse( 'F', '*NONFORMATTED*', '*******************************************************************************' );
|
|
$out = `echo "$out" | grep -v "^export zxcatIVP_xcatUserPw"`;
|
|
logResponse( 'F', '*NONFORMATTED*', $out );
|
|
}
|
|
|
|
if ( $warningCnt != 0 ) {
|
|
$notify = 1;
|
|
}
|
|
|
|
if ( $tmpDir ne '' ) {
|
|
$cmd = "rm -Rf $tmpDir 2>&1";
|
|
($rc, $out) = sshToNode( $openstackIP, $openstackUser, "$cmd");
|
|
$rc = $?;
|
|
if ( $rc != 0 ) {
|
|
$rc = logResponse( 'DFS', 'CLNUP01', $openstackIP, $tmpDir, $cmd, $rc, $out );
|
|
$notify = 1;
|
|
}
|
|
}
|
|
return $retRC;
|
|
}
|
|
|
|
|
|
#-------------------------------------------------------
|
|
|
|
=head3 scheduleIVP
|
|
|
|
Description : Schedule an automated IVP.
|
|
Arguments : Global defaults set as command line parameters provide input
|
|
to this routine:
|
|
$comments - Comments for IVP
|
|
$disable - Disable option
|
|
$id - ID to update or 'NEW'
|
|
$openstackIP - IP for OpenStack compute node
|
|
$openstackUser - User for OpenStack compute node
|
|
$orchParms - Parms for this script when IVP run
|
|
$prepParms - Parms for preparation script
|
|
$schedule - Hours to schedule
|
|
$scheduledType - Type of IVP
|
|
$zxcatParms - Parms for zxcatIVP script
|
|
Returns : 0 - No error
|
|
non-zero - Terminating error detected.
|
|
Example : $rc = scheduleIVP();
|
|
|
|
=cut
|
|
|
|
#-------------------------------------------------------
|
|
sub scheduleIVP {
|
|
my $cmd;
|
|
my $out;
|
|
my $rc;
|
|
my $ivpId;
|
|
my %ids;
|
|
my $junk;
|
|
|
|
# Get the list of IVP ids in the zvmivp table.
|
|
$cmd = '/opt/xcat/sbin/tabdump zvmivp | grep -v "^#"';
|
|
$out = `$cmd`;
|
|
$rc = $?;
|
|
if ( $rc ne 0 ) {
|
|
logResponse( 'DF', 'GNRL04', $cmd, $rc, $out );
|
|
goto FINISH_scheduleIVP;
|
|
}
|
|
my @lines = split( '\n', $out );
|
|
foreach my $line ( @lines ) {
|
|
chomp( $line );
|
|
my ($ivpId, $junk) = split( ',', $line, 2 );
|
|
$ivpId =~ s/^\"|\"$//g; # trim quotes from both ends of the string
|
|
$ivpId =~ s/^\s+|\s+$//g; # trim spaces from both ends of the string
|
|
if ( $ivpId ne '' ) {
|
|
$ids{$ivpId} = 1;
|
|
}
|
|
}
|
|
|
|
# Validate the specified id. Either generate a new ID if this is a new
|
|
# request (specified as 'new' or omitted or use the id that was passed.
|
|
if ( $id eq '' or $id eq 'NEW' ) {
|
|
# Generate the new Id
|
|
my $i;
|
|
for ( $i = 10; $i < 10010; $i++ ) {
|
|
if ( ! exists $ids{$i} ) {
|
|
last;
|
|
}
|
|
}
|
|
if ( $i <= 10010 ) {
|
|
$id = $i;
|
|
} else {
|
|
# Could not find an available number in the first 10000 IDs.
|
|
logResponse( 'DF', 'ID01' );
|
|
goto FINISH_scheduleIVP;
|
|
}
|
|
} else {
|
|
# Validate the Id
|
|
if ( ! exists $ids{$id} ) {
|
|
logResponse( 'DF', 'ID02', $id );
|
|
}
|
|
}
|
|
|
|
if ( $scheduledType eq '' ) {
|
|
logResponse( 'DF', 'OPER01', '--type' );
|
|
goto FINISH_scheduleIVP;
|
|
}
|
|
$scheduledType = lc( $scheduledType );
|
|
if ( $scheduledType ne 'basicivp' and $scheduledType ne 'fullivp' ) {
|
|
logResponse( 'DF', 'OPER02', '--type', $scheduledType, '\'basicivp\' or \'fullivp\'' );
|
|
goto FINISH_scheduleIVP;
|
|
}
|
|
if ( $scheduledType eq 'basicivp' ) {
|
|
# Ignore any FULLIVP parms
|
|
$openstackIP = '';
|
|
$openstackUser = '';
|
|
$prepParms = '';
|
|
}
|
|
|
|
# Normalize the schedule.
|
|
my %hours;
|
|
my @parts = split( ' ', $schedule );
|
|
foreach my $hour ( @parts ) {
|
|
$hour =~ s/^\s+|\s+$//g; # trim spaces from both ends of the string
|
|
$hours{$hour} = 1;
|
|
}
|
|
$schedule = '';
|
|
for ( my $hour = 0; $hour <= 23; $hour++ ) {
|
|
if ( exists $hours{$hour} ) {
|
|
$schedule = "$schedule $hour";
|
|
delete $hours{$hour};
|
|
}
|
|
}
|
|
$schedule =~ s/^\s+|\s+$//g; # trim spaces from both ends of the string
|
|
my (@leftOver) = keys %hours;
|
|
if ( scalar @leftOver != 0 ) {
|
|
my $badValues = join( '\' and \'', @leftOver );
|
|
logResponse( 'DF', 'OPER02', '--schedule', $badValues, '0-23' );
|
|
goto FINISH_scheduleIVP;
|
|
}
|
|
|
|
if ( $comments =~ /,/ ) {
|
|
$comments =~ s/,//g; # trim commas from the string
|
|
}
|
|
|
|
my $disableFlag = '';
|
|
if ( $disable == 1 ) {
|
|
$disableFlag = 'YES';
|
|
}
|
|
|
|
# Update the table.
|
|
logResponse( 'DS', '*NONFORMATTED*', "Updating the z/VM IVP table (zvmivp) for $id" );
|
|
my $chtabCmd = "/opt/xcat/sbin/chtab id=\'$id\' ".
|
|
"zvmivp.ip=\'$openstackIP\' ".
|
|
"zvmivp.schedule=\'$schedule\' ".
|
|
"zvmivp.last_run=\'\' ".
|
|
"zvmivp.type_of_run=\'$scheduledType\' ".
|
|
"zvmivp.access_user=\'" . hexEncode( $openstackUser ) . "\' ".
|
|
"zvmivp.orch_parms=\'" . hexEncode( $orchParms ) . "\' ".
|
|
"zvmivp.prep_parms=\'" . hexEncode( $prepParms ) . "\' ".
|
|
"zvmivp.main_ivp_parms=\'" . hexEncode( $zxcatParms ) . "\' ".
|
|
"zvmivp.comments=\'" . hexEncode( $comments ) . "\' ".
|
|
"zvmivp.disable=\'$disableFlag\'";
|
|
$out = `$chtabCmd`;
|
|
$rc = $?;
|
|
if ( $rc != 0 ) {
|
|
logResponse( 'DS', '*NONFORMATTED*', "\'$chtabCmd\' failed for ID($id), rc: $rc, out: $out\n" );
|
|
$rc = 1;
|
|
}
|
|
|
|
FINISH_scheduleIVP:
|
|
return $rc;
|
|
}
|
|
|
|
#-------------------------------------------------------
|
|
|
|
=head3 setupLogFile
|
|
|
|
Description : Set up the log file for this run.
|
|
Arguments : None
|
|
Returns : 0 - No error
|
|
non-zero - Terminating error detected.
|
|
Example : $rc = setupLogFile();
|
|
|
|
=cut
|
|
|
|
#-------------------------------------------------------
|
|
sub setupLogFile {
|
|
my $out;
|
|
my $rc = 0;
|
|
my $ipString = '';
|
|
|
|
# Indicate that the log file will need to be finished.
|
|
# This is a safety in case something goes wrong during
|
|
# the IVP run and causes the program to break.
|
|
$needToFinishLogFile = 1;
|
|
|
|
# Determine the IP address to associate with the log file.
|
|
if ( $openstackIP eq '' ) {
|
|
# Must be doing either a generic zxcatIVP or a full
|
|
# IVP for the compute node sharing the system with this
|
|
# xCAT MN. Use the string "XCAT".
|
|
$ipString = 'XCAT';
|
|
} else {
|
|
if ( exists $localIPs{$openstackIP} ) {
|
|
$ipString = 'XCAT';
|
|
} else {
|
|
$ipString = $openstackIP;
|
|
}
|
|
}
|
|
|
|
# Determine the timestamp to use for the log.
|
|
my ($sec, $min, $hour, $mday, $mon, $year, $wday, $yday, $isdst) = localtime(time);
|
|
$startTime = sprintf("%02d:%02d:%02d on %04d-%02d-%02d",
|
|
$hour, $min, $sec, $year + 1900, $mon + 1, $mday );
|
|
my $currTime = sprintf("%04d-%02d-%02d_%02d:%02d:%02d",
|
|
$year + 1900, $mon + 1, $mday, $hour, $min, $sec);
|
|
|
|
# If the log directory does not exist then create it.
|
|
if ( !-d $logDir ) {
|
|
# Directory did not exist. Create it.
|
|
$out = `mkdir -p $logDir`;
|
|
$rc = $?;
|
|
if ( $rc != 0 ) {
|
|
logResponse( 'DFS', 'GENERIC_RESPONSE', 'Failed to set up the log directory. IVP will run without a log file.' );
|
|
$rc = 0; # Continue processing.
|
|
goto FINISH_setupLogFile;
|
|
}
|
|
}
|
|
|
|
# Create the log file.
|
|
$logFile = "IVP\_$ipString\_$currTime\.log";
|
|
open $logFileHandle, ">", "$logDir/$logFile";
|
|
$rc = $?;
|
|
if ( $rc != 0 ) {
|
|
logResponse( 'DFS', 'GENERIC_RESPONSE', "Failed to open the log file: $logFile. IVP will run without a log file. $!" );
|
|
$rc = 0; # Continue processing.
|
|
goto FINISH_setupLogFile;
|
|
}
|
|
$out = `chmod 644 "$logDir/$logFile"`;
|
|
|
|
push @ivpSummary, "The following logfile was created for this run: $logFile";
|
|
|
|
FINISH_setupLogFile:
|
|
return $rc;
|
|
}
|
|
|
|
|
|
#-------------------------------------------------------
|
|
|
|
=head3 showCILevel
|
|
|
|
Description : Show the cloud-init version installed
|
|
in the node.
|
|
Arguments : None
|
|
Returns : 0 - No error
|
|
non-zero - Terminating error detected.
|
|
Example : $rc = showCILevel();
|
|
|
|
=cut
|
|
|
|
#-------------------------------------------------------
|
|
sub showCILevel {
|
|
my $ciLevel = '';
|
|
my $out = '';
|
|
my $rc = 0;
|
|
|
|
# Attempted to obtain the version using the routine
|
|
# accessible through the $PATH variable.
|
|
($rc, $out) = sshToNode( $nodeName, '', 'cloud-init --version 2>&1');
|
|
if ( $rc == 255 ) {
|
|
logResponse( 'DF', 'GENERIC_RESPONSE', $out );
|
|
goto FINISH_showCILevel;
|
|
} elsif ( $rc == 0 ) {
|
|
if ( $out =~ /^cloud-init / ) {
|
|
($ciLevel) = $out =~ /^cloud-init (.*)/;
|
|
my $msgText = "Version of cloud-init: $ciLevel";
|
|
logResponse( 'DF', 'GENERIC_RESPONSE', $msgText );
|
|
# <todo> Add code to compare the version to what is available in xcat.
|
|
goto FINISH_showCILevel;
|
|
}
|
|
}
|
|
|
|
# Attempt to locate the version in the various cloud-init scripts
|
|
# in the target node.
|
|
# <todo> Add support to look for versions on the node if the invocation
|
|
# using the path did not show the version.
|
|
|
|
FINISH_showCILevel:
|
|
return $rc;
|
|
}
|
|
|
|
|
|
#-------------------------------------------------------
|
|
|
|
=head3 showHelp
|
|
|
|
Description : Show the help inforamtion.
|
|
Arguments : None.
|
|
Returns : None.
|
|
Example : showHelp();
|
|
|
|
=cut
|
|
|
|
#-------------------------------------------------------
|
|
sub showHelp{
|
|
print "$0 verify the capabilities of a node.\n\n";
|
|
print $usage_string;
|
|
return;
|
|
}
|
|
|
|
|
|
#-------------------------------------------------------
|
|
|
|
=head3 sshToNode
|
|
|
|
Description : SSH to a node and issue a command.
|
|
Check for SSH errors.
|
|
Arguments : Node name or IP address/DNS name
|
|
user if we need to specify SUDO
|
|
Command to issue
|
|
Options (char string):
|
|
q - Quiet, Don't generate an error message on failure
|
|
Returns : Return code:
|
|
0 - Normal Linux success
|
|
255 - Unable to SSH to system
|
|
non-zero - command error
|
|
Output from the command or a error string on an SSH failure.
|
|
Example : ($rc, $out) = sshToNode( $nodeName, $user, $command );
|
|
|
|
=cut
|
|
|
|
#-------------------------------------------------------
|
|
sub sshToNode{
|
|
my ( $nodeName, $user, $cmd, $options ) = @_;
|
|
my $rc = 0;
|
|
my $out = '';
|
|
|
|
if ( ! defined $options ) {
|
|
$options = '';
|
|
}
|
|
|
|
if ( $user eq '' ) {
|
|
$out = `ssh $nodeName -qoBatchMode=yes '$cmd'`;
|
|
$rc = $? >> 8;
|
|
} else {
|
|
$out = `ssh $user\@$nodeName -qoBatchMode=yes '$cmd'`;
|
|
$rc = $? >> 8;
|
|
}
|
|
|
|
if ( $rc == 255 and $options !~ /q/ ) {
|
|
logResponse( 'DFS', 'VSTN01', $nodeName ); # Keep $rc = 255.
|
|
$out = "Unable to SSH to $nodeName";
|
|
}
|
|
|
|
return ($rc, $out);
|
|
}
|
|
|
|
|
|
#-------------------------------------------------------
|
|
|
|
=head3 verifyAccess
|
|
|
|
Description : Verify the xCAT MN can access a system.
|
|
Arguments : Node name or IP address
|
|
Returns : 0 - No error
|
|
non-zero - Terminating error detected.
|
|
Example : $rc = verifyAccess();
|
|
|
|
=cut
|
|
|
|
#-------------------------------------------------------
|
|
sub verifyAccess{
|
|
my ( $nodeOrIP ) = @_;
|
|
|
|
my $rc = 0;
|
|
my $out = '';
|
|
|
|
($rc, $out) = sshToNode( $nodeOrIP, '', 'pwd 2>/dev/null' );
|
|
goto FINISH_verifyAccess if ( $rc == 255 ); # Already generated a message
|
|
if ( $rc != 0 ) {
|
|
$rc = logResponse( 'DFS', 'VA01' );
|
|
}
|
|
|
|
FINISH_verifyAccess:
|
|
return $rc;
|
|
}
|
|
|
|
|
|
#-------------------------------------------------------
|
|
|
|
=head3 verifyBasicXCAT
|
|
|
|
Description : Verify the basic setup of xCAT.
|
|
Arguments : None
|
|
Returns : 0 - No error
|
|
non-zero - Terminating error detected.
|
|
Example : $rc = verifyBasicXCAT();
|
|
|
|
=cut
|
|
|
|
#-------------------------------------------------------
|
|
sub verifyBasicXCAT{
|
|
my $infoCnt = 0;
|
|
my @lines;
|
|
my $rc = 0;
|
|
my $warningCnt = 0;
|
|
|
|
$rc = logResponse( 'DF', 'GENERIC_RESPONSE', 'The IVP will be invoked with BYPASS messages suppressed. '.
|
|
'This is because a basic IVP run will produce a number of BYPASS messages due to the fact that '.
|
|
'it is not driven with a full set of configuration operands causing it to avoid some tests. '.
|
|
'Instead you will see an information line indicating that the message was ignored.' );
|
|
$ENV{'zxcatIVP_bypassMsg'} = 0; # No bypass messages
|
|
$ENV{'zxcatIVP_moreCmdOps'} = $zxcatParms;
|
|
my $out = `/opt/xcat/bin/zxcatIVP.pl`;
|
|
|
|
$rc = logResponse( 'DF', '*NONFORMATTED*', $out );
|
|
# Determine how many warnings were generated so that we can produce a summary message upon completion of the IVP.
|
|
@lines = split( '\n', $out );
|
|
foreach my $line ( @lines ) {
|
|
if ( $line =~ /^Warning/ ) { $warningCnt++; }
|
|
if ( $line =~ /^Info/ ) { $infoCnt++; }
|
|
}
|
|
push @ivpSummary, "The zxcatIVP.pl script generated $warningCnt warnings and $infoCnt information messages.";
|
|
|
|
FINISH_verifyBasicXCAT:
|
|
if ( $warningCnt != 0 ) {
|
|
$notify = 1;
|
|
}
|
|
return $rc;
|
|
}
|
|
|
|
|
|
#-------------------------------------------------------
|
|
|
|
=head3 verifyDistro
|
|
|
|
Description : Verify that the distro is supported.
|
|
Arguments : None
|
|
Returns : 0 - No error
|
|
non-zero - Terminating error detected.
|
|
Example : $rc = verifyDistro();
|
|
|
|
=cut
|
|
|
|
#-------------------------------------------------------
|
|
sub verifyDistro{
|
|
my $rc = 0;
|
|
|
|
my $supported = xCAT::zvmUtils->isOSVerSupported( $tgtOS );
|
|
if ( !$supported ) {
|
|
$rc = logResponse( 'DF', 'VD01', ($tgtOS) );
|
|
}
|
|
|
|
return $rc;
|
|
}
|
|
|
|
|
|
#-------------------------------------------------------
|
|
|
|
=head3 verifyLogResponse
|
|
|
|
Description : Script test function to test functioning
|
|
of the logResponse() routine.
|
|
Arguments : None
|
|
Returns : 0 - No error
|
|
non-zero - Terminating error detected.
|
|
Example : $rc = verifyLogResponse();
|
|
|
|
=cut
|
|
|
|
#-------------------------------------------------------
|
|
sub verifyLogResponse{
|
|
my $rc = 0;
|
|
my $sub2 = 'sub2';
|
|
|
|
$rc = logResponse( 'DF', 'VX01' );
|
|
print "rc: $rc\n";
|
|
logResponse( 'DF', 'VPF01', ($nodeName, 0, 'result message') );
|
|
$rc = logResponse( 'DF', 'VX03', $nodeName, $sub2);
|
|
print "rc: $rc\n";
|
|
$rc = logResponse( 'DF', 'VX03', $nodeName, 'sub2a');
|
|
print "rc: $rc\n";
|
|
|
|
return 0;
|
|
}
|
|
|
|
|
|
#-------------------------------------------------------
|
|
|
|
=head3 verifyPower
|
|
|
|
Description : Verify the node is powered on.
|
|
Arguments : None
|
|
Returns : 0 - No error
|
|
non-zero - Terminating error detected.
|
|
Example : $rc = verifyPower();
|
|
|
|
=cut
|
|
|
|
#-------------------------------------------------------
|
|
sub verifyPower {
|
|
my ( $nodeName ) = @_;
|
|
my $rc = 0;
|
|
my $powerState;
|
|
|
|
my $out = `/opt/xcat/bin/rpower $nodeName stat 2>&1`;
|
|
$rc = $?;
|
|
if ( $rc != 0 ) {
|
|
$rc = logResponse( 'DF', 'VPF01', ($nodeName, $rc, $out) );
|
|
goto FINISH_verifyPower if ( $rc > 0 );
|
|
}
|
|
|
|
$out = lc( $out );
|
|
if ( $out =~ /$nodeName: / ) {
|
|
chomp $out;
|
|
($powerState) = $out =~ /$nodeName: (.*)/;
|
|
}
|
|
|
|
if ( $powerState ne 'on' ) {
|
|
$rc = logResponse( 'DFS', 'VP02', ($nodeName) );
|
|
}
|
|
|
|
FINISH_verifyPower:
|
|
return $rc;
|
|
}
|
|
|
|
|
|
#-------------------------------------------------------
|
|
|
|
=head3 verifyService
|
|
|
|
Description : Verify that specified services are running.
|
|
Arguments : Node name or IP address
|
|
Blank delimitted list of service names
|
|
service name begins with the name and
|
|
has a comma separated list of run levels
|
|
which are to be verified as 'on'
|
|
Returns : 0 - No error
|
|
non-zero - Terminating error detected.
|
|
Example : $rc = verifyService();
|
|
|
|
=cut
|
|
|
|
#-------------------------------------------------------
|
|
sub verifyService {
|
|
my ( $nodeOrIP, $namesAndLevels ) = @_;
|
|
my @serviceList = split ( / /, $namesAndLevels );
|
|
|
|
my $rc = 0;
|
|
my $serviceOut = '';
|
|
|
|
# Get the list of service names and run levels into an array.
|
|
|
|
# Get the list of configured services. The output looks similar to:
|
|
# service_name 0:off 1:off 2:off 3:off 4:off 5:off 6:off
|
|
($rc, $serviceOut) = sshToNode( $nodeOrIP, '', "chkconfig --list 2>&1" );
|
|
goto FINISH_verifyService if ( $rc == 255 );
|
|
|
|
# Look for missing services
|
|
my @missingService;
|
|
my @foundService;
|
|
my @nonrunningService;
|
|
foreach my $serviceInfo ( @serviceList ) {
|
|
my @offLevels;
|
|
my $levels;
|
|
my @serviceRunLevels = split( /,/, $serviceInfo );
|
|
my $service = shift @serviceRunLevels;
|
|
($levels) = $serviceOut =~ /^$service(.*)\s(.*)\n/m;
|
|
$levels =~ s/^\s+|\s+$//g if ( defined $levels );
|
|
if (( defined $levels ) and ( $levels ne '' )) {
|
|
# Verify the run levels are enabled.
|
|
foreach my $level ( @serviceRunLevels ) {
|
|
if ( $levels !~ /$level:on/ ) {
|
|
push @offLevels, $level;
|
|
}
|
|
}
|
|
if ( @offLevels ) {
|
|
$levels = join(", ", @offLevels);
|
|
$rc = logResponse( 'DF', 'VS06', $service, $levels );
|
|
goto FINISH_verifyService if ( $rc > 0 );
|
|
} else {
|
|
push @foundService, $service;
|
|
}
|
|
} else {
|
|
push @nonrunningService, $service;
|
|
}
|
|
}
|
|
|
|
if ( @foundService ) {
|
|
my $list = join(", ", @foundService);
|
|
$rc = logResponse( 'DF', 'GENERIC_RESPONSE', "The following services are configured to start: $list" );
|
|
goto FINISH_verifyService if ( $rc > 0 );
|
|
}
|
|
if ( @nonrunningService ) {
|
|
my $list = join(", ", @nonrunningService);
|
|
$rc = logResponse( 'DF', 'VS05', $list );
|
|
goto FINISH_verifyService if ( $rc > 0 );
|
|
}
|
|
|
|
FINISH_verifyService:
|
|
return $rc;
|
|
}
|
|
|
|
|
|
#-------------------------------------------------------
|
|
|
|
=head3 verifyXcatconf4z
|
|
|
|
Description : Verify xcatconf4z is properly installed
|
|
and is the correct version.
|
|
Arguments : Node name or IP address
|
|
Returns : 0 - No error
|
|
non-zero - Terminating error detected.
|
|
Example : $rc = verifyXcatconf4z();
|
|
|
|
=cut
|
|
|
|
#-------------------------------------------------------
|
|
sub verifyXcatconf4z{
|
|
my ( $nodeOrIP ) = @_;
|
|
my $rc = 0;
|
|
my $out;
|
|
my ($mnVersion, $tgtVersion);
|
|
|
|
# Verify service is installed
|
|
$rc = verifyService( $nodeOrIP, 'xcatconf4z,2,3,5' );
|
|
goto FINISH_verifyXcatconf4z if ( $rc == 255 );
|
|
|
|
# Get the xCAT MN's xcatconf4z version level.
|
|
$out = `/opt/xcat/share/xcat/scripts/xcatconf4z version`;
|
|
if ( $out =~ /xcatconf4z version: / ) {
|
|
chomp $out;
|
|
($mnVersion) = $out =~ /xcatconf4z version: (.*)/;
|
|
} else {
|
|
$rc = logResponse( 'DFS', 'VX01' );
|
|
goto FINISH_verifyXcatconf4z if ( $rc > 0 );
|
|
}
|
|
|
|
# <todo> Verify that the node contains xcatconf4z in the correct location and is executable.
|
|
#$out = `ssh $nodeOrIP 'ls -al /opt/xcatconf4z 2>&1'`;
|
|
($rc, $out) = sshToNode( $nodeOrIP, '', 'ls -al /opt/xcatconf4z 2>&1');
|
|
goto FINISH_verifyXcatconf4z if ( $rc == 255 );
|
|
if ( $out =~ /No such file or directory/ ) {
|
|
$rc = logResponse( 'DF', 'VX06' );
|
|
goto FINISH_verifyXcatconf4z if ( $rc > 0 );
|
|
} else {
|
|
# <todo> Verify that it is executable
|
|
}
|
|
|
|
# Get the node's xcatconf4z version level.
|
|
#$out = `ssh $nodeOrIP '/opt/xcatconf4z version'`;
|
|
($rc, $out) = sshToNode( $nodeOrIP, '', '/opt/xcatconf4z version');
|
|
goto FINISH_verifyXcatconf4z if ( $rc == 255 );
|
|
if ( $out =~ /xcatconf4z version: / ) {
|
|
chomp $out;
|
|
($tgtVersion) = $out =~ /xcatconf4z version: (.*)/;
|
|
} else {
|
|
$rc = logResponse( 'DF', 'VX02' );
|
|
goto FINISH_verifyXcatconf4z if ( $rc > 0 );
|
|
}
|
|
|
|
# Verify the version is up to date.
|
|
if ( defined $tgtVersion ) {
|
|
if ( $mnVersion > $tgtVersion ) {
|
|
$rc = logResponse( 'DF', 'VX03', ($tgtVersion, $mnVersion) );
|
|
goto FINISH_verifyXcatconf4z if ( $rc > 0 );
|
|
}
|
|
}
|
|
|
|
# Verify that authorized senders is setup.
|
|
#$out = `ssh $nodeOrIP '/opt/xcatconf4z status'`;
|
|
($rc, $out) = sshToNode( $nodeOrIP, '', '/opt/xcatconf4z status');
|
|
goto verifyXcatconf4z if ( $rc == 255 );
|
|
if ( $out =~ /xcatconf4z is disabled / ) {
|
|
$rc = logResponse( 'DF', 'VX04' );
|
|
goto FINISH_verifyXcatconf4z if ( $rc > 0 );
|
|
} else {
|
|
# Get list of authorized users and put out an info message.
|
|
}
|
|
|
|
# Verify that mkisofs is available
|
|
($rc, $out) = sshToNode( $nodeOrIP, '', 'stat /opt/bin/mkisofs');
|
|
goto verifyXcatconf4z if ( $rc == 255 );
|
|
if ( $rc != 0 ) {
|
|
$rc = logResponse( 'DF', 'VX07' );
|
|
goto FINISH_verifyXcatconf4z if ( $rc > 0 );
|
|
}
|
|
|
|
FINISH_verifyXcatconf4z:
|
|
return $rc;
|
|
}
|
|
|
|
|
|
#*****************************************************************************
|
|
# Main routine
|
|
#*****************************************************************************
|
|
my $ignoreOpt;
|
|
my $rc = 0;
|
|
my $thisScript = $0;
|
|
my $out;
|
|
|
|
# Parse the arguments
|
|
$Getopt::Long::ignorecase = 0;
|
|
Getopt::Long::Configure( "bundling" );
|
|
if (!GetOptions(
|
|
'a|access' => \$verifyAccess,
|
|
'basicivp' => \$runBasicIVP,
|
|
'capturereqs' => \$verifyCaptureReqs,
|
|
'c|cloudinit' => \$verifyCloudInit,
|
|
'comments=s' => \$comments,
|
|
'cron' => \$runCron,
|
|
'decode=s' => \$decode,
|
|
'disable' => \$disable,
|
|
'enable' => \$enable,
|
|
'encode=s' => \$encode,
|
|
'file=s' => \$dataFile,
|
|
'fullivp' => \$runFullIVP,
|
|
'h|help' => \$displayHelp,
|
|
'i|id=s' => \$id,
|
|
'ignore=s' => \$ignoreOpt,
|
|
'cmdoveriucv=s' => \$issueCmdOverIUCV,
|
|
'cmdtonode=s' => \$issueCmdToNode,
|
|
'n|node=s' => \$nodeName,
|
|
'notify' => \$notifyOnErrorOrWarning,
|
|
'openstackuser=s' => \$openstackUser,
|
|
'openstackip=s' => \$openstackIP,
|
|
'orchparms=s' => \$orchParms,
|
|
'prepparms=s' => \$prepParms,
|
|
'remove' => \$remove,
|
|
'schedule=s' => \$schedule,
|
|
'type=s' => \$scheduledType,
|
|
'v|version' => \$versionOpt,
|
|
'x|xcatconf4z' => \$verifyXcatconf4z,
|
|
'zxcatparms=s' => \$zxcatParms,
|
|
)) {
|
|
print $usage_string;
|
|
}
|
|
|
|
if ( $versionOpt ) {
|
|
logResponse( 'DF', 'GENERIC_RESPONSE_NOINDENT', "Version: $version\n" );
|
|
}
|
|
|
|
if ( $displayHelp ) {
|
|
showHelp();
|
|
}
|
|
|
|
if ( $displayHelp or $versionOpt ) {
|
|
goto FINISH_main;
|
|
}
|
|
|
|
# Convert any encoded operand values back to their unencoded value.
|
|
my @convertibleParms = ( 'comments', 'nodeName', 'openstackUser', 'orchParms', 'prepParms', 'zxcatParms' );
|
|
foreach my $parmName ( @convertibleParms ) {
|
|
eval( "\$$parmName = hexDecode( \$$parmName )" );
|
|
}
|
|
|
|
if ( $schedule eq '' and $orchParms ne '' ) {
|
|
# If not scheduling an IVP run and orchestrator parms are present then use them.
|
|
my $unrecognizedOps = '';
|
|
$rc = GetOptionsFromString(
|
|
$orchParms,
|
|
'ignore=s' => \$ignoreOpt,
|
|
);
|
|
if ( $rc == 0 ) {
|
|
print "Unrecognized option in --orchParms. ".
|
|
"Only --ignore is allowed as an orchestrator option for an immediate run.\n";
|
|
$rc = 2;
|
|
goto FINISH_main;
|
|
}
|
|
}
|
|
|
|
# Handle messages to ignore.
|
|
if ( defined( $ignoreOpt ) ) {
|
|
# Make hash from the specified ignore operands
|
|
$ignoreOpt = uc( $ignoreOpt );
|
|
my @ingoreList;
|
|
if ( $ignoreOpt =~ ',' ) {
|
|
@ingoreList = split( ',', $ignoreOpt );
|
|
} else {
|
|
@ingoreList = split( ' ', $ignoreOpt );
|
|
}
|
|
|
|
%msgsToIgnore = map { $_ => 1 } @ingoreList;
|
|
|
|
# Convert general severity type operands to their numeric value.
|
|
if ( $msgsToIgnore{'BYPASS'} ) {
|
|
delete $msgsToIgnore{'BYPASS'};
|
|
$msgsToIgnore{'2'} = 1;
|
|
}
|
|
if ( $msgsToIgnore{'INFO'} ) {
|
|
delete $msgsToIgnore{'INFO'};
|
|
$msgsToIgnore{'3'} = 1;
|
|
}
|
|
if ( $msgsToIgnore{'WARNING'} ) {
|
|
delete $msgsToIgnore{'WARNING'};
|
|
$msgsToIgnore{'4'} = 1;
|
|
}
|
|
if ( $msgsToIgnore{'ERROR'} ) {
|
|
delete $msgsToIgnore{'ERROR'};
|
|
$msgsToIgnore{'5'} = 1;
|
|
}
|
|
}
|
|
|
|
if ( $openstackIP ne '' ) {
|
|
$openstackIP = uc( $openstackIP );
|
|
}
|
|
|
|
# Handle the combinations of disable and enable options.
|
|
if ( $disable eq '' ) {
|
|
$disable = 0;
|
|
}
|
|
if ( $enable eq '' ) {
|
|
$enable = 0;
|
|
}
|
|
if ( $disable == 1 and $enable == 1 ) {
|
|
$rc = logResponse( 'DFS', 'OPER03', '--disable and --enable' );
|
|
goto FINISH_main;
|
|
}
|
|
|
|
if ( $runFullIVP or $runBasicIVP ) {
|
|
# Set up a log file for an IVP run.
|
|
setupLogFile();
|
|
}
|
|
|
|
# Determine the information about the xCAT managed node and the z/VM
|
|
# host environment where it is running (if running on z/VM).
|
|
getMNInfo();
|
|
|
|
if ( $decode ne '' or
|
|
$disable or
|
|
$enable or
|
|
$encode ne '' or
|
|
$runFullIVP or
|
|
$runCron or
|
|
$runBasicIVP or
|
|
$remove or
|
|
$schedule ne '' ) {
|
|
# IVP runs do not need the node.
|
|
} elsif ( $nodeName eq '' ) {
|
|
$rc = logResponse( 'DFS', 'OPER01', '-n or --node' );
|
|
goto FINISH_main;
|
|
}
|
|
|
|
($::SUDOER, $::SUDO) = xCAT::zvmUtils->getSudoer();
|
|
|
|
# Setup for a test run of logResponse. Normally, commented out.
|
|
#($verifyAccess, $verifyCaptureReqs, $verifyCloudInit, $verifyXcatconf4z) = 0;
|
|
#my $test = 1;
|
|
|
|
# Run the selected tests
|
|
foreach my $verify (keys %verifySets) {
|
|
if ( eval $verify ) {
|
|
if ( $verifySets{$verify}[0] ne '' ) {
|
|
logResponse( 'D', 'GENERIC_RESPONSE_NOINDENT', "\n********************************************************************\n" );
|
|
logResponse( 'D', 'GENERIC_RESPONSE_NOINDENT', "$verifySets{$verify}[0]\n" );
|
|
logResponse( 'D', 'GENERIC_RESPONSE_NOINDENT', "********************************************************************\n" );
|
|
}
|
|
my $size = @{$verifySets{$verify}};
|
|
logResponse( 'D', '*RESET*' );
|
|
for (my $i = 1; $i < $size; $i++) {
|
|
($rc) = eval $verifySets{$verify}[$i];
|
|
if ( ! defined $rc ) {
|
|
logResponse( 'DFS', 'PERL01', $thisScript, $verifySets{$verify}[$i], $@ );
|
|
$rc = 666;
|
|
last;
|
|
}
|
|
|
|
if ( $rc > 0 ) {
|
|
last;
|
|
}
|
|
}
|
|
|
|
if ( $warnErrCnt == 0 ) {
|
|
logResponse( 'DF', 'GENERIC_RESPONSE_NOINDENT', "\nProcessing completed." );
|
|
} else {
|
|
logResponse( 'DF', 'GENERIC_RESPONSE_NOINDENT', "\nProcessing completed with warning or error messages. " .
|
|
"See previous messages for information.\n" );
|
|
}
|
|
}
|
|
}
|
|
|
|
FINISH_main:
|
|
if ( $needToFinishLogFile ) {
|
|
# A log file was started. We need to complete it by adding the summary
|
|
# section. If necessary, we will send the log file to the notify user.
|
|
finishLogFile();
|
|
}
|
|
|
|
exit $rc; |