From 1f1ef56e41320f8470ae328747f00b84ccbdfb04 Mon Sep 17 00:00:00 2001
From: "Steven P. Gessner" <gessners@us.ibm.com>
Date: Thu, 26 Oct 2017 14:33:00 -0400
Subject: [PATCH] z/VM IVP should tolerate slow xcatd startup.

The IVP expects xcatd to be fully operational by the
time is begins a run.  It is possible for xcatd to
be delayed.  The delay should not ause a corruption
of the zvmnotify property in the site table.
In addition, tests for tabdump failures should be
corrected and the log file should be set up sooner
so that debug information is available for similar
problems.

Change-Id: I2b0e3e875cc28f740a35b348faa0cc74f5acb7bf
---
 perl-xCAT/xCAT/zvmMsgs.pm  | 13 +++++++++++++
 xCAT-client/bin/verifynode | 38 ++++++++++++++++++++++++++++----------
 2 files changed, 41 insertions(+), 10 deletions(-)

diff --git a/perl-xCAT/xCAT/zvmMsgs.pm b/perl-xCAT/xCAT/zvmMsgs.pm
index e36c4cfe7..6544f941a 100644
--- a/perl-xCAT/xCAT/zvmMsgs.pm
+++ b/perl-xCAT/xCAT/zvmMsgs.pm
@@ -559,6 +559,19 @@ my %verifyMsgs = (
            'userResp'  => 'This function should be obtained from the Linux distribution and '.
                           'installed.'
          },
+    'WAIT01' =>
+         { 'severity'  => 2,
+           'recAction' => 0,
+           'text'      => 'Sleeping %s seconds to allow xcatd to start. Wait %s of %s.',
+           'explain'   => 'The xCAT daemon (xcatd) is not running.',
+           'sysAct'    => 'The process will wait the indicated time and then recheck the status. '.
+                          'If the xCAT daemon does not start after a number of attempts then the '.
+                          'process will stop waiting.  A subsequent attempt to use xCAT daemon '.
+                          'functions is expected to fail and will log the problem.',
+           'userResp'  => 'It may take a short while for the xCAT daemon to start up or resume '.
+                          'after a restart.  A delay should not be considered a problem unless '.
+                          'the xCAT daemon does not start up.'
+         },
     );
 
 #******************************************************************************
diff --git a/xCAT-client/bin/verifynode b/xCAT-client/bin/verifynode
index 66c29ffdb..b031ba968 100644
--- a/xCAT-client/bin/verifynode
+++ b/xCAT-client/bin/verifynode
@@ -129,10 +129,8 @@ my %verifySets = (
          ],
     '$runBasicIVP' =>
          [ 'Verifying the basic general setup of xCAT',
-           'setupLogFile()',
            'addLogHeader( \'basicivp\' )',
            'verifyBasicXCAT()',
-           'finishLogFile()',
          ],
     '$runCron' =>
          [ 'Cron job for automated verification',
@@ -140,14 +138,12 @@ my %verifySets = (
          ],
     '$runFullIVP' =>
          [ 'Verifying the full end-to-end setup of the xCAT and the compute node',
-           'setupLogFile()',
            'addLogHeader( \'fullivp\' )',
            'checkForDefault( \'openstackIP\', \'--openstackIP was not specified\', \'xCAT_master\' )',
            'verifyAccess( $openstackIP ); $rc = 0',
            'getDistro( $openstackIP )',
            'runPrepScript()',
            'runDriverScript()',
-           'finishLogFile()',
          ],
     '$schedule ne \'\'' =>
          [ 'Scheduling an automated periodic IVP',
@@ -1164,11 +1160,25 @@ sub getMNInfo {
     # Get the list of Local IP addresses
     getLocalIPs();
 
+    # Wait up to 10 minutes for xcatd to come up.
+    my $sleepTime = 60;
+    my $maxWaits = 10;
+    for ( my $i=1; $i<=$maxWaits; $i++ ) {
+        # lsdef uses the xCAT daemon.  We use that to see if it is started.
+        `/opt/xcat/bin/lsdef`;
+        if ( $? == 0 ) {
+            last;
+        } else {
+            logResponse( 'DFS', 'WAIT01', $sleepTime, $i, $maxWaits );
+            sleep( $sleepTime );
+        }
+    }
+
     # Get information related to all of the host nodes
     $cmd = '/opt/xcat/bin/lsdef -w mgt=zvm -w hosttype=zvm -i hcp';
     $out = `$cmd`;
     $rc = $?;
-    if ( $rc eq 0 ) {
+    if ( $rc == 0 ) {
         @lines = split( '\n', $out );
         my $host = '';
         foreach my $line ( @lines ) {
@@ -1188,11 +1198,12 @@ sub getMNInfo {
 
     # Get key info related to the xCAT MN and the notify targets from the site table.
     $mnInfo{'PruneIVP'} = 7;
-    $cmd = "/opt/xcat/sbin/tabdump site | grep -e '^\"master\"' -e '^\"zvmnotify\"' -e '^\"zvmpruneivp\"'";
+    $cmd = "/opt/xcat/sbin/tabdump site";
     $out = `$cmd`;
     $rc = $?;
     if ( $rc == 0 ) {
         @lines = split( '\n', $out );
+        @lines = grep( /^\"master\"|^\"zvmnotify\"|^\"zvmpruneivp\"/, @lines );
         foreach my $line ( @lines ) {
             $line =~ s/^\s+|\s+$//g;         # trim blanks from both ends of the string
             @parts = split( ',', $line );
@@ -2942,6 +2953,11 @@ if ( $disable == 1 and $enable == 1 ) {
     goto FINISH_main;
 }
 
+if ( $runFullIVP or $runBasicIVP ) {
+    # Set up a log file for an IVP run.
+    setupLogFile();
+}
+
 # Determine the information about the xCAT managed node and the z/VM
 # host environment where it is running (if running on z/VM).
 getMNInfo();
@@ -2990,10 +3006,6 @@ foreach my $verify (keys %verifySets) {
             }
         }
 
-        if ( $needToFinishLogFile ) {
-            finishLogFile();
-        }
-
         if ( $warnErrCnt == 0 ) {
             logResponse( 'DF', 'GENERIC_RESPONSE_NOINDENT', "\nProcessing completed." );
         } else {
@@ -3004,4 +3016,10 @@ foreach my $verify (keys %verifySets) {
 }
 
 FINISH_main:
+if ( $needToFinishLogFile ) {
+    # A log file was started.  We need to complete it by adding the summary
+    # section.  If necessary, we will send the log file to the notify user.
+    finishLogFile();
+}
+
 exit $rc;
\ No newline at end of file