z/VM IVP should tolerate slow xcatd startup.

The IVP expects xcatd to be fully operational by the
time is begins a run.  It is possible for xcatd to
be delayed.  The delay should not ause a corruption
of the zvmnotify property in the site table.
In addition, tests for tabdump failures should be
corrected and the log file should be set up sooner
so that debug information is available for similar
problems.

Change-Id: I2b0e3e875cc28f740a35b348faa0cc74f5acb7bf

perl-xCAT/xCAT/zvmMsgs.pm

@@ -559,6 +559,19 @@ my %verifyMsgs = (
            'userResp'  => 'This function should be obtained from the Linux distribution and '.
                           'installed.'
          },
+    'WAIT01' =>
+         { 'severity'  => 2,
+           'recAction' => 0,
+           'text'      => 'Sleeping %s seconds to allow xcatd to start. Wait %s of %s.',
+           'explain'   => 'The xCAT daemon (xcatd) is not running.',
+           'sysAct'    => 'The process will wait the indicated time and then recheck the status. '.
+                          'If the xCAT daemon does not start after a number of attempts then the '.
+                          'process will stop waiting.  A subsequent attempt to use xCAT daemon '.
+                          'functions is expected to fail and will log the problem.',
+           'userResp'  => 'It may take a short while for the xCAT daemon to start up or resume '.
+                          'after a restart.  A delay should not be considered a problem unless '.
+                          'the xCAT daemon does not start up.'
+         },
     );
 
 #******************************************************************************

xCAT-client/bin/verifynode

@@ -129,10 +129,8 @@ my %verifySets = (
          ],
     '$runBasicIVP' =>
          [ 'Verifying the basic general setup of xCAT',
-           'setupLogFile()',
            'addLogHeader( \'basicivp\' )',
            'verifyBasicXCAT()',
-           'finishLogFile()',
          ],
     '$runCron' =>
          [ 'Cron job for automated verification',
@@ -140,14 +138,12 @@ my %verifySets = (
          ],
     '$runFullIVP' =>
          [ 'Verifying the full end-to-end setup of the xCAT and the compute node',
-           'setupLogFile()',
            'addLogHeader( \'fullivp\' )',
            'checkForDefault( \'openstackIP\', \'--openstackIP was not specified\', \'xCAT_master\' )',
            'verifyAccess( $openstackIP ); $rc = 0',
            'getDistro( $openstackIP )',
            'runPrepScript()',
            'runDriverScript()',
-           'finishLogFile()',
          ],
     '$schedule ne \'\'' =>
          [ 'Scheduling an automated periodic IVP',
@@ -1164,11 +1160,25 @@ sub getMNInfo {
     # Get the list of Local IP addresses
     getLocalIPs();
 
+    # Wait up to 10 minutes for xcatd to come up.
+    my $sleepTime = 60;
+    my $maxWaits = 10;
+    for ( my $i=1; $i<=$maxWaits; $i++ ) {
+        # lsdef uses the xCAT daemon.  We use that to see if it is started.
+        `/opt/xcat/bin/lsdef`;
+        if ( $? == 0 ) {
+            last;
+        } else {
+            logResponse( 'DFS', 'WAIT01', $sleepTime, $i, $maxWaits );
+            sleep( $sleepTime );
+        }
+    }
+
     # Get information related to all of the host nodes
     $cmd = '/opt/xcat/bin/lsdef -w mgt=zvm -w hosttype=zvm -i hcp';
     $out = `$cmd`;
     $rc = $?;
-    if ( $rc eq 0 ) {
+    if ( $rc == 0 ) {
         @lines = split( '\n', $out );
         my $host = '';
         foreach my $line ( @lines ) {
@@ -1188,11 +1198,12 @@ sub getMNInfo {
 
     # Get key info related to the xCAT MN and the notify targets from the site table.
     $mnInfo{'PruneIVP'} = 7;
-    $cmd = "/opt/xcat/sbin/tabdump site | grep -e '^\"master\"' -e '^\"zvmnotify\"' -e '^\"zvmpruneivp\"'";
+    $cmd = "/opt/xcat/sbin/tabdump site";
     $out = `$cmd`;
     $rc = $?;
     if ( $rc == 0 ) {
         @lines = split( '\n', $out );
+        @lines = grep( /^\"master\"|^\"zvmnotify\"|^\"zvmpruneivp\"/, @lines );
         foreach my $line ( @lines ) {
             $line =~ s/^\s+|\s+$//g;         # trim blanks from both ends of the string
             @parts = split( ',', $line );
@@ -2942,6 +2953,11 @@ if ( $disable == 1 and $enable == 1 ) {
     goto FINISH_main;
 }
 
+if ( $runFullIVP or $runBasicIVP ) {
+    # Set up a log file for an IVP run.
+    setupLogFile();
+}
+
 # Determine the information about the xCAT managed node and the z/VM
 # host environment where it is running (if running on z/VM).
 getMNInfo();
@@ -2990,10 +3006,6 @@ foreach my $verify (keys %verifySets) {
             }
         }
 
-        if ( $needToFinishLogFile ) {
-            finishLogFile();
-        }
-
         if ( $warnErrCnt == 0 ) {
             logResponse( 'DF', 'GENERIC_RESPONSE_NOINDENT', "\nProcessing completed." );
         } else {
@@ -3004,4 +3016,10 @@ foreach my $verify (keys %verifySets) {
 }
 
 FINISH_main:
+if ( $needToFinishLogFile ) {
+    # A log file was started.  We need to complete it by adding the summary
+    # section.  If necessary, we will send the log file to the notify user.
+    finishLogFile();
+}
+
 exit $rc;