xcat-core/xCAT-server/lib/xcat/plugins/yaboot.pm

# IBM(c) 2007 EPL license http://www.eclipse.org/legal/epl-v10.html
package xCAT_plugin::yaboot;
use Data::Dumper;
use Sys::Syslog;
use xCAT::Scope;
use xCAT::Utils;
use xCAT::TableUtils;
use xCAT::ServiceNodeUtils;
use xCAT::NetworkUtils;
use xCAT::MsgUtils;
use File::Path;
use Socket;
use Getopt::Long;
use xCAT::Table;

my $request;
my %breaknetbootnodes;
our %normalnodes;
my $callback;
my $sub_req;
my $dhcpconf = "/etc/dhcpd.conf";
my $globaltftpdir = xCAT::TableUtils->getTftpDir();
#my $dhcpver = 3;

my %usage = (
    "nodeset" => "Usage: nodeset <noderange> [install|shell|boot|runcmd=bmcsetup|netboot|iscsiboot|osimage[=<imagename>]|offline]",
);
sub handled_commands {
  return {
    nodeset => "noderes:netboot"
  }
}

sub check_dhcp {
  return 1;
  #TODO: omapi magic to do things right
  my $node = shift;
  my $dhcpfile;
  open ($dhcpfile,$dhcpconf);
  while (<$dhcpfile>) {
    if (/host $node\b/) {
      close $dhcpfile;
      return 1;
    }
  }
  close $dhcpfile;
  return 0;
}

sub _slow_get_tftpdir { #make up for paths where tftpdir is not passed in
    my $node=shift;
    my $nrtab = xCAT::Table->new('noderes',-create=>0); #in order to detect per-node tftp directories
    unless ($nrtab) { return $globaltftpdir; }
    my $ent = $nrtab->getNodeAttribs($node,["tftpdir"]);
    if ($ent and $ent->{tftpdir}) {
	return $ent->{tftpdir};
    } else {
        return $globaltftpdir;
    }
}
sub getstate {
  my $node = shift;
  my $tftpdir = shift;
  unless ($tftpdir) { $tftpdir = _slow_get_tftpdir($node); }
  if (check_dhcp($node)) {
    if (-r $tftpdir . "/etc/".$node) {
      my $fhand;
      open ($fhand,$tftpdir . "/etc/".$node);
      my $headline = <$fhand>;
      close $fhand;
      $headline =~ s/^#//;
      chomp($headline);
      return $headline;
    } else {
      return "boot";
    }
  } else {
    return "discover";
  }
}

sub setstate {
=pod

  This function will manipulate the yaboot structure to match what the noderes/chain tables indicate the node should be booting.

=cut
  my $node = shift;
  my %bphash = %{shift()};
  my %chainhash = %{shift()};
  my %machash = %{shift()};
  my $tftpdir = shift;
  my %nrhash = %{shift()};
  my $linuximghash = shift();
  my $kern = $bphash{$node}->[0]; #$bptab->getNodeAttribs($node,['kernel','initrd','kcmdline']);
  if ($kern->{kcmdline} =~ /!myipfn!/) {
      my $ipfn = xCAT::NetworkUtils->my_ip_facing($node);
      unless ($ipfn) {
          my $servicenodes = $nrhash{$node}->[0];
          if ($servicenodes and $servicenodes->{servicenode}) {
              my @sns = split /,/, $servicenodes->{servicenode};
              foreach my $sn ( @sns ) {
                  # We are in the service node pools, print error if no facing ip.
                  if (xCAT::InstUtils->is_me($sn)) {
                      my @myself = xCAT::NetworkUtils->determinehostname();
                      my $myname = $myself[(scalar @myself)-1];
                      $::callback->(
                          {
                          error => [
                          "$myname: Unable to determine the image server for $node on service node $sn"
                          ],
                          errorcode => [1]
                          }
                      );
                      return;
                  }
              }
          } else {
              $::callback->(
                          {
                          error => [
                          "$myname: Unable to determine the image server for $node"
                          ],
                          errorcode => [1]
                          }
                      );
              return;
          }
      } else {
          $kern->{kcmdline} =~ s/!myipfn!/$ipfn/g;
      }
  }
  if ($kern->{addkcmdline}) {
      $kern->{kcmdline} .= " ".$kern->{addkcmdline};
  }
  
  if($linuximghash and $linuximghash->{'addkcmdline'})
  {
      unless($linuximghash->{'boottarget'}) 
      {
          $kern->{kcmdline} .= " ".$linuximghash->{'addkcmdline'};
      } 
  }
   

  my $pcfg;
  unless (-d "$tftpdir/etc") {
     mkpath("$tftpdir/etc");
  }
  my $nodemac;
  my %client_nethash = xCAT::DBobjUtils->getNetwkInfo( [$node] );
  if ( $client_nethash{$node}{mgtifname} =~ /hf/ ) {
    my $mactab = xCAT::Table->new('mac');
    if ($mactab) {
      my $ment = $machash{$node}->[0]; #$mactab->getNodeAttribs($node,['mac']);
      if ($ment and $ment->{mac}) {
        my @macs = split(/\|/,$ment->{mac});
        my $count = 0;
        foreach my $mac (@macs) {
          if ( $mac !~ /!(.*)/) {
            my $hostname;
            if ( $node !~ /^(.*)-hf(.*)$/ ) {
                $hostname = $node . "-hf" . $count;
            } else {
                $hostname = $1 . "-hf" . $count;
            }
            open($pcfg,'>',$tftpdir."/etc/".$hostname);
            my $cref=$chainhash{$node}->[0]; #$chaintab->getNodeAttribs($node,['currstate']);
            if ($cref->{currstate}) {
              print $pcfg "#".$cref->{currstate}."\n";
            }

            print $pcfg "timeout=5\n";
            $normalnodes{$node}=1;
            if ($cref and $cref->{currstate} eq "boot") {
              $breaknetbootnodes{$node}=1;
              delete $normalnodes{$node}; #Signify to omit this from one makedhcp command
              #$sub_req->({command=>['makedhcp'], #batched elsewhere, this code is stale, hopefully
              #       node=>[$node],
              #        arg=>['-s','filename = \"xcat/nonexistant_file_to_intentionally_break_netboot_for_localboot_to_work\";']},$callback);
              print $pcfg "bye\n";
              close($pcfg);
            } elsif ($kern and $kern->{kernel}) {
              #It's time to set yaboot for this node to boot the kernel..
              print $pcfg "image=".$kern->{kernel}."\n\tlabel=xcat\n";
              if ($kern and $kern->{initrd}) {
                print $pcfg "\tinitrd=".$kern->{initrd}."\n";
              }
              if ($kern and $kern->{kcmdline}) {
                my $kcmdline = $kern->{kcmdline};
                $kcmdline =~ s/(.*ifname=.*):@macs[0].*( netdev.*)/$1:$mac$2/g;
                print $pcfg "\tappend=\"".$kcmdline."\"\n";
              }
              close($pcfg);
              my $inetn = xCAT::NetworkUtils->getipaddr($node);
              unless ($inetn) {
               syslog("local1|err","xCAT unable to resolve IP for $node in yaboot plugin");
               return;
              }
            } else { #TODO: actually, should possibly default to xCAT image?
              print $pcfg "bye\n";
              close($pcfg);
            }

            if ($mac =~ /:/) {
              $nodemac = $mac;
              my $tmp = $mac;
              $tmp =~ s/(..):(..):(..):(..):(..):(..)/$1-$2-$3-$4-$5-$6/g;
              my $pname = "25-" . $tmp;
              unlink($tftpdir."/etc/".$pname);
              link($tftpdir."/etc/".$hostname,$tftpdir."/etc/".$pname);
            }
          }
          $count = $count + 2;
        }
      }
    }


  } else {

    open($pcfg,'>',$tftpdir."/etc/".$node);
    my $cref=$chainhash{$node}->[0]; #$chaintab->getNodeAttribs($node,['currstate']);
    if ($cref->{currstate}) {
      print $pcfg "#".$cref->{currstate}."\n";
    }
    print $pcfg "timeout=5\n";
    $normalnodes{$node}=1; #Assume a normal netboot (well, normal dhcp, 
                        #which is normally with a valid 'filename' field,
                        #but the typical ppc case will be 'special' makedhcp
                        #to clear the filename field, so the logic is a little
                        #opposite
    #  $sub_req->({command=>['makedhcp'], #This is currently batched elswhere
    #         node=>[$node]},$callback);  #It hopefully will perform correctly
    if ($cref and $cref->{currstate} eq "boot") {
      $breaknetbootnodes{$node}=1;
      delete $normalnodes{$node}; #Signify to omit this from one makedhcp command
      #$sub_req->({command=>['makedhcp'], #batched elsewhere, this code is stale, hopefully
      #       node=>[$node],
      #        arg=>['-s','filename = \"xcat/nonexistant_file_to_intentionally_break_netboot_for_localboot_to_work\";']},$callback);
      print $pcfg "bye\n";
      close($pcfg);
    } elsif ($kern and $kern->{kernel}) {
      #It's time to set yaboot for this node to boot the kernel..
      print $pcfg "image=".$kern->{kernel}."\n\tlabel=xcat\n";
      if ($kern and $kern->{initrd}) {
        print $pcfg "\tinitrd=".$kern->{initrd}."\n";
      }
      if ($kern and $kern->{kcmdline}) {
        print $pcfg "\tappend=\"".$kern->{kcmdline}."\"\n";
      }
      close($pcfg);
      my $inetn = xCAT::NetworkUtils->getipaddr($node);
      unless ($inetn) {
       syslog("local1|err","xCAT unable to resolve IP for $node in yaboot plugin");
       return;
      }
    } else { #TODO: actually, should possibly default to xCAT image?
      print $pcfg "bye\n";
      close($pcfg);
    }
    my $ip = xCAT::NetworkUtils->getipaddr($node);
    unless ($ip) {
      syslog("local1|err","xCAT unable to resolve IP in yaboot plugin");
      return;
    }
    my $mactab = xCAT::Table->new('mac');
    my %ipaddrs;
    $ipaddrs{$ip} = 1;
    if ($mactab) {
      my $ment = $machash{$node}->[0]; #$mactab->getNodeAttribs($node,['mac']);
      if ($ment and $ment->{mac}) {
        my @macs = split(/\|/,$ment->{mac});
        foreach (@macs) {
           $nodemac = $_;
           if (/!(.*)/) {
              my $ipaddr = xCAT::NetworkUtils->getipaddr($1);
              if ($ipaddr) {
               $ipaddrs{$ipaddr} = 1;
              }
           }
        }
      }
    }
    # Do not use symbolic link, p5 does not support symbolic link in /tftpboot
    #  my $hassymlink = eval { symlink("",""); 1 };
    foreach $ip (keys %ipaddrs) {
      my @ipa=split(/\./,$ip);
      my $pname = sprintf("%02x%02x%02x%02x",@ipa);
      unlink($tftpdir."/etc/".$pname);
      link($tftpdir."/etc/".$node,$tftpdir."/etc/".$pname);
    }
  }
  if ($nodemac =~ /:/) {
      my $tmp = $nodemac;
      $tmp =~ s/(..):(..):(..):(..):(..):(..)/$1-$2-$3-$4-$5-$6/g;
      my $pname = "yaboot.conf-" . $tmp;
      unlink($tftpdir."/".$pname);
      link($tftpdir."/etc/".$node,$tftpdir."/".$pname); 
  }
  return;      
}
  

    
my $errored = 0;
sub pass_along { 
    my $resp = shift;

#    print Dumper($resp);
    
    $callback->($resp);
    if ($resp and ($resp->{errorcode} and $resp->{errorcode}->[0]) or ($resp->{error} and $resp->{error}->[0])) {
        $errored=1;
    }
    foreach (@{$resp->{node}}) {
       if ($_->{error} or $_->{errorcode}) {
          $errored=1;
       }
    }
}

  
sub preprocess_request {
    my $req = shift;
    if ($req->{_xcatpreprocessed}->[0] == 1) { return [$req]; }

    my $callback1 = shift;
    my $command  = $req->{command}->[0];
    my $sub_req = shift;
    my @args=();
    if (ref($req->{arg})) {
	@args=@{$req->{arg}};
    } else {
	@args=($req->{arg});
    }
    @ARGV = @args;
    my $nodes = $req->{node};
    #use Getopt::Long;
    Getopt::Long::Configure("bundling");
    Getopt::Long::Configure("pass_through");
    if (!GetOptions('h|?|help'  => \$HELP, 'v|version' => \$VERSION) ) {
      if($usage{$command}) {
          my %rsp;
          $rsp{data}->[0]=$usage{$command};
          $callback1->(\%rsp);
      }
      return;
    }

    if ($HELP) { 
	if($usage{$command}) {
	    my %rsp;
	    $rsp{data}->[0]=$usage{$command};
	    $callback1->(\%rsp);
	}
	return;
    }

    if ($VERSION) {
	my $ver = xCAT::Utils->Version();
	my %rsp;
	$rsp{data}->[0]="$ver";
	$callback1->(\%rsp);
	return; 
    }

    if (@ARGV==0) {
	if($usage{$command}) {
	    my %rsp;
	    $rsp{data}->[0]=$usage{$command};
	    $callback1->(\%rsp);
	}
	return;
    }


   #Assume shared tftp directory for boring people, but for cool people, help sync up tftpdirectory contents when 
   #if they specify no sharedtftp in site table
   my @entries =  xCAT::TableUtils->get_site_attribute("sharedtftp");
   my $t_entry = $entries[0];
   if ( defined($t_entry)  and ($t_entry == 0 or $t_entry =~ /no/i)) {
      # check for  computenodes and servicenodes from the noderange, if so error out
       my @SN;
       my @CN;
       xCAT::ServiceNodeUtils->getSNandCPnodes(\@$nodes, \@SN, \@CN);
       if ((@SN > 0) && (@CN >0 )) { # there are both SN and CN
            my $rsp;
            $rsp->{data}->[0] = 
              "Nodeset was run with a noderange containing both service nodes and compute nodes. This is not valid. You must submit with either compute nodes in the noderange or service nodes. \n";
            xCAT::MsgUtils->message("E", $rsp, $callback1);       
            return; 
           
       } 

      $req->{'_disparatetftp'}=[1];
      if ($req->{inittime}->[0]) {
          return [$req];
      }
      if (@CN >0 ) { # if compute nodes broadcast to all servicenodes
       return xCAT::Scope->get_broadcast_scope($req,@_);
      }
   }
   return [$req];
}


sub process_request {
  $request = shift;
  $callback = shift;
  $::callback=$callback;
  $sub_req = shift;
  my $command  = $request->{command}->[0];
  %breaknetbootnodes=();
  %normalnodes=();

  my @args;
  my @nodes;
  my @rnodes;
  if (ref($request->{node})) {
    @rnodes = @{$request->{node}};
  } else {
    if ($request->{node}) { @rnodes = ($request->{node}); }
  }
  unless (@rnodes) {
      if ($usage{$request->{command}->[0]}) {
          $callback->({data=>$usage{$request->{command}->[0]}});
      }
      return;
  }

  #if not shared tftpdir, then filter, otherwise, set up everything
  if ($request->{'_disparatetftp'}->[0]) { #reading hint from preprocess_command
   @nodes = ();
   foreach (@rnodes) {
     if (xCAT::NetworkUtils->nodeonmynet($_)) {
        push @nodes,$_;
     } else {
        xCAT::MsgUtils->message("S", "$_: yaboot netboot: stop configuration because of none sharedtftp and not on same network with its xcatmaster.");
     }
   }
  } else {
     @nodes = @rnodes;
  }

  # return directly if no nodes in the same network
  unless (@nodes) {
     xCAT::MsgUtils->message("S", "xCAT: yaboot netboot: no valid nodes. Stop the operation on this server.");
     return;
  }

  if (ref($request->{arg})) {
    @args=@{$request->{arg}};
  } else {
    @args=($request->{arg});
  }
  
  #now run the begin part of the prescripts
  unless ($args[0] eq 'stat') { # or $args[0] eq 'enact') {
      $errored=0;
      if ($request->{'_disparatetftp'}->[0]) {  #the call is distrubuted to the service node already, so only need to handles my own children
	  $sub_req->({command=>['runbeginpre'],
		      node=>\@nodes,
		      arg=>[$args[0], '-l']},\&pass_along);
      } else { #nodeset did not distribute to the service node, here we need to let runednpre to distribute the nodes to their masters
	  $sub_req->({command=>['runbeginpre'],   
		      node=>\@rnodes,
		      arg=>[$args[0]]},\&pass_along);
      }
      if ($errored) {
	  my $rsp;
 	  $rsp->{errorcode}->[0]=1;
	  $rsp->{error}->[0]="Failed in running begin prescripts.\n";
	  $callback->($rsp);
	  return; 
      }
  } 

  #back to normal business
  my $inittime=0;
  if (exists($request->{inittime})) { $inittime= $request->{inittime}->[0];}
  if (!$inittime) { $inittime=0;}
  $errored=0;
  unless ($args[0] eq 'stat') { # or $args[0] eq 'enact') {
    $sub_req->({command=>['setdestiny'],
		node=>\@nodes,
		inittime=>[$inittime],
		arg=>\@args},\&pass_along);
  }
  if ($errored) { return; }

  my $bptab=xCAT::Table->new('bootparams',-create=>1);
  my $bphash = $bptab->getNodesAttribs(\@nodes,['kernel','initrd','kcmdline','addkcmdline']);
  my $chaintab=xCAT::Table->new('chain',-create=>1);
  my $chainhash=$chaintab->getNodesAttribs(\@nodes,['currstate']);
  my $noderestab=xCAT::Table->new('noderes',-create=>1);
  my $nodereshash=$noderestab->getNodesAttribs(\@nodes,['tftpdir']);
  my $mactab=xCAT::Table->new('mac',-create=>1);
  my $machash=$mactab->getNodesAttribs(\@nodes,['mac']);
  my $nrtab=xCAT::Table->new('noderes',-create=>1);
  my $nrhash=$nrtab->getNodesAttribs(\@nodes,['servicenode']);
  my $typetab=xCAT::Table->new('nodetype',-create=>1);
  my $typehash=$typetab->getNodesAttribs(\@nodes,['os','provmethod','arch','profile']);
  my $linuximgtab=xCAT::Table->new('linuximage',-create=>1);
  my $osimagetab=xCAT::Table->new('osimage',-create=>1);

  my $rc;
  my $errstr;

  my $tftpdir;
  foreach (@nodes) {
    my %response;
    if ($nodereshash->{$_} and $nodereshash->{$_}->[0] and $nodereshash->{$_}->[0]->{tftpdir}) {
       $tftpdir =  $nodereshash->{$_}->[0]->{tftpdir};
    } else {
       $tftpdir = $globaltftpdir;
    }
    $response{node}->[0]->{name}->[0]=$_;
    if ($args[0] eq 'stat') {
      $response{node}->[0]->{data}->[0]= getstate($_,$tftpdir);
      $callback->(\%response);
    } elsif ($args[0]) { #If anything else, send it on to the destiny plugin, then setstate
      my $ent = $typehash->{$_}->[0]; 
      my $osimgname = $ent->{'provmethod'};
      my $linuximghash=undef;
      unless($osimgname =~ /^(install|netboot|statelite)$/){
        $linuximghash = $linuximgtab->getAttribs({imagename => $osimgname}, 'boottarget', 'addkcmdline');
      }      

      ($rc,$errstr) = setstate($_,$bphash,$chainhash,$machash,$tftpdir,$nrhash,$linuximghash);
      if ($rc) {
        $response{node}->[0]->{errorcode}->[0]= $rc;
        $response{node}->[0]->{errorc}->[0]= $errstr;
        $callback->(\%response);
      }
    }
  }# end of foreach node    

  my @normalnodeset = keys %normalnodes;
  my @breaknetboot=keys %breaknetbootnodes;
  #print "yaboot:inittime=$inittime; normalnodeset=@normalnodeset; breaknetboot=@breaknetboot\n";
    my %osimagenodehash;
    for my $nn (@normalnodeset){
        #record the os version for node
        my $ent = $typehash->{$nn}->[0];
        my $osimage=$ent->{'provmethod'};
        if($osimage =~ /^(install|netboot|statelite)$/){
           $osimage=($ent->{'os'}).'-'.($ent->{'arch'}).'-'.($ent->{'provmethod'}).'-'.($ent->{'profile'});
        }
        push @{$osimagenodehash{$osimage}}, $nn;
        
    }

    foreach my $osimage (keys %osimagenodehash) {
        my $osimgent = $osimagetab->getAttribs({imagename => $osimage },'osvers');       
        my $os = $osimgent->{'osvers'};    

        my $osv;
        my $osn;
        my $osm;
        if ($os =~ /(\D+)(\d+)\.(\d+)/) {
            $osv = $1;
            $osn = $2;
            $osm = $3;
        
        } elsif ($os =~ /(\D+)(\d+)/){
            $osv = $1;
            $osn = $2;
            $osm = 0;   
        }
    
        if (($osv =~ /rh/ and int($osn) < 6) or 
            ($osv =~ /sles/ and int($osn) < 11)) {
            # check if yaboot-xcat installed
            my $yf = $tftpdir . "/yaboot";
            unless (-e $yf) {
                my $rsp;
                push @{$rsp->{data}},
                  "stop configuration because yaboot-xcat need to be installed for $os.\n";
                xCAT::MsgUtils->message("E", $rsp, $callback);       
                return; 
              }
        } elsif (($osv =~ /rh/ and int($osn) >= 6) or
                 ($osv =~ /sles/ and int($osn) >= 11)) {
            # copy yaboot from cn's repository
            my $cmd = '/usr/bin/rsync';
            if (!-f $cmd || !-x $cmd) {  
                my $rsp;
                push @{$rsp->{data}},
                  "stop configuration because rsync does not exist or is not executable.\n";
                xCAT::MsgUtils->message("E", $rsp, $callback);            
                return;
           }      
            my $yabootpath = $tftpdir."/yb/".$os;
            mkpath $yabootpath;     

            my $linuximgent = $linuximgtab->getAttribs({imagename => $osimage},'pkgdir');
            my @pkgdirlist = split  /,/, $linuximgent->{'pkgdir'};
            my $pkgdir = $pkgdirlist[0];
            $pkgdir =~ s/\/+$//;            
                   

            my $yabootfile;   
            if ($os =~ /sles/) {
                $yabootfile = $pkgdir."/1/suseboot/yaboot";
            } elsif ($os =~ /rh/){
                $yabootfile = $pkgdir."/ppc/chrp/yaboot";
            }  
            unless (-e "$yabootfile") {
                my $rsp;
                push @{$rsp->{data}},
                  "stop configuration because Unable to find the os shipped yaboot file.\n";
                xCAT::MsgUtils->message("E", $rsp, $callback);
                return; 
              }          

            $cmd = $cmd." ".$yabootfile." ".$yabootpath; #???
            ($rc,$errstr) = xCAT::Utils->runcmd($cmd, 0);
            if ($rc)
            {
                my $rsp;
                push @{$rsp->{data}},
                  "stop configuration because $synccmd failed.\n";
                xCAT::MsgUtils->message("E", $rsp, $callback);
               return; 
            } 
        }
    } #end of foreach osimagenodehash
  
  #Don't bother to try dhcp binding changes if sub_req not passed, i.e. service node build time
  unless (($args[0] eq 'stat') || ($inittime) || ($args[0] eq 'offline')) {
      #dhcp stuff
      my $do_dhcpsetup=1;
      #my $sitetab = xCAT::Table->new('site');
      #if ($sitetab) {
          #(my $ref) = $sitetab->getAttribs({key => 'dhcpsetup'}, 'value');
          my @entries =  xCAT::TableUtils->get_site_attribute("dhcpsetup");
          my $t_entry = $entries[0];
          if (defined($t_entry) ) {
             if ($t_entry =~ /0|n|N/) { $do_dhcpsetup=0; }
          }
      #}

      if ($do_dhcpsetup) {
        if (%osimagenodehash) {
            foreach my $osimage (keys %osimagenodehash) {
                my $osimgent = $osimagetab->getAttribs({imagename => $osimage },'osvers');
                my $osentry = $osimgent->{'osvers'};

                my $osv;
                my $osn;
                my $osm;
                if ($osentry =~ /(\D+)(\d+)\.(\d+)/) {
                    $osv = $1;
                    $osn = $2;
                    $osm = $3;
                
                } elsif ($osentry =~ /(\D+)(\d+)/){
                    $osv = $1;
                    $osn = $2;
                    $osm = 0;   
                }
                if (($osv =~ /rh/ and int($osn) >= 6) or 
                    ($osv =~ /sles/ and int($osn) >= 11)) {
                    my $fpath = "/yb/". $osentry."/yaboot"; 
                    if ($request->{'_disparatetftp'}->[0]) { #reading hint from preprocess_command
                    $sub_req->({command=>['makedhcp'],
                         node=>\@{$osimagenodehash{$osimage}},
                         arg=>['-l','-s','filename = \"'.$fpath.'\";']},$callback);
                    } else {
                    $sub_req->({command=>['makedhcp'],
                         node=>\@{$osimagenodehash{$osimage}},
                         arg=>['-s','filename = \"'.$fpath.'\";']},$callback);
                    }
                } else {
                    if ($request->{'_disparatetftp'}->[0]) { #reading hint from preprocess_command, only change local settings if already farmed
                    $sub_req->({command=>['makedhcp'],arg=>['-l'],
                           node=>\@{$osimagenodehash{$osimage}}},$callback);
                    } else {
                    $sub_req->({command=>['makedhcp'],
                         node=>\@{$osimagenodehash{$osimage}}},$callback);
                    }
                }
            }
        } else {
            if ($request->{'_disparatetftp'}->[0]) { #reading hint from preprocess_command, only change local settings if already farmed
            $sub_req->({command=>['makedhcp'],arg=>['-l'],
                   node=>\@normalnodeset},$callback);
            } else {
            $sub_req->({command=>['makedhcp'],
                 node=>\@normalnodeset},$callback);
            }
        }
        if ($request->{'_disparatetftp'}->[0]) { #reading hint from preprocess_command
            $sub_req->({command=>['makedhcp'],
             node=>\@breaknetboot,
             arg=>['-l','-s','filename = \"xcat/nonexistant_file_to_intentionally_break_netboot_for_localboot_to_work\";']},$callback);
        } else {
            $sub_req->({command=>['makedhcp'],
             node=>\@breaknetboot,
             arg=>['-s','filename = \"xcat/nonexistant_file_to_intentionally_break_netboot_for_localboot_to_work\";']},$callback);
        }
     }
  }
  
  #now run the end part of the prescripts
  unless ($args[0] eq 'stat') { # or $args[0] eq 'enact') 
      $errored=0;
      if ($request->{'_disparatetftp'}->[0]) {  #the call is distrubuted to the service node already, so only need to handles my own children
	  $sub_req->({command=>['runendpre'],
		      node=>\@nodes,
		      arg=>[$args[0], '-l']},\&pass_along);
      } else { #nodeset did not distribute to the service node, here we need to let runednpre to distribute the nodes to their masters
	  $sub_req->({command=>['runendpre'],   
		      node=>\@rnodes,
		      arg=>[$args[0]]},\&pass_along);
      }
      if ($errored) { 
	  my $rsp;
	  $rsp->{errorcode}->[0]=1;
	  $rsp->{error}->[0]="Failed in running end prescripts\n";
	  $callback->($rsp);
	  return; 
      }
  }
}

#----------------------------------------------------------------------------
=head3  getNodesetStates
       returns the nodeset state for the given nodes. The possible nodeset
           states are: netboot, install, boot and discover.
    Arguments:
        nodes  --- a pointer to an array of nodes
        states -- a pointer to a hash table. This hash will be filled by this
             function.The key is the nodeset status and the value is a pointer
             to an array of nodes.  
    Returns:
       (return code, error message)
=cut
#-----------------------------------------------------------------------------
sub getNodesetStates {
  my $noderef=shift;
  if ($noderef =~ /xCAT_plugin::yaboot/) {
    $noderef=shift;
  }
  my @nodes=@$noderef;
  my $hashref=shift; 
  my $noderestab = xCAT::Table->new('noderes'); #in order to detect per-node tftp directories
  my %nrhash = %{$noderestab->getNodesAttribs(\@nodes,[qw(tftpdir)])};
  
  if (@nodes>0) {
    foreach my $node (@nodes) {
      my $tftpdir;
      if ($nrhash{$node}->[0] and $nrhash{$node}->[0]->{tftpdir}) {
 	$tftpdir = $nrhash{$node}->[0]->{tftpdir};
      } else {
         $tftpdir = $globaltftpdir;
      }
      my $tmp=getstate($node, $tftpdir);
      my @a=split(' ', $tmp);
      $stat = $a[0];
      if (exists($hashref->{$stat})) {
	  my $pa=$hashref->{$stat};
	  push(@$pa, $node);
      }
      else {
	  $hashref->{$stat}=[$node];
      }
    }
  }
  return (0, "");
}

1;