From 1b4468021719a8ca49a091a52cffaf0cf6ebe4f2 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 14 Jan 2016 11:25:03 -0500 Subject: [PATCH 01/13] Preliminary aarch64 support Add hooks to do a draft implementation of aarch64 using grub --- xCAT-server/lib/xcat/plugins/anaconda.pm | 2 +- xCAT-server/lib/xcat/plugins/armgrub.pm | 665 +++++++++++++++++++++++ xCAT-server/lib/xcat/plugins/dhcp.pm | 3 + 3 files changed, 669 insertions(+), 1 deletion(-) create mode 100644 xCAT-server/lib/xcat/plugins/armgrub.pm diff --git a/xCAT-server/lib/xcat/plugins/anaconda.pm b/xCAT-server/lib/xcat/plugins/anaconda.pm index 541cec729..1e6f74518 100644 --- a/xCAT-server/lib/xcat/plugins/anaconda.pm +++ b/xCAT-server/lib/xcat/plugins/anaconda.pm @@ -1225,7 +1225,7 @@ sub mkinstall if ( ( - $arch =~ /x86/ and + ( $arch =~ /x86/ or $arch =~ /aarch64/ ) and ( -r "$pkgdir/images/pxeboot/vmlinuz" and $kernpath = "$pkgdir/images/pxeboot/vmlinuz" diff --git a/xCAT-server/lib/xcat/plugins/armgrub.pm b/xCAT-server/lib/xcat/plugins/armgrub.pm new file mode 100644 index 000000000..dcea3b824 --- /dev/null +++ b/xCAT-server/lib/xcat/plugins/armgrub.pm @@ -0,0 +1,665 @@ +# IBM(c) 2007 EPL license http://www.eclipse.org/legal/epl-v10.html +# Copyright Lenovo 2016 +package xCAT_plugin::armgrub; +use Data::Dumper; +use Sys::Syslog; +use xCAT::Scope; +use xCAT::MsgUtils; +use Socket; +use File::Copy; +use File::Path; +use Getopt::Long; +require xCAT::Utils; +require xCAT::TableUtils; +use xCAT::ServiceNodeUtils; +my $dhcpconf = "/etc/dhcpd.conf"; +my $globaltftpdir = xCAT::TableUtils->getTftpDir(); +#my $dhcpver = 3; + +my %usage = ( + "nodeset" => "Usage: nodeset [shell|boot|runcmd=bmcsetup|iscsiboot|osimage[=]|offline]", +); +sub handled_commands { + return { + nodeset => "noderes:netboot" + } +} + +sub check_dhcp { + return 1; + #TODO: omapi magic to do things right + my $node = shift; + my $dhcpfile; + open ($dhcpfile,$dhcpconf); + while (<$dhcpfile>) { + if (/host $node\b/) { + close $dhcpfile; + return 1; + } + } + close $dhcpfile; + return 0; +} + +sub _slow_get_tftpdir { #make up for paths where tftpdir is not passed in + my $node = shift; + my $nrtab = xCAT::Table->new('noderes',-create=>0); #in order to detect per-node tftp directories + unless ($nrtab) { return $globaltftpdir; } + my $ent = $nrtab->getNodeAttribs($node,["tftpdir"]); + if ($ent and $ent->{tftpdir}) { + return $ent->{tftpdir}; + } else { + return $globaltftpdir; + } +} +sub getstate { + my $node = shift; + my $tftpdir = shift; + unless ($tftpdir) { $tftpdir = _slow_get_tftpdir($node); } + if (check_dhcp($node)) { + if (-r $tftpdir . "/armgrub/".$node) { + my $fhand; + open ($fhand,$tftpdir . "/armgrub/".$node); + my $headline = <$fhand>; + close $fhand; + $headline =~ s/^#//; + chomp($headline); + return $headline; + } else { + return "boot"; + } + } else { + return "discover"; + } +} + +sub setstate { +=pod + + This function will manipulate the armgrub structure to match what the noderes/chain tables indicate the node should be booting. + +=cut + my $node = shift; + my %bphash = %{shift()}; + my %chainhash = %{shift()}; + my %machash = %{shift()}; + my %nthash = %{shift()}; + my $tftpdir = shift; + my %linuximghash = (); + my $linuximghashref = shift; + if (ref $linuximghashref) { %linuximghash = %{$linuximghashref}; } + my $imgaddkcmdline=($linuximghash{'boottarget'})? undef:$linuximghash{'addkcmdline'}; + + my $kern = $bphash{$node}->[0]; #$bptab->getNodeAttribs($node,['kernel','initrd','kcmdline']); + if (not $::PXE_addkcmdlinehandled->{$node} and ($kern->{addkcmdline} or ($imgaddkcmdline))) { + +#Implement the kcmdline append here for +#most generic, least code duplication + +###hack start +# This is my comment. There are many others like it, but this one is mine. +# My comment is my best friend. It is my life. I must master it as I must master my life. +# Without me, my comment is useless. Without my comment, I am useless. + +# Jarrod to clean up. It really should be in Table.pm and support +# the new statelite $table notation. + +#I dislike spaces, tabs are cleaner, I'm too tired to change all the xCAT code. +#I give in. + + my $kcmdlinehack = ($imgaddkcmdline)?$kern->{addkcmdline}." ".$imgaddkcmdline : $kern->{addkcmdline}; + + my $cmdhashref; + if($kcmdlinehack){ + $cmdhashref=xCAT::Utils->splitkcmdline($kcmdlinehack); + } + + if($cmdhashref and $cmdhashref->{volatile}) + { + $kcmdlinehack=$cmdhashref->{volatile}; + } + + + while ($kcmdlinehack =~ /#NODEATTRIB:([^:#]+):([^:#]+)#/) { + my $natab = xCAT::Table->new($1); + my $naent = $natab->getNodeAttribs($node,[$2]); + my $naval = $naent->{$2}; + $kcmdlinehack =~ s/#NODEATTRIB:([^:#]+):([^:#]+)#/$naval/; + } + while ($kcmdlinehack =~ /#TABLE:([^:#]+):([^:#]+):([^:#]+)#/) { + my $tabname = $1; + my $keyname = $2; + my $colname = $3; + if ($2 =~ /THISNODE/ or $2 =~ /\$NODE/) { + my $natab = xCAT::Table->new($tabname); + my $naent = $natab->getNodeAttribs($node,[$colname]); + my $naval = $naent->{$colname}; + $kcmdlinehack =~ s/#TABLE:([^:#]+):([^:#]+):([^:#]+)#/$naval/; + } else { + my $msg = "Table key of $2 not yet supported by boottarget mini-template"; + $::PXE_callback->({ + error => ["$msg"], + errorcode => [1] + }); + } + } + + #$kern->{kcmdline} .= " ".$kern->{addkcmdline}; + + $kern->{kcmdline} .= " ".$kcmdlinehack; + +###hack end + + } + if ($kern->{kcmdline} =~ /!myipfn!/) { + my $ipfn; + my @ipfnd = xCAT::NetworkUtils->my_ip_facing($node); + unless ($ipfnd[0]) { $ipfn = $ipfnd[1];} + unless ($ipfn) { + my @myself = xCAT::NetworkUtils->determinehostname(); + my $myname = $myself[(scalar @myself)-1]; + $::PXE_callback->( + { + error => [ + "$myname: Unable to determine or reasonably guess the image server for $node" + ], + errorcode => [1] + } + ); + } + $kern->{kcmdline} =~ s/!myipfn!/$ipfn/g; + } + my $pcfg; + unless (-d $tftpdir."/armgrub/") { + mkpath($tftpdir."/armgrub/"); + } + + open($pcfg,'>',$tftpdir."/armgrub/".$node); + my $cref=$chainhash{$node}->[0]; #$chaintab->getNodeAttribs($node,['currstate']); + if ($cref->{currstate}) { + print $pcfg "#".$cref->{currstate}."\n"; + } + print $pcfg "set default=\"1\"\n"; + print $pcfg 'regexp --set 1:mac1 --set 2:mac2 --set 3:mac3 --set 4:mac4 --set 5:mac5 --set 6:mac6 (..):(..):(..):(..):(..):(..) $net_efinet0_mac'."\n"; + if ($cref and $cref->{currstate} eq "boot") { + print $pcfg "exit\n"; + close($pcfg); + } elsif ($kern and $kern->{kernel}) { + #It's time to set pxelinux for this node to boot the kernel.. + print $pcfg " menuentry 'XCAT Install' --class os {\n linux ".$kern->{kernel}; + if ($kern->{initrd} or $kern->{kcmdline}) { + print $pcfg " " . $kern->{kcmdline}. ' BOOTIF=01-$mac1-$mac2-$mac3-$mac4-$mac5-$mac6'."\n"; + } + print $pcfg "\n"; + if ($kern and $kern->{initrd}) { + print $pcfg " initrd " . $kern->{initrd}."\n"; + } + print $pcfg "}\n"; + + close($pcfg); + } else { #TODO: actually, should possibly default to xCAT image? + print $pcfg "LOCALBOOT 0\n"; + close($pcfg); + } + my $mactab = xCAT::Table->new('mac'); #to get all the hostnames + my %ipaddrs; + unless (inet_aton($node)) { + syslog("local4|err","xCAT unable to resolve IP in pxe plugin"); + return; + } + my $ip = inet_ntoa(inet_aton($node));; + unless ($ip) { + syslog("local4|err","xCAT unable to resolve IP in pxe plugin"); + return; + } + $ipaddrs{$ip} = 1; + if ($mactab) { + my $ment = $machash{$node}->[0]; #$mactab->getNodeAttribs($node,['mac']); + if ($ment and $ment->{mac}) { + my @macs = split(/\|/,$ment->{mac}); + foreach (@macs) { + if (/!(.*)/) { + if (inet_aton($1)) { + $ipaddrs{inet_ntoa(inet_aton($1))} = 1; + } + } + } + } + } + my $hassymlink = eval { symlink("",""); 1 }; + foreach $ip (keys %ipaddrs) { + my @ipa=split(/\./,$ip); + my $pname = sprintf("%02X%02X%02X%02X",@ipa); + unlink($tftpdir."/armgrub/grub.cfg-".$pname); + if ($hassymlink) { + symlink($node,$tftpdir."/armgrub/grub.cfg-".$pname); + } else { + link($tftpdir."/armgrub/".$node,$tftpdir."/armgrub/grub.cfg-".$pname); + } + } +} + + + +my $errored = 0; +sub pass_along { + my $resp = shift; + if ($resp and ($resp->{errorcode} and $resp->{errorcode}->[0]) or ($resp->{error} and $resp->{error}->[0])) { + $errored=1; + } + foreach (@{$resp->{node}}) { + if ($_->{error} or $_->{errorcode}) { + $errored=1; + } + if ($_->{_addkcmdlinehandled}) { + $::PXE_addkcmdlinehandled->{$_->{name}->[0]}=1; + return; #Don't send back to client this internal hint + } + } + $::PXE_callback->($resp); +} + + + +sub preprocess_request { + #Assume shared tftp directory for boring people, but for cool people, help sync up tftpdirectory contents when + #they specify no sharedtftp in site table + #my $stab = xCAT::Table->new('site'); + my $req = shift; + my $callback1 = shift; + my $command = $req->{command}->[0]; + my $sub_req = shift; + my $nodes = $req->{node}; + my @args=(); + if (ref($req->{arg})) { + @args=@{$req->{arg}}; + } else { + @args=($req->{arg}); + } + @ARGV = @args; + my $HELP; + my $VERSION; + my $VERBOSE; + Getopt::Long::Configure("bundling"); + Getopt::Long::Configure("pass_through"); + if (!GetOptions('h|?|help' => \$HELP, + 'v|version' => \$VERSION, + 'V' => \$VERBOSE #>>>>>>>used for trace log>>>>>>> + ) ) { + if($usage{$command}) { + my %rsp; + $rsp{data}->[0]=$usage{$command}; + $callback1->(\%rsp); + } + return; + } + + #>>>>>>>used for trace log start>>>>>> + my $verbose_on_off=0; + if($VERBOSE){$verbose_on_off=1;} + #>>>>>>>used for trace log end>>>>>>> + + if ($HELP) { + if($usage{$command}) { + my %rsp; + $rsp{data}->[0]=$usage{$command}; + $callback1->(\%rsp); + } + return; + } + + if ($VERSION) { + my $ver = xCAT::Utils->Version(); + my %rsp; + $rsp{data}->[0]="$ver"; + $callback1->(\%rsp); + return; + } + + if (@ARGV==0) { + if($usage{$command}) { + my %rsp; + $rsp{data}->[0]=$usage{$command}; + $callback1->(\%rsp); + } + return; + } + + #my $sent = $stab->getAttribs({key=>'sharedtftp'},'value'); + my @entries = xCAT::TableUtils->get_site_attribute("sharedtftp"); + my $t_entry = $entries[0]; + xCAT::MsgUtils->trace($verbose_on_off,"d","pxe: sharedtftp=$t_entry"); + if ( defined($t_entry) and ($t_entry eq "0" or $t_entry eq "no" or $t_entry eq "NO")) { + # check for computenodes and servicenodes from the noderange, if so error out + my @SN; + my @CN; + xCAT::ServiceNodeUtils->getSNandCPnodes(\@$nodes, \@SN, \@CN); + unless (($args[0] eq 'stat') or ($args[0] eq 'enact')) { # ok for these options + if ((@SN > 0) && (@CN >0 )) { # there are both SN and CN + my $rsp; + $rsp->{data}->[0] = + "Nodeset was run with a noderange containing both service nodes and compute nodes. This is not valid. You must submit with either compute nodes in the noderange or service nodes. \n"; + xCAT::MsgUtils->message("E", $rsp, $callback1); + return; + + } + } + + $req->{'_disparatetftp'}=[1]; + if ($req->{inittime}->[0]) { + return [$req]; + } + if (@CN >0 ) { # if compute nodes broadcast to all servicenodes + return xCAT::Scope->get_broadcast_scope($req,@_); + } + } + return [$req]; +} + +sub process_request { + $::PXE_request = shift; + $::PXE_callback = shift; + my $sub_req = shift; + undef $::PXE_addkcmdlinehandled; + my @args; + my @nodes; + my @rnodes; + + #>>>>>>>used for trace log start>>>>>>> + my %opt; + my $verbose_on_off=0; + if (ref($::PXE_request->{arg})) { + @args=@{$::PXE_request->{arg}}; + } else { + @args=($::PXE_request->{arg}); + } + @ARGV = @args; + GetOptions('V' => \$opt{V}); + if($opt{V}){$verbose_on_off=1;} + #>>>>>>>used for trace log end>>>>>>> + + if (ref($::PXE_request->{node})) { + @rnodes = @{$::PXE_request->{node}}; + } else { + if ($::PXE_request->{node}) { @rnodes = ($::PXE_request->{node}); } + } + + unless (@rnodes) { + if ($usage{$::PXE_request->{command}->[0]}) { + $::PXE_callback->({data=>$usage{$::PXE_request->{command}->[0]}}); + } + return; + } + + #if not shared, then help sync up + if ($::PXE_request->{'_disparatetftp'}->[0]) { #reading hint from preprocess_command + @nodes = (); + foreach (@rnodes) { + if (xCAT::NetworkUtils->nodeonmynet($_)) { + push @nodes,$_; + } else { + xCAT::MsgUtils->message("S", "$_: pxe netboot: stop configuration because of none sharedtftp and not on same network with its xcatmaster."); + } + } + } else { + @nodes = @rnodes; + } + + #>>>>>>>used for trace log>>>>>>> + my $str_node = join(" ",@nodes); + xCAT::MsgUtils->trace($verbose_on_off,"d","pxe: nodes are $str_node"); + + # return directly if no nodes in the same network + unless (@nodes) { + xCAT::MsgUtils->message("S", "xCAT: pxe netboot: no valid nodes. Stop the operation on this server."); + return; + } + + if (ref($::PXE_request->{arg})) { + @args=@{$::PXE_request->{arg}}; + } else { + @args=($::PXE_request->{arg}); + } + + #now run the begin part of the prescripts + unless ($args[0] eq 'stat') { # or $args[0] eq 'enact') { + $errored=0; + if ($::PXE_request->{'_disparatetftp'}->[0]) { #the call is distrubuted to the service node already, so only need to handles my own children + xCAT::MsgUtils->trace($verbose_on_off,"d","pxe: the call is distrubuted to the service node already, so only need to handles my own children"); + xCAT::MsgUtils->trace($verbose_on_off,"d","pxe: issue runbeginpre request"); + $sub_req->({command=>['runbeginpre'], + node=>\@nodes, + arg=>[$args[0], '-l']},\&pass_along); + } else { #nodeset did not distribute to the service node, here we need to let runednpre to distribute the nodes to their masters + xCAT::MsgUtils->trace($verbose_on_off,"d","pxe: nodeset did not distribute to the service node"); + xCAT::MsgUtils->trace($verbose_on_off,"d","pxe: issue runbeginpre request"); + $sub_req->({command=>['runbeginpre'], + node=>\@rnodes, + arg=>[$args[0]]},\&pass_along); + } + if ($errored) { + my $rsp; + $rsp->{errorcode}->[0]=1; + $rsp->{error}->[0]="Failed in running begin prescripts. Processing will still continue.\n"; + $::PXE_callback->($rsp); + } + } + +#end prescripts code + if (! -r "$tftpdir/pxelinux.0") { + unless (-r "/usr/lib/syslinux/pxelinux.0" or -r "/usr/share/syslinux/pxelinux.0") { + $::PXE_callback->({error=>["Unable to find pxelinux.0 "],errorcode=>[1]}); + return; + } + if (-r "/usr/lib/syslinux/pxelinux.0") { + copy("/usr/lib/syslinux/pxelinux.0","$tftpdir/pxelinux.0"); + } else { + copy("/usr/share/syslinux/pxelinux.0","$tftpdir/pxelinux.0"); + } + chmod(0644,"$tftpdir/pxelinux.0"); + } + unless ( -r "$tftpdir/pxelinux.0" ) { + $::PXE_callback->({errror=>["Unable to find pxelinux.0 from syslinux"],errorcode=>[1]}); + return; + } + + + $errored=0; + my $inittime=0; + if (exists($::PXE_request->{inittime})) { $inittime= $::PXE_request->{inittime}->[0];} + if (!$inittime) { $inittime=0;} + unless ($args[0] eq 'stat') { # or $args[0] eq 'enact') { + xCAT::MsgUtils->trace($verbose_on_off,"d","pxe: issue setdestiny request"); + $sub_req->({command=>['setdestiny'], + node=>\@nodes, + inittime=>[$inittime], + arg=>\@args},\&pass_along); + } + if ($errored) { return; } + #Time to actually configure the nodes, first extract database data with the scalable calls + my $bptab = xCAT::Table->new('bootparams',-create=>1); + my $chaintab = xCAT::Table->new('chain'); + my $mactab = xCAT::Table->new('mac'); #to get all the hostnames + my $typetab = xCAT::Table->new('nodetype'); + my $restab = xCAT::Table->new('noderes'); + my $linuximgtab=xCAT::Table->new('linuximage',-create=>1); + my %nrhash = %{$restab->getNodesAttribs(\@nodes,[qw(tftpdir)])}; + my %bphash = %{$bptab->getNodesAttribs(\@nodes,[qw(kernel initrd kcmdline addkcmdline)])}; + my %chainhash = %{$chaintab->getNodesAttribs(\@nodes,[qw(currstate)])}; + my %machash = %{$mactab->getNodesAttribs(\@nodes,[qw(mac)])}; + my %nthash = %{$typetab->getNodesAttribs(\@nodes,[qw(os provmethod)])}; + foreach (@nodes) { + my %response; + my $tftpdir; + if ($nrhash{$_} and $nrhash{$_}->[0] and $nrhash{$_}->[0]->{tftpdir}) { + $tftpdir = $nrhash{$_}->[0]->{tftpdir}; + } else { + $tftpdir = $globaltftpdir; + } + $response{node}->[0]->{name}->[0]=$_; + if ($args[0] eq 'stat') { + $response{node}->[0]->{data}->[0]= getstate($_,$tftpdir); + $::PXE_callback->(\%response); + } elsif ($args[0]) { #If anything else, send it on to the destiny plugin, then setstate + my $ent = $nthash{$_}->[0]; + my $osimgname = $ent->{'provmethod'}; + my $linuximghash=undef; + unless($osimgname =~ /^(install|netboot|statelite)$/){ + $linuximghash = $linuximgtab->getAttribs({imagename => $osimgname}, 'boottarget', 'addkcmdline'); + } + ($rc,$errstr) = setstate($_,\%bphash,\%chainhash,\%machash,\%nthash,$tftpdir,$linuximghash); + if ($rc) { + $response{node}->[0]->{errorcode}->[0]= $rc; + $response{node}->[0]->{errorc}->[0]= $errstr; + $::PXE_callback->(\%response); + } + } + } + + my $inittime=0; + if (exists($::PXE_request->{inittime})) { $inittime= $::PXE_request->{inittime}->[0];} + if (!$inittime) { $inittime=0;} + + #dhcp stuff -- inittime is set when xcatd on sn is started + unless (($args[0] eq 'stat') || ($inittime) || ($args[0] eq 'offline')) { + my $do_dhcpsetup=1; + #my $sitetab = xCAT::Table->new('site'); + #if ($sitetab) { + #(my $ref) = $sitetab->getAttribs({key => 'dhcpsetup'}, 'value'); + my @entries = xCAT::TableUtils->get_site_attribute("dhcpsetup"); + my $t_entry = $entries[0]; + if ( defined($t_entry) ) { + if ($t_entry =~ /0|n|N/) { $do_dhcpsetup=0; } + } + #} + + if ($do_dhcpsetup) { + if ($::PXE_request->{'_disparatetftp'}->[0]) { #reading hint from preprocess_command + xCAT::MsgUtils->trace($verbose_on_off,"d","pxe: issue makedhcp request"); + $sub_req->({command=>['makedhcp'],arg=>['-l'], + node=>\@nodes},$::PXE_callback); + } else { + xCAT::MsgUtils->trace($verbose_on_off,"d","pxe: issue makedhcp request"); + $sub_req->({command=>['makedhcp'], + node=>\@nodes},$::PXE_callback); + } + } + + } + #unlink the files for 'offline' command + if($args[0] eq 'offline') { + foreach my $node (@nodes) { + my %ipaddrs; + unless (inet_aton($node)) { + syslog("local4|err","xCAT unable to resolve IP in pxe plugin"); + return; + } + my $ip = inet_ntoa(inet_aton($node));; + unless ($ip) { + syslog("local4|err","xCAT unable to resolve IP in pxe plugin"); + return; + } + $ipaddrs{$ip} = 1; + if ($mactab) { + my $ment = $machash{$node}->[0]; #$mactab->getNodeAttribs($node,['mac']); + if ($ment and $ment->{mac}) { + my @macs = split(/\|/,$ment->{mac}); + foreach (@macs) { + if (/!(.*)/) { + if (inet_aton($1)) { + $ipaddrs{inet_ntoa(inet_aton($1))} = 1; + } + } + } + } + } + my $hassymlink = eval { symlink("",""); 1 }; + + unlink($tftpdir."/armgrub/".$node); + + foreach $ip (keys %ipaddrs) { + my @ipa=split(/\./,$ip); + my $pname = sprintf("%02X%02X%02X%02X",@ipa); + unlink($tftpdir."/armgrub/".$pname); + #if ($hassymlink) { + #symlink($node,$tftpdir."/armgrub/".$pname); + #} else { + #link($tftpdir."/armgrub/".$node,$tftpdir."/armgrub/".$pname); + #} + } + } + } + + #now run the end part of the prescripts + unless ($args[0] eq 'stat') { # or $args[0] eq 'enact') + $errored=0; + if ($::PXE_request->{'_disparatetftp'}->[0]) { #the call is distrubuted to the service node already, so only need to handles my own children + xCAT::MsgUtils->trace($verbose_on_off,"d","pxe: issue runendpre request"); + $sub_req->({command=>['runendpre'], + node=>\@nodes, + arg=>[$args[0], '-l']},\&pass_along); + } else { #nodeset did not distribute to the service node, here we need to let runednpre to distribute the nodes to their masters + xCAT::MsgUtils->trace($verbose_on_off,"d","pxe: issue runendpre request"); + $sub_req->({command=>['runendpre'], + node=>\@rnodes, + arg=>[$args[0]]},\&pass_along); + } + if ($errored) { + my $rsp; + $rsp->{errorcode}->[0]=1; + $rsp->{error}->[0]="Failed in running end prescripts. Processing will still continue.\n"; + $::PXE_callback->($rsp); + } + } + + + +} + + +#---------------------------------------------------------------------------- +=head3 getNodesetStates + returns the nodeset state for the given nodes. The possible nodeset + states are: netboot, install, boot and discover. + Arguments: + nodes --- a pointer to an array of nodes + states -- a pointer to a hash table. This hash will be filled by this + function. The key is the nodeset status and the value is a pointer + to an array of nodes. + Returns: + (return code, error message) +=cut +#----------------------------------------------------------------------------- +sub getNodesetStates { + my $noderef=shift; + if ($noderef =~ /xCAT_plugin::pxe/) { + $noderef=shift; + } + my @nodes=@$noderef; + my $hashref=shift; + my $noderestab = xCAT::Table->new('noderes'); #in order to detect per-node tftp directories + my %nrhash = %{$noderestab->getNodesAttribs(\@nodes,[qw(tftpdir)])}; + if (@nodes>0) { + foreach my $node (@nodes) { + my $tftpdir; + if ($nrhash{$node}->[0] and $nrhash{$node}->[0]->{tftpdir}) { + $tftpdir = $nrhash{$node}->[0]->{tftpdir}; + } else { + $tftpdir = $globaltftpdir; + } + my $tmp=getstate($node, $tftpdir); + my @a=split(' ', $tmp); + $stat = $a[0]; + if (exists($hashref->{$stat})) { + my $pa=$hashref->{$stat}; + push(@$pa, $node); + } + else { + $hashref->{$stat}=[$node]; + } + } + } + return (0, ""); +} + +1; diff --git a/xCAT-server/lib/xcat/plugins/dhcp.pm b/xCAT-server/lib/xcat/plugins/dhcp.pm index 17a116b78..f9e67075e 100644 --- a/xCAT-server/lib/xcat/plugins/dhcp.pm +++ b/xCAT-server/lib/xcat/plugins/dhcp.pm @@ -2740,6 +2740,9 @@ sub addnet push @netent, " } else if option client-architecture = 00:02 { #ia64\n "; push @netent, " filename \"elilo.efi\";\n"; + push @netent, + " } else if option client-architecture = 00:0b { #aaarch64\n "; + push @netent, " filename \"armgrub/grubaa64.efi\";\n"; push @netent, " } else if option client-architecture = 00:0e { #OPAL-v3\n "; push @netent, " option conf-file = \"http://$tftp:$httpport/tftpboot/pxelinux.cfg/p/" . $net . "_" . $maskbits . "\";\n"; From 64039874004c0ca90aec5a77574055f9dec24d7c Mon Sep 17 00:00:00 2001 From: Markus Hilger Date: Wed, 18 May 2022 18:05:22 +0200 Subject: [PATCH 02/13] new kernel for aarch64 --- xCAT-server/lib/xcat/plugins/anaconda.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xCAT-server/lib/xcat/plugins/anaconda.pm b/xCAT-server/lib/xcat/plugins/anaconda.pm index 1e6f74518..542c4e75a 100644 --- a/xCAT-server/lib/xcat/plugins/anaconda.pm +++ b/xCAT-server/lib/xcat/plugins/anaconda.pm @@ -2685,7 +2685,7 @@ sub insert_dd { # and copy it to the /tftpboot my @new_kernels = <$dd_dir/rpm/boot/vmlinuz*>; foreach my $new_kernel (@new_kernels) { - if (-r $new_kernel && $new_kernel =~ /\/vmlinuz-(.*(x86_64|ppc64|el\d+|ppc64le))$/) { + if (-r $new_kernel && $new_kernel =~ /\/vmlinuz-(.*(x86_64|ppc64|el\d+|ppc64le|aarch64))$/) { $new_kernel_ver = $1; $cmd = "/bin/mv -f $new_kernel $kernelpath"; xCAT::Utils->runcmd($cmd, -1); From 826a8a830840408da18bda85c5250288172c7248 Mon Sep 17 00:00:00 2001 From: Markus Hilger Date: Wed, 18 May 2022 18:19:07 +0200 Subject: [PATCH 03/13] Support x86, ppc64 and arm in grub Arm and ppc64 do not have linuxefi and initrdefi. --- xCAT-server/lib/xcat/plugins/grub2.pm | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/xCAT-server/lib/xcat/plugins/grub2.pm b/xCAT-server/lib/xcat/plugins/grub2.pm index b1e5becdc..cbe57965c 100644 --- a/xCAT-server/lib/xcat/plugins/grub2.pm +++ b/xCAT-server/lib/xcat/plugins/grub2.pm @@ -254,14 +254,19 @@ sub setstate { $protocolrootdir = $tftpdir; } + my $efi = ""; + if ($nodearch =~ /x86/i) { + $efi = "efi"; + } + if ($kern and $kern->{kcmdline}) { - print $pcfg " linux $protocolrootdir/$kern->{kernel} $kern->{kcmdline}\n"; + print $pcfg " linux$efi $protocolrootdir/$kern->{kernel} $kern->{kcmdline} BOOTIF=\$net_default_mac\n"; } else { - print $pcfg " linux $protocolrootdir/$kern->{kernel}\n"; + print $pcfg " linux$efi $protocolrootdir/$kern->{kernel} BOOTIF=\$net_default_mac\n"; } print $pcfg " echo Loading initial ramdisk ...\n"; if ($kern and $kern->{initrd}) { - print $pcfg " initrd $protocolrootdir/$kern->{initrd}\n"; + print $pcfg " initrd$efi $protocolrootdir/$kern->{initrd}\n"; } print $pcfg "}"; From 699a26d3eca562917a18f5105fd5bba198fa0da5 Mon Sep 17 00:00:00 2001 From: Markus Hilger Date: Thu, 19 May 2022 12:47:26 +0200 Subject: [PATCH 04/13] Fix nodeset and rinstall for arm --- perl-xCAT/xCAT/Utils.pm | 2 ++ 1 file changed, 2 insertions(+) diff --git a/perl-xCAT/xCAT/Utils.pm b/perl-xCAT/xCAT/Utils.pm index bb3827a16..1c8ee9d5a 100644 --- a/perl-xCAT/xCAT/Utils.pm +++ b/perl-xCAT/xCAT/Utils.pm @@ -4839,6 +4839,8 @@ sub lookupNetboot { } } elsif ($osarch =~ /^ppc64le$/i or $osarch =~ /^ppc64el$/i) { $ret = "petitboot,grub2,grub2-tftp,grub2-http"; + } elsif ($osarch =~ /^aarch64$/i) { + $ret = "grub2"; } } return $ret; From 40ac583410e06ba290cdac5c62373666cf36cd9f Mon Sep 17 00:00:00 2001 From: Markus Hilger Date: Thu, 19 May 2022 13:20:10 +0200 Subject: [PATCH 05/13] Do not use hacked armgrub plugin --- xCAT-server/lib/xcat/plugins/armgrub.pm | 665 ------------------------ xCAT-server/lib/xcat/plugins/dhcp.pm | 2 +- 2 files changed, 1 insertion(+), 666 deletions(-) delete mode 100644 xCAT-server/lib/xcat/plugins/armgrub.pm diff --git a/xCAT-server/lib/xcat/plugins/armgrub.pm b/xCAT-server/lib/xcat/plugins/armgrub.pm deleted file mode 100644 index dcea3b824..000000000 --- a/xCAT-server/lib/xcat/plugins/armgrub.pm +++ /dev/null @@ -1,665 +0,0 @@ -# IBM(c) 2007 EPL license http://www.eclipse.org/legal/epl-v10.html -# Copyright Lenovo 2016 -package xCAT_plugin::armgrub; -use Data::Dumper; -use Sys::Syslog; -use xCAT::Scope; -use xCAT::MsgUtils; -use Socket; -use File::Copy; -use File::Path; -use Getopt::Long; -require xCAT::Utils; -require xCAT::TableUtils; -use xCAT::ServiceNodeUtils; -my $dhcpconf = "/etc/dhcpd.conf"; -my $globaltftpdir = xCAT::TableUtils->getTftpDir(); -#my $dhcpver = 3; - -my %usage = ( - "nodeset" => "Usage: nodeset [shell|boot|runcmd=bmcsetup|iscsiboot|osimage[=]|offline]", -); -sub handled_commands { - return { - nodeset => "noderes:netboot" - } -} - -sub check_dhcp { - return 1; - #TODO: omapi magic to do things right - my $node = shift; - my $dhcpfile; - open ($dhcpfile,$dhcpconf); - while (<$dhcpfile>) { - if (/host $node\b/) { - close $dhcpfile; - return 1; - } - } - close $dhcpfile; - return 0; -} - -sub _slow_get_tftpdir { #make up for paths where tftpdir is not passed in - my $node = shift; - my $nrtab = xCAT::Table->new('noderes',-create=>0); #in order to detect per-node tftp directories - unless ($nrtab) { return $globaltftpdir; } - my $ent = $nrtab->getNodeAttribs($node,["tftpdir"]); - if ($ent and $ent->{tftpdir}) { - return $ent->{tftpdir}; - } else { - return $globaltftpdir; - } -} -sub getstate { - my $node = shift; - my $tftpdir = shift; - unless ($tftpdir) { $tftpdir = _slow_get_tftpdir($node); } - if (check_dhcp($node)) { - if (-r $tftpdir . "/armgrub/".$node) { - my $fhand; - open ($fhand,$tftpdir . "/armgrub/".$node); - my $headline = <$fhand>; - close $fhand; - $headline =~ s/^#//; - chomp($headline); - return $headline; - } else { - return "boot"; - } - } else { - return "discover"; - } -} - -sub setstate { -=pod - - This function will manipulate the armgrub structure to match what the noderes/chain tables indicate the node should be booting. - -=cut - my $node = shift; - my %bphash = %{shift()}; - my %chainhash = %{shift()}; - my %machash = %{shift()}; - my %nthash = %{shift()}; - my $tftpdir = shift; - my %linuximghash = (); - my $linuximghashref = shift; - if (ref $linuximghashref) { %linuximghash = %{$linuximghashref}; } - my $imgaddkcmdline=($linuximghash{'boottarget'})? undef:$linuximghash{'addkcmdline'}; - - my $kern = $bphash{$node}->[0]; #$bptab->getNodeAttribs($node,['kernel','initrd','kcmdline']); - if (not $::PXE_addkcmdlinehandled->{$node} and ($kern->{addkcmdline} or ($imgaddkcmdline))) { - -#Implement the kcmdline append here for -#most generic, least code duplication - -###hack start -# This is my comment. There are many others like it, but this one is mine. -# My comment is my best friend. It is my life. I must master it as I must master my life. -# Without me, my comment is useless. Without my comment, I am useless. - -# Jarrod to clean up. It really should be in Table.pm and support -# the new statelite $table notation. - -#I dislike spaces, tabs are cleaner, I'm too tired to change all the xCAT code. -#I give in. - - my $kcmdlinehack = ($imgaddkcmdline)?$kern->{addkcmdline}." ".$imgaddkcmdline : $kern->{addkcmdline}; - - my $cmdhashref; - if($kcmdlinehack){ - $cmdhashref=xCAT::Utils->splitkcmdline($kcmdlinehack); - } - - if($cmdhashref and $cmdhashref->{volatile}) - { - $kcmdlinehack=$cmdhashref->{volatile}; - } - - - while ($kcmdlinehack =~ /#NODEATTRIB:([^:#]+):([^:#]+)#/) { - my $natab = xCAT::Table->new($1); - my $naent = $natab->getNodeAttribs($node,[$2]); - my $naval = $naent->{$2}; - $kcmdlinehack =~ s/#NODEATTRIB:([^:#]+):([^:#]+)#/$naval/; - } - while ($kcmdlinehack =~ /#TABLE:([^:#]+):([^:#]+):([^:#]+)#/) { - my $tabname = $1; - my $keyname = $2; - my $colname = $3; - if ($2 =~ /THISNODE/ or $2 =~ /\$NODE/) { - my $natab = xCAT::Table->new($tabname); - my $naent = $natab->getNodeAttribs($node,[$colname]); - my $naval = $naent->{$colname}; - $kcmdlinehack =~ s/#TABLE:([^:#]+):([^:#]+):([^:#]+)#/$naval/; - } else { - my $msg = "Table key of $2 not yet supported by boottarget mini-template"; - $::PXE_callback->({ - error => ["$msg"], - errorcode => [1] - }); - } - } - - #$kern->{kcmdline} .= " ".$kern->{addkcmdline}; - - $kern->{kcmdline} .= " ".$kcmdlinehack; - -###hack end - - } - if ($kern->{kcmdline} =~ /!myipfn!/) { - my $ipfn; - my @ipfnd = xCAT::NetworkUtils->my_ip_facing($node); - unless ($ipfnd[0]) { $ipfn = $ipfnd[1];} - unless ($ipfn) { - my @myself = xCAT::NetworkUtils->determinehostname(); - my $myname = $myself[(scalar @myself)-1]; - $::PXE_callback->( - { - error => [ - "$myname: Unable to determine or reasonably guess the image server for $node" - ], - errorcode => [1] - } - ); - } - $kern->{kcmdline} =~ s/!myipfn!/$ipfn/g; - } - my $pcfg; - unless (-d $tftpdir."/armgrub/") { - mkpath($tftpdir."/armgrub/"); - } - - open($pcfg,'>',$tftpdir."/armgrub/".$node); - my $cref=$chainhash{$node}->[0]; #$chaintab->getNodeAttribs($node,['currstate']); - if ($cref->{currstate}) { - print $pcfg "#".$cref->{currstate}."\n"; - } - print $pcfg "set default=\"1\"\n"; - print $pcfg 'regexp --set 1:mac1 --set 2:mac2 --set 3:mac3 --set 4:mac4 --set 5:mac5 --set 6:mac6 (..):(..):(..):(..):(..):(..) $net_efinet0_mac'."\n"; - if ($cref and $cref->{currstate} eq "boot") { - print $pcfg "exit\n"; - close($pcfg); - } elsif ($kern and $kern->{kernel}) { - #It's time to set pxelinux for this node to boot the kernel.. - print $pcfg " menuentry 'XCAT Install' --class os {\n linux ".$kern->{kernel}; - if ($kern->{initrd} or $kern->{kcmdline}) { - print $pcfg " " . $kern->{kcmdline}. ' BOOTIF=01-$mac1-$mac2-$mac3-$mac4-$mac5-$mac6'."\n"; - } - print $pcfg "\n"; - if ($kern and $kern->{initrd}) { - print $pcfg " initrd " . $kern->{initrd}."\n"; - } - print $pcfg "}\n"; - - close($pcfg); - } else { #TODO: actually, should possibly default to xCAT image? - print $pcfg "LOCALBOOT 0\n"; - close($pcfg); - } - my $mactab = xCAT::Table->new('mac'); #to get all the hostnames - my %ipaddrs; - unless (inet_aton($node)) { - syslog("local4|err","xCAT unable to resolve IP in pxe plugin"); - return; - } - my $ip = inet_ntoa(inet_aton($node));; - unless ($ip) { - syslog("local4|err","xCAT unable to resolve IP in pxe plugin"); - return; - } - $ipaddrs{$ip} = 1; - if ($mactab) { - my $ment = $machash{$node}->[0]; #$mactab->getNodeAttribs($node,['mac']); - if ($ment and $ment->{mac}) { - my @macs = split(/\|/,$ment->{mac}); - foreach (@macs) { - if (/!(.*)/) { - if (inet_aton($1)) { - $ipaddrs{inet_ntoa(inet_aton($1))} = 1; - } - } - } - } - } - my $hassymlink = eval { symlink("",""); 1 }; - foreach $ip (keys %ipaddrs) { - my @ipa=split(/\./,$ip); - my $pname = sprintf("%02X%02X%02X%02X",@ipa); - unlink($tftpdir."/armgrub/grub.cfg-".$pname); - if ($hassymlink) { - symlink($node,$tftpdir."/armgrub/grub.cfg-".$pname); - } else { - link($tftpdir."/armgrub/".$node,$tftpdir."/armgrub/grub.cfg-".$pname); - } - } -} - - - -my $errored = 0; -sub pass_along { - my $resp = shift; - if ($resp and ($resp->{errorcode} and $resp->{errorcode}->[0]) or ($resp->{error} and $resp->{error}->[0])) { - $errored=1; - } - foreach (@{$resp->{node}}) { - if ($_->{error} or $_->{errorcode}) { - $errored=1; - } - if ($_->{_addkcmdlinehandled}) { - $::PXE_addkcmdlinehandled->{$_->{name}->[0]}=1; - return; #Don't send back to client this internal hint - } - } - $::PXE_callback->($resp); -} - - - -sub preprocess_request { - #Assume shared tftp directory for boring people, but for cool people, help sync up tftpdirectory contents when - #they specify no sharedtftp in site table - #my $stab = xCAT::Table->new('site'); - my $req = shift; - my $callback1 = shift; - my $command = $req->{command}->[0]; - my $sub_req = shift; - my $nodes = $req->{node}; - my @args=(); - if (ref($req->{arg})) { - @args=@{$req->{arg}}; - } else { - @args=($req->{arg}); - } - @ARGV = @args; - my $HELP; - my $VERSION; - my $VERBOSE; - Getopt::Long::Configure("bundling"); - Getopt::Long::Configure("pass_through"); - if (!GetOptions('h|?|help' => \$HELP, - 'v|version' => \$VERSION, - 'V' => \$VERBOSE #>>>>>>>used for trace log>>>>>>> - ) ) { - if($usage{$command}) { - my %rsp; - $rsp{data}->[0]=$usage{$command}; - $callback1->(\%rsp); - } - return; - } - - #>>>>>>>used for trace log start>>>>>> - my $verbose_on_off=0; - if($VERBOSE){$verbose_on_off=1;} - #>>>>>>>used for trace log end>>>>>>> - - if ($HELP) { - if($usage{$command}) { - my %rsp; - $rsp{data}->[0]=$usage{$command}; - $callback1->(\%rsp); - } - return; - } - - if ($VERSION) { - my $ver = xCAT::Utils->Version(); - my %rsp; - $rsp{data}->[0]="$ver"; - $callback1->(\%rsp); - return; - } - - if (@ARGV==0) { - if($usage{$command}) { - my %rsp; - $rsp{data}->[0]=$usage{$command}; - $callback1->(\%rsp); - } - return; - } - - #my $sent = $stab->getAttribs({key=>'sharedtftp'},'value'); - my @entries = xCAT::TableUtils->get_site_attribute("sharedtftp"); - my $t_entry = $entries[0]; - xCAT::MsgUtils->trace($verbose_on_off,"d","pxe: sharedtftp=$t_entry"); - if ( defined($t_entry) and ($t_entry eq "0" or $t_entry eq "no" or $t_entry eq "NO")) { - # check for computenodes and servicenodes from the noderange, if so error out - my @SN; - my @CN; - xCAT::ServiceNodeUtils->getSNandCPnodes(\@$nodes, \@SN, \@CN); - unless (($args[0] eq 'stat') or ($args[0] eq 'enact')) { # ok for these options - if ((@SN > 0) && (@CN >0 )) { # there are both SN and CN - my $rsp; - $rsp->{data}->[0] = - "Nodeset was run with a noderange containing both service nodes and compute nodes. This is not valid. You must submit with either compute nodes in the noderange or service nodes. \n"; - xCAT::MsgUtils->message("E", $rsp, $callback1); - return; - - } - } - - $req->{'_disparatetftp'}=[1]; - if ($req->{inittime}->[0]) { - return [$req]; - } - if (@CN >0 ) { # if compute nodes broadcast to all servicenodes - return xCAT::Scope->get_broadcast_scope($req,@_); - } - } - return [$req]; -} - -sub process_request { - $::PXE_request = shift; - $::PXE_callback = shift; - my $sub_req = shift; - undef $::PXE_addkcmdlinehandled; - my @args; - my @nodes; - my @rnodes; - - #>>>>>>>used for trace log start>>>>>>> - my %opt; - my $verbose_on_off=0; - if (ref($::PXE_request->{arg})) { - @args=@{$::PXE_request->{arg}}; - } else { - @args=($::PXE_request->{arg}); - } - @ARGV = @args; - GetOptions('V' => \$opt{V}); - if($opt{V}){$verbose_on_off=1;} - #>>>>>>>used for trace log end>>>>>>> - - if (ref($::PXE_request->{node})) { - @rnodes = @{$::PXE_request->{node}}; - } else { - if ($::PXE_request->{node}) { @rnodes = ($::PXE_request->{node}); } - } - - unless (@rnodes) { - if ($usage{$::PXE_request->{command}->[0]}) { - $::PXE_callback->({data=>$usage{$::PXE_request->{command}->[0]}}); - } - return; - } - - #if not shared, then help sync up - if ($::PXE_request->{'_disparatetftp'}->[0]) { #reading hint from preprocess_command - @nodes = (); - foreach (@rnodes) { - if (xCAT::NetworkUtils->nodeonmynet($_)) { - push @nodes,$_; - } else { - xCAT::MsgUtils->message("S", "$_: pxe netboot: stop configuration because of none sharedtftp and not on same network with its xcatmaster."); - } - } - } else { - @nodes = @rnodes; - } - - #>>>>>>>used for trace log>>>>>>> - my $str_node = join(" ",@nodes); - xCAT::MsgUtils->trace($verbose_on_off,"d","pxe: nodes are $str_node"); - - # return directly if no nodes in the same network - unless (@nodes) { - xCAT::MsgUtils->message("S", "xCAT: pxe netboot: no valid nodes. Stop the operation on this server."); - return; - } - - if (ref($::PXE_request->{arg})) { - @args=@{$::PXE_request->{arg}}; - } else { - @args=($::PXE_request->{arg}); - } - - #now run the begin part of the prescripts - unless ($args[0] eq 'stat') { # or $args[0] eq 'enact') { - $errored=0; - if ($::PXE_request->{'_disparatetftp'}->[0]) { #the call is distrubuted to the service node already, so only need to handles my own children - xCAT::MsgUtils->trace($verbose_on_off,"d","pxe: the call is distrubuted to the service node already, so only need to handles my own children"); - xCAT::MsgUtils->trace($verbose_on_off,"d","pxe: issue runbeginpre request"); - $sub_req->({command=>['runbeginpre'], - node=>\@nodes, - arg=>[$args[0], '-l']},\&pass_along); - } else { #nodeset did not distribute to the service node, here we need to let runednpre to distribute the nodes to their masters - xCAT::MsgUtils->trace($verbose_on_off,"d","pxe: nodeset did not distribute to the service node"); - xCAT::MsgUtils->trace($verbose_on_off,"d","pxe: issue runbeginpre request"); - $sub_req->({command=>['runbeginpre'], - node=>\@rnodes, - arg=>[$args[0]]},\&pass_along); - } - if ($errored) { - my $rsp; - $rsp->{errorcode}->[0]=1; - $rsp->{error}->[0]="Failed in running begin prescripts. Processing will still continue.\n"; - $::PXE_callback->($rsp); - } - } - -#end prescripts code - if (! -r "$tftpdir/pxelinux.0") { - unless (-r "/usr/lib/syslinux/pxelinux.0" or -r "/usr/share/syslinux/pxelinux.0") { - $::PXE_callback->({error=>["Unable to find pxelinux.0 "],errorcode=>[1]}); - return; - } - if (-r "/usr/lib/syslinux/pxelinux.0") { - copy("/usr/lib/syslinux/pxelinux.0","$tftpdir/pxelinux.0"); - } else { - copy("/usr/share/syslinux/pxelinux.0","$tftpdir/pxelinux.0"); - } - chmod(0644,"$tftpdir/pxelinux.0"); - } - unless ( -r "$tftpdir/pxelinux.0" ) { - $::PXE_callback->({errror=>["Unable to find pxelinux.0 from syslinux"],errorcode=>[1]}); - return; - } - - - $errored=0; - my $inittime=0; - if (exists($::PXE_request->{inittime})) { $inittime= $::PXE_request->{inittime}->[0];} - if (!$inittime) { $inittime=0;} - unless ($args[0] eq 'stat') { # or $args[0] eq 'enact') { - xCAT::MsgUtils->trace($verbose_on_off,"d","pxe: issue setdestiny request"); - $sub_req->({command=>['setdestiny'], - node=>\@nodes, - inittime=>[$inittime], - arg=>\@args},\&pass_along); - } - if ($errored) { return; } - #Time to actually configure the nodes, first extract database data with the scalable calls - my $bptab = xCAT::Table->new('bootparams',-create=>1); - my $chaintab = xCAT::Table->new('chain'); - my $mactab = xCAT::Table->new('mac'); #to get all the hostnames - my $typetab = xCAT::Table->new('nodetype'); - my $restab = xCAT::Table->new('noderes'); - my $linuximgtab=xCAT::Table->new('linuximage',-create=>1); - my %nrhash = %{$restab->getNodesAttribs(\@nodes,[qw(tftpdir)])}; - my %bphash = %{$bptab->getNodesAttribs(\@nodes,[qw(kernel initrd kcmdline addkcmdline)])}; - my %chainhash = %{$chaintab->getNodesAttribs(\@nodes,[qw(currstate)])}; - my %machash = %{$mactab->getNodesAttribs(\@nodes,[qw(mac)])}; - my %nthash = %{$typetab->getNodesAttribs(\@nodes,[qw(os provmethod)])}; - foreach (@nodes) { - my %response; - my $tftpdir; - if ($nrhash{$_} and $nrhash{$_}->[0] and $nrhash{$_}->[0]->{tftpdir}) { - $tftpdir = $nrhash{$_}->[0]->{tftpdir}; - } else { - $tftpdir = $globaltftpdir; - } - $response{node}->[0]->{name}->[0]=$_; - if ($args[0] eq 'stat') { - $response{node}->[0]->{data}->[0]= getstate($_,$tftpdir); - $::PXE_callback->(\%response); - } elsif ($args[0]) { #If anything else, send it on to the destiny plugin, then setstate - my $ent = $nthash{$_}->[0]; - my $osimgname = $ent->{'provmethod'}; - my $linuximghash=undef; - unless($osimgname =~ /^(install|netboot|statelite)$/){ - $linuximghash = $linuximgtab->getAttribs({imagename => $osimgname}, 'boottarget', 'addkcmdline'); - } - ($rc,$errstr) = setstate($_,\%bphash,\%chainhash,\%machash,\%nthash,$tftpdir,$linuximghash); - if ($rc) { - $response{node}->[0]->{errorcode}->[0]= $rc; - $response{node}->[0]->{errorc}->[0]= $errstr; - $::PXE_callback->(\%response); - } - } - } - - my $inittime=0; - if (exists($::PXE_request->{inittime})) { $inittime= $::PXE_request->{inittime}->[0];} - if (!$inittime) { $inittime=0;} - - #dhcp stuff -- inittime is set when xcatd on sn is started - unless (($args[0] eq 'stat') || ($inittime) || ($args[0] eq 'offline')) { - my $do_dhcpsetup=1; - #my $sitetab = xCAT::Table->new('site'); - #if ($sitetab) { - #(my $ref) = $sitetab->getAttribs({key => 'dhcpsetup'}, 'value'); - my @entries = xCAT::TableUtils->get_site_attribute("dhcpsetup"); - my $t_entry = $entries[0]; - if ( defined($t_entry) ) { - if ($t_entry =~ /0|n|N/) { $do_dhcpsetup=0; } - } - #} - - if ($do_dhcpsetup) { - if ($::PXE_request->{'_disparatetftp'}->[0]) { #reading hint from preprocess_command - xCAT::MsgUtils->trace($verbose_on_off,"d","pxe: issue makedhcp request"); - $sub_req->({command=>['makedhcp'],arg=>['-l'], - node=>\@nodes},$::PXE_callback); - } else { - xCAT::MsgUtils->trace($verbose_on_off,"d","pxe: issue makedhcp request"); - $sub_req->({command=>['makedhcp'], - node=>\@nodes},$::PXE_callback); - } - } - - } - #unlink the files for 'offline' command - if($args[0] eq 'offline') { - foreach my $node (@nodes) { - my %ipaddrs; - unless (inet_aton($node)) { - syslog("local4|err","xCAT unable to resolve IP in pxe plugin"); - return; - } - my $ip = inet_ntoa(inet_aton($node));; - unless ($ip) { - syslog("local4|err","xCAT unable to resolve IP in pxe plugin"); - return; - } - $ipaddrs{$ip} = 1; - if ($mactab) { - my $ment = $machash{$node}->[0]; #$mactab->getNodeAttribs($node,['mac']); - if ($ment and $ment->{mac}) { - my @macs = split(/\|/,$ment->{mac}); - foreach (@macs) { - if (/!(.*)/) { - if (inet_aton($1)) { - $ipaddrs{inet_ntoa(inet_aton($1))} = 1; - } - } - } - } - } - my $hassymlink = eval { symlink("",""); 1 }; - - unlink($tftpdir."/armgrub/".$node); - - foreach $ip (keys %ipaddrs) { - my @ipa=split(/\./,$ip); - my $pname = sprintf("%02X%02X%02X%02X",@ipa); - unlink($tftpdir."/armgrub/".$pname); - #if ($hassymlink) { - #symlink($node,$tftpdir."/armgrub/".$pname); - #} else { - #link($tftpdir."/armgrub/".$node,$tftpdir."/armgrub/".$pname); - #} - } - } - } - - #now run the end part of the prescripts - unless ($args[0] eq 'stat') { # or $args[0] eq 'enact') - $errored=0; - if ($::PXE_request->{'_disparatetftp'}->[0]) { #the call is distrubuted to the service node already, so only need to handles my own children - xCAT::MsgUtils->trace($verbose_on_off,"d","pxe: issue runendpre request"); - $sub_req->({command=>['runendpre'], - node=>\@nodes, - arg=>[$args[0], '-l']},\&pass_along); - } else { #nodeset did not distribute to the service node, here we need to let runednpre to distribute the nodes to their masters - xCAT::MsgUtils->trace($verbose_on_off,"d","pxe: issue runendpre request"); - $sub_req->({command=>['runendpre'], - node=>\@rnodes, - arg=>[$args[0]]},\&pass_along); - } - if ($errored) { - my $rsp; - $rsp->{errorcode}->[0]=1; - $rsp->{error}->[0]="Failed in running end prescripts. Processing will still continue.\n"; - $::PXE_callback->($rsp); - } - } - - - -} - - -#---------------------------------------------------------------------------- -=head3 getNodesetStates - returns the nodeset state for the given nodes. The possible nodeset - states are: netboot, install, boot and discover. - Arguments: - nodes --- a pointer to an array of nodes - states -- a pointer to a hash table. This hash will be filled by this - function. The key is the nodeset status and the value is a pointer - to an array of nodes. - Returns: - (return code, error message) -=cut -#----------------------------------------------------------------------------- -sub getNodesetStates { - my $noderef=shift; - if ($noderef =~ /xCAT_plugin::pxe/) { - $noderef=shift; - } - my @nodes=@$noderef; - my $hashref=shift; - my $noderestab = xCAT::Table->new('noderes'); #in order to detect per-node tftp directories - my %nrhash = %{$noderestab->getNodesAttribs(\@nodes,[qw(tftpdir)])}; - if (@nodes>0) { - foreach my $node (@nodes) { - my $tftpdir; - if ($nrhash{$node}->[0] and $nrhash{$node}->[0]->{tftpdir}) { - $tftpdir = $nrhash{$node}->[0]->{tftpdir}; - } else { - $tftpdir = $globaltftpdir; - } - my $tmp=getstate($node, $tftpdir); - my @a=split(' ', $tmp); - $stat = $a[0]; - if (exists($hashref->{$stat})) { - my $pa=$hashref->{$stat}; - push(@$pa, $node); - } - else { - $hashref->{$stat}=[$node]; - } - } - } - return (0, ""); -} - -1; diff --git a/xCAT-server/lib/xcat/plugins/dhcp.pm b/xCAT-server/lib/xcat/plugins/dhcp.pm index f9e67075e..05dd98d9e 100644 --- a/xCAT-server/lib/xcat/plugins/dhcp.pm +++ b/xCAT-server/lib/xcat/plugins/dhcp.pm @@ -2742,7 +2742,7 @@ sub addnet push @netent, " filename \"elilo.efi\";\n"; push @netent, " } else if option client-architecture = 00:0b { #aaarch64\n "; - push @netent, " filename \"armgrub/grubaa64.efi\";\n"; + push @netent, " filename \"boot/grub2/grub2.aarch64\";\n"; push @netent, " } else if option client-architecture = 00:0e { #OPAL-v3\n "; push @netent, " option conf-file = \"http://$tftp:$httpport/tftpboot/pxelinux.cfg/p/" . $net . "_" . $maskbits . "\";\n"; From bf57a2b13ef5978a3e05076b195f01dc3f2ddb35 Mon Sep 17 00:00:00 2001 From: Markus Hilger Date: Thu, 19 May 2022 15:51:42 +0200 Subject: [PATCH 06/13] Build xcat-core for aarch64 --- buildcore.sh | 4 +++- buildlocal.sh | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/buildcore.sh b/buildcore.sh index eb5da381e..0288929a1 100755 --- a/buildcore.sh +++ b/buildcore.sh @@ -371,6 +371,8 @@ if [ "$OSNAME" != "AIX" ]; then if [ $? -ne 0 ]; then FAILEDRPMS="$FAILEDRPMS xCAT-genesis-scripts-x86_64"; fi ./makerpm xCAT-genesis-scripts ppc64 "$EMBED" if [ $? -ne 0 ]; then FAILEDRPMS="$FAILEDRPMS xCAT-genesis-scripts-ppc64"; fi + ./makerpm xCAT-genesis-scripts aarch64 "$EMBED" + if [ $? -ne 0 ]; then FAILEDRPMS="$FAILEDRPMS xCAT-genesis-scripts-aarch64"; fi if [ "$FAILEDRPMS" = "$ORIGFAILEDRPMS" ]; then # all succeeded rm -f $DESTDIR/xCAT-genesis-scripts*rpm rm -f $SRCDIR/xCAT-genesis-scripts*rpm @@ -392,7 +394,7 @@ for rpmname in xCAT xCATsn; do ./makerpm $rpmname "$EMBED" if [ $? -ne 0 ]; then FAILEDRPMS="$FAILEDRPMS $rpmname"; fi else - for arch in x86_64 ppc64 ppc64le s390x; do + for arch in x86_64 ppc64 ppc64le s390x aarch64; do if [ "$rpmname" = "xCAT-OpenStack" -a "$arch" != "x86_64" ] || [ "$rpmname" = "xCAT-OpenStack-baremetal" -a "$arch" != "x86_64" ] ; then continue; fi # only bld openstack for x86_64 for now ./makerpm $rpmname $arch "$EMBED" if [ $? -ne 0 ]; then FAILEDRPMS="$FAILEDRPMS $rpmname-$arch"; fi diff --git a/buildlocal.sh b/buildlocal.sh index 54bd06a86..853471758 100755 --- a/buildlocal.sh +++ b/buildlocal.sh @@ -123,7 +123,7 @@ echo "This is an $OSNAME system" $CURDIR/makerpm $rpmname if [ $? -ne 0 ]; then FAILEDRPMS="$FAILEDRPMS $rpmname"; fi else - for arch in x86_64 ppc64 s390x; do + for arch in x86_64 ppc64 s390x aarch64; do $CURDIR/makerpm $rpmname $arch if [ $? -ne 0 ]; then FAILEDRPMS="$FAILEDRPMS $rpmname-$arch"; fi done From 295dff5c51e687dfdac7551c88717b56199bf813 Mon Sep 17 00:00:00 2001 From: Markus Hilger Date: Thu, 19 May 2022 18:09:56 +0200 Subject: [PATCH 07/13] Fix genesis builds for aarch64 --- xCAT-genesis-builder/xCAT-genesis-base.spec | 3 +++ xCAT-genesis-scripts/xCAT-genesis-scripts.spec | 3 +++ 2 files changed, 6 insertions(+) diff --git a/xCAT-genesis-builder/xCAT-genesis-base.spec b/xCAT-genesis-builder/xCAT-genesis-base.spec index 03b66047b..54f157017 100644 --- a/xCAT-genesis-builder/xCAT-genesis-base.spec +++ b/xCAT-genesis-builder/xCAT-genesis-base.spec @@ -10,6 +10,9 @@ Release: %{?release:%{release}}%{!?release:snap%(date +"%Y%m%d%H%M")} %ifarch ppc ppc64 ppc64le %define tarch ppc64 %endif +%ifarch aarch64 +%define tarch aarch64 +%endif BuildArch: noarch %define name xCAT-genesis-base-%{tarch} %define __spec_install_post : diff --git a/xCAT-genesis-scripts/xCAT-genesis-scripts.spec b/xCAT-genesis-scripts/xCAT-genesis-scripts.spec index 48edf3267..a0384f6f2 100755 --- a/xCAT-genesis-scripts/xCAT-genesis-scripts.spec +++ b/xCAT-genesis-scripts/xCAT-genesis-scripts.spec @@ -7,6 +7,9 @@ %ifarch ppc ppc64 %define tarch ppc64 %endif +%ifarch aarch64 +%define tarch aarch64 +%endif %define rpminstallroot /opt/xcat/share/xcat/netboot/genesis/%{tarch}/fs BuildArch: noarch %define name xCAT-genesis-scripts-%{tarch} From 02aaa24c54d1e5fb34c6d72b58b542cdefbae77c Mon Sep 17 00:00:00 2001 From: Markus Hilger Date: Fri, 20 May 2022 10:45:20 +0200 Subject: [PATCH 08/13] Do not build xCAT-genesis-scripts-aarch64 yet --- buildcore.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/buildcore.sh b/buildcore.sh index 0288929a1..ddb7e4d52 100755 --- a/buildcore.sh +++ b/buildcore.sh @@ -371,8 +371,9 @@ if [ "$OSNAME" != "AIX" ]; then if [ $? -ne 0 ]; then FAILEDRPMS="$FAILEDRPMS xCAT-genesis-scripts-x86_64"; fi ./makerpm xCAT-genesis-scripts ppc64 "$EMBED" if [ $? -ne 0 ]; then FAILEDRPMS="$FAILEDRPMS xCAT-genesis-scripts-ppc64"; fi - ./makerpm xCAT-genesis-scripts aarch64 "$EMBED" - if [ $? -ne 0 ]; then FAILEDRPMS="$FAILEDRPMS xCAT-genesis-scripts-aarch64"; fi + # Do not build xCAT-genesis-scripts-aarch64 yet + #./makerpm xCAT-genesis-scripts aarch64 "$EMBED" + #if [ $? -ne 0 ]; then FAILEDRPMS="$FAILEDRPMS xCAT-genesis-scripts-aarch64"; fi if [ "$FAILEDRPMS" = "$ORIGFAILEDRPMS" ]; then # all succeeded rm -f $DESTDIR/xCAT-genesis-scripts*rpm rm -f $SRCDIR/xCAT-genesis-scripts*rpm From b27f419b253ec7cde373435b3588895f6dfbab9d Mon Sep 17 00:00:00 2001 From: Markus Hilger Date: Tue, 24 May 2022 22:51:18 +0200 Subject: [PATCH 09/13] qemu-user-static support for genimage + aarch64 support --- xCAT-server/share/xcat/netboot/rh/genimage | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/xCAT-server/share/xcat/netboot/rh/genimage b/xCAT-server/share/xcat/netboot/rh/genimage index 166a38247..6063bba3d 100755 --- a/xCAT-server/share/xcat/netboot/rh/genimage +++ b/xCAT-server/share/xcat/netboot/rh/genimage @@ -277,6 +277,8 @@ if ($netdriver) { # Add the default driver list if ($arch eq 'x86' or $arch eq 'x86_64') { push @ndrivers, qw/tg3 bnx2 bnx2x e1000 e1000e igb mlx_en virtio_net be2net/; +} elsif ($arch eq 'aarch64') { + push @ndrivers, qw/tg3 bnx2 bnx2x e1000e igb mlx_en virtio_net/; } elsif ($arch eq 'ppc64') { push @ndrivers, qw/e1000 e1000e igb ibmveth ehea/; } elsif ($arch eq 's390x') { @@ -330,8 +332,20 @@ if($onlyinitrd){ open($yumconfig, ">", "/tmp/genimage.$$.yum.conf"); #yum/rpm/zypper has defect on calculating diskspace usage when installing rpm on a NFS mounted installroot + my $main_section = 0; if (isNFSdir("$rootimg_dir")) { print $yumconfig "[main]\ndiskspacecheck=0\n\n"; + $main_section = 1; + } + + # try to force non-native arch and use qemu-user-static if available + my $host_arch = `uname -m`; + chomp($host_arch); + $host_arch = "x86" if ($arch =~ /i.86$/); + if ($host_arch ne $arch) { + print $yumconfig "[main]\n" unless ($main_section); + print $yumconfig "arch=$arch\nignorearch=True\n\n"; + $main_section = 1; } my $repnum = 0; @@ -1843,7 +1857,7 @@ EOMS } } - if ($arch =~ /x86_64/) { + if ($arch =~ /x86_64/ or $arch =~ /aarch64/) { push @filestoadd, "lib64/libnss_dns.so.2"; push @filestoadd, "lib64/libresolv.so.2"; } else { From 0d2c97022c31edb4afb2a8a31ff05b97667c59c2 Mon Sep 17 00:00:00 2001 From: Markus Hilger Date: Wed, 21 Feb 2024 12:04:18 +0100 Subject: [PATCH 10/13] Update man pages with aarch64 and qemu-user-static --- perl-xCAT/xCAT/Schema.pm | 7 ++++--- xCAT-client/pods/man1/genimage.1.pod | 4 +++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/perl-xCAT/xCAT/Schema.pm b/perl-xCAT/xCAT/Schema.pm index 9a46920d8..69c36e5cb 100755 --- a/perl-xCAT/xCAT/Schema.pm +++ b/perl-xCAT/xCAT/Schema.pm @@ -678,11 +678,12 @@ passed as argument rather than by table value', netboot => 'The type of network booting to use for this node. Valid values: Arch OS valid netboot options - x86, x86_64 ALL pxe, xnba + x86, x86_64 ALL pxe, xnba, grub2 ppc64 <=rhel6, <=sles11.3 yaboot ppc64 >=rhels7, >=sles11.4 grub2,grub2-http,grub2-tftp ppc64le NonVirtualize ALL petitboot ppc64le PowerKVM Guest ALL grub2,grub2-http,grub2-tftp + aarch64 >=el8 grub2 ', tftpserver => 'The TFTP server for this node (as known by this node). If not set, it defaults to networks.tftpserver.', @@ -757,7 +758,7 @@ passed as argument rather than by table value', descriptions => { node => 'The node name or group name.', os => 'The operating system deployed on this node. Valid values: AIX, rhels*,rhelc*, rhas*,centos*, alma*, rocky*,SL*, fedora*, sles* (where * is the version #). As a special case, if this is set to "boottarget", then it will use the initrd/kernel/parameters specified in the row in the boottarget table in which boottarget.bprofile equals nodetype.profile.', - arch => 'The hardware architecture of this node. Valid values: x86_64, ppc64, x86, ia64.', + arch => 'The hardware architecture of this node. Valid values: x86_64, ppc64, x86, ia64, aarch64.', profile => 'The string to use to locate a kickstart or autoyast template to use for OS deployment of this node. If the provmethod attribute is set to an osimage name, that takes precedence, and profile need not be defined. Otherwise, the os, profile, and arch are used to search for the files in /install/custom first, and then in /opt/xcat/share/xcat.', provmethod => 'The provisioning method for node deployment. The valid values are install, netboot, statelite or an os image name from the osimage table. If an image name is specified, the osimage definition stored in the osimage table and the linuximage table (for Linux) or nimimage table (for AIX) are used to locate the files for templates, pkglists, syncfiles, etc. On Linux, if install, netboot or statelite is specified, the os, profile, and arch are used to search for the files in /install/custom first, and then in /opt/xcat/share/xcat.', supportedarchs => 'Comma delimited list of architectures this node can execute.', @@ -800,7 +801,7 @@ passed as argument rather than by table value', profile => 'The node usage category. For example compute, service.', osname => 'Operating system name- AIX or Linux.', osvers => 'The Linux operating system deployed on this node. Valid values: rhels*,rhelc*, rhas*,centos*,alma*, rocky*,SL*, fedora*, sles* (where * is the version #).', - osarch => 'The hardware architecture of this node. Valid values: x86_64, ppc64, x86, ia64.', + osarch => 'The hardware architecture of this node. For netboot/statelite images, QEMU emulation for non-native architectures is used if qemu-user-static is installed and configured via systemd-binfmt. Valid values: x86_64, ppc64, x86, ia64, aarch64.', synclists => 'The fully qualified name of a file containing a list of files to synchronize on the nodes. Can be a comma separated list of multiple synclist files. The synclist generated by PCM named /install/osimages//synclist.cfm is reserved for use only by PCM and should not be edited by the admin.', postscripts => 'Comma separated list of scripts that should be run on this image after diskful installation or diskless boot. For installation of RedHat, CentOS, Fedora, the scripts will be run before the reboot. For installation of SLES, the scripts will be run after the reboot but before the init.d process. For diskless deployment, the scripts will be run at the init.d time, and xCAT will automatically add the list of scripts from the postbootscripts attribute to run after postscripts list. For installation of AIX, the scripts will run after the reboot and acts the same as the postbootscripts attribute. For AIX, use the postbootscripts attribute. See the site table runbootscripts attribute.', postbootscripts => 'Comma separated list of scripts that should be run on this after diskful installation or diskless boot. On AIX these scripts are run during the processing of /etc/inittab. On Linux they are run at the init.d time. xCAT automatically adds the scripts in the xcatdefaults.postbootscripts attribute to run first in the list. See the site table runbootscripts attribute.', diff --git a/xCAT-client/pods/man1/genimage.1.pod b/xCAT-client/pods/man1/genimage.1.pod index 8145a637e..8ecf47baa 100644 --- a/xCAT-client/pods/man1/genimage.1.pod +++ b/xCAT-client/pods/man1/genimage.1.pod @@ -32,7 +32,9 @@ Besides prompting for some parameter values, the B command takes defau If B<--onlyinitrd> is specified, genimage only regenerates the initrd for a stateless image to be used for a diskless install. -The B command must be run on a system that is the same architecture and same distro with same major release version as the nodes it will be used on. If the management node is not the same architecture or same distro level, copy the contents of +The B command must be run on a system that is the same architecture and same distro with same major release version as the nodes it will be used on. +For different architectures, B will try to use QEMU emulation if qemu-user-static is installed and configured via systemd-binfmt. +If this does not work or the management node is not the same distro level, copy the contents of /opt/xcat/share/xcat/netboot/ to a system that is the proper architecture, and mount /install from the management node to that system. Then change directory to /opt/xcat/share/xcat/netboot/ and run ./genimage. From 857e2909d9ae9727205750bbe19f6a44ef263679 Mon Sep 17 00:00:00 2001 From: Markus Hilger Date: Fri, 12 Jul 2024 17:12:20 +0200 Subject: [PATCH 11/13] Update man pages for aarch64 --- .../guides/admin-guides/basic_concepts/xcat_object/node.rst | 6 ++++-- .../guides/admin-guides/references/man1/genimage.1.rst | 6 ++++-- .../guides/admin-guides/references/man5/noderes.5.rst | 3 ++- .../guides/admin-guides/references/man5/nodetype.5.rst | 2 +- .../guides/admin-guides/references/man5/osimage.5.rst | 2 +- docs/source/guides/admin-guides/references/man7/group.7.rst | 5 +++-- docs/source/guides/admin-guides/references/man7/node.7.rst | 5 +++-- .../guides/admin-guides/references/man7/osimage.7.rst | 2 +- xCAT-client/pods/man1/genimage.1.pod | 2 +- 9 files changed, 20 insertions(+), 13 deletions(-) diff --git a/docs/source/guides/admin-guides/basic_concepts/xcat_object/node.rst b/docs/source/guides/admin-guides/basic_concepts/xcat_object/node.rst index ce02b697e..15f3d48d3 100644 --- a/docs/source/guides/admin-guides/basic_concepts/xcat_object/node.rst +++ b/docs/source/guides/admin-guides/basic_concepts/xcat_object/node.rst @@ -13,7 +13,7 @@ Key Attributes The operating system deployed on this node. Valid values: AIX, rhels*, rhelc*, rhas*, centos*, SL*, fedora*, sles* (where * is the version #) * arch: - The hardware architecture of this node. Valid values: x86_64, ppc64, x86, ia64. + The hardware architecture of this node. Valid values: x86_64, ppc64, x86, ia64, aarch64. * groups: Usually, there are a set of nodes with some attributes in common, xCAT admin can define a node group containing these nodes, so that the management task can be issued against the group instead of individual nodes. A node can be a member of different groups, so the value of this attributes is a comma-delimited list of groups. At least one group is required to create a node. The new created group names should not be prefixed with "__" as this token has been preserved as the internal group name. @@ -33,7 +33,7 @@ Key Attributes +--------------------------+----------------------+-----------------------------------+ | Arch and Machine Type | OS | valid netboot options | +==========================+======================+===================================+ - | x86, x86_64 | ALL | pxe, xnba | + | x86, x86_64 | ALL | pxe, xnba, grub2 | +--------------------------+----------------------+-----------------------------------+ | ppc64 | <=rhel6, <=sles11.3 | yaboot | +--------------------------+----------------------+-----------------------------------+ @@ -42,6 +42,8 @@ Key Attributes | ppc64le NonVirtualize | ALL | petitboot | +--------------------------+----------------------+-----------------------------------+ | ppc64le PowerKVM Guest | ALL | grub2,grub2-http,grub2-tftp | + +--------------------------+----------------------+-----------------------------------+ + | aarch64 | >=el8 | grub2 | +-------------------------------------------------+-----------------------------------+ * postscripts: diff --git a/docs/source/guides/admin-guides/references/man1/genimage.1.rst b/docs/source/guides/admin-guides/references/man1/genimage.1.rst index 85107fe69..1d13e6ede 100644 --- a/docs/source/guides/admin-guides/references/man1/genimage.1.rst +++ b/docs/source/guides/admin-guides/references/man1/genimage.1.rst @@ -49,7 +49,9 @@ Besides prompting for some parameter values, the \ **genimage**\ command takes If \ **-**\ **-onlyinitrd**\ is specified, genimage only regenerates the initrd for a stateless image to be used for a diskless install. -The \ **genimage**\ command must be run on a system that is the same architecture and same distro with same major release version as the nodes it will be used on. If the management node is not the same architecture or same distro level, copy the contents of +The \ **genimage**\ command must be run on a system that is the same architecture and same distro with same major release version as the nodes it will be used on. +For different architectures, \ **genimage**\ will try to use QEMU emulation if qemu-user-static is installed and configured via systemd-binfmt. +If this does not work or the management node is not the same distro level, copy the contents of /opt/xcat/share/xcat/netboot/ to a system that is the proper architecture, and mount /install from the management node to that system. Then change directory to /opt/xcat/share/xcat/netboot/ and run ./genimage. @@ -70,7 +72,7 @@ OPTIONS \ **-a**\ \ *arch*\ - The hardware architecture of this node: ppc64le, x86_64, ppc64, x86, ia64, etc. If omitted, the current hardware architecture will be used. + The hardware architecture of this node: ppc64le, x86_64, ppc64, x86, ia64, aarch64 etc. If omitted, the current hardware architecture will be used. diff --git a/docs/source/guides/admin-guides/references/man5/noderes.5.rst b/docs/source/guides/admin-guides/references/man5/noderes.5.rst index 98bc13b24..67bb5fde7 100644 --- a/docs/source/guides/admin-guides/references/man5/noderes.5.rst +++ b/docs/source/guides/admin-guides/references/man5/noderes.5.rst @@ -56,11 +56,12 @@ noderes Attributes: .. code-block:: perl Arch OS valid netboot options - x86, x86_64 ALL pxe, xnba + x86, x86_64 ALL pxe, xnba, grub2 ppc64 <=rhel6, <=sles11.3 yaboot ppc64 >=rhels7, >=sles11.4 grub2,grub2-http,grub2-tftp ppc64le NonVirtualize ALL petitboot ppc64le PowerKVM Guest ALL grub2,grub2-http,grub2-tftp + aarch64 >=el8 grub2 diff --git a/docs/source/guides/admin-guides/references/man5/nodetype.5.rst b/docs/source/guides/admin-guides/references/man5/nodetype.5.rst index 311832594..72e6d6f4b 100644 --- a/docs/source/guides/admin-guides/references/man5/nodetype.5.rst +++ b/docs/source/guides/admin-guides/references/man5/nodetype.5.rst @@ -50,7 +50,7 @@ nodetype Attributes: \ **arch**\ - The hardware architecture of this node. Valid values: x86_64, ppc64, x86, ia64. + The hardware architecture of this node. Valid values: x86_64, ppc64, x86, ia64, aarch64. diff --git a/docs/source/guides/admin-guides/references/man5/osimage.5.rst b/docs/source/guides/admin-guides/references/man5/osimage.5.rst index 959007aa6..b5bf84f82 100644 --- a/docs/source/guides/admin-guides/references/man5/osimage.5.rst +++ b/docs/source/guides/admin-guides/references/man5/osimage.5.rst @@ -110,7 +110,7 @@ osimage Attributes: \ **osarch**\ - The hardware architecture of this node. Valid values: x86_64, ppc64, x86, ia64. + The hardware architecture of this node. For netboot/statelite images, QEMU emulation for non-native architectures is used if qemu-user-static is installed and configured via systemd-binfmt. Valid values: x86_64, ppc64, x86, ia64, aarch64. diff --git a/docs/source/guides/admin-guides/references/man7/group.7.rst b/docs/source/guides/admin-guides/references/man7/group.7.rst index c8f339e5a..8865f8347 100644 --- a/docs/source/guides/admin-guides/references/man7/group.7.rst +++ b/docs/source/guides/admin-guides/references/man7/group.7.rst @@ -47,7 +47,7 @@ group Attributes: \ **arch**\ (nodetype.arch) - The hardware architecture of this node. Valid values: x86_64, ppc64, x86, ia64. + The hardware architecture of this node. Valid values: x86_64, ppc64, x86, ia64, aarch64. @@ -591,11 +591,12 @@ group Attributes: .. code-block:: perl Arch OS valid netboot options - x86, x86_64 ALL pxe, xnba + x86, x86_64 ALL pxe, xnba, grub2 ppc64 <=rhel6, <=sles11.3 yaboot ppc64 >=rhels7, >=sles11.4 grub2,grub2-http,grub2-tftp ppc64le NonVirtualize ALL petitboot ppc64le PowerKVM Guest ALL grub2,grub2-http,grub2-tftp + aarch64 >=el8 grub2 diff --git a/docs/source/guides/admin-guides/references/man7/node.7.rst b/docs/source/guides/admin-guides/references/man7/node.7.rst index 48ca38dc1..163702384 100644 --- a/docs/source/guides/admin-guides/references/man7/node.7.rst +++ b/docs/source/guides/admin-guides/references/man7/node.7.rst @@ -59,7 +59,7 @@ node Attributes: \ **arch**\ (nodetype.arch) - The hardware architecture of this node. Valid values: x86_64, ppc64, x86, ia64. + The hardware architecture of this node. Valid values: x86_64, ppc64, x86, ia64, aarch64. @@ -591,11 +591,12 @@ node Attributes: .. code-block:: perl Arch OS valid netboot options - x86, x86_64 ALL pxe, xnba + x86, x86_64 ALL pxe, xnba, grub2 ppc64 <=rhel6, <=sles11.3 yaboot ppc64 >=rhels7, >=sles11.4 grub2,grub2-http,grub2-tftp ppc64le NonVirtualize ALL petitboot ppc64le PowerKVM Guest ALL grub2,grub2-http,grub2-tftp + aarch64 >=el8 grub2 diff --git a/docs/source/guides/admin-guides/references/man7/osimage.7.rst b/docs/source/guides/admin-guides/references/man7/osimage.7.rst index cf2767a56..ad59cdcab 100644 --- a/docs/source/guides/admin-guides/references/man7/osimage.7.rst +++ b/docs/source/guides/admin-guides/references/man7/osimage.7.rst @@ -225,7 +225,7 @@ osimage Attributes: \ **osarch**\ (osimage.osarch) - The hardware architecture of this node. Valid values: x86_64, ppc64, x86, ia64. + The hardware architecture of this node. For netboot/statelite images, QEMU emulation for non-native architectures is used if qemu-user-static is installed and configured via systemd-binfmt. Valid values: x86_64, ppc64, x86, ia64, aarch64. diff --git a/xCAT-client/pods/man1/genimage.1.pod b/xCAT-client/pods/man1/genimage.1.pod index 8ecf47baa..60d9c78be 100644 --- a/xCAT-client/pods/man1/genimage.1.pod +++ b/xCAT-client/pods/man1/genimage.1.pod @@ -50,7 +50,7 @@ I specifies the name of an os image definition to be used. The specif =item B<-a> I -The hardware architecture of this node: ppc64le, x86_64, ppc64, x86, ia64, etc. If omitted, the current hardware architecture will be used. +The hardware architecture of this node: ppc64le, x86_64, ppc64, x86, ia64, aarch64 etc. If omitted, the current hardware architecture will be used. =item B<-o> I From 8329cf6631505650232c7ee6d7dc7cd96ba7d816 Mon Sep 17 00:00:00 2001 From: Markus Hilger Date: Fri, 12 Jul 2024 18:45:49 +0200 Subject: [PATCH 12/13] Update documentation for aarch64 and grub2 support --- .../building_stateless_images.rst | 22 ++++++ docs/source/advanced/mixed_cluster/index.rst | 2 +- .../advanced/mixed_cluster/support_matrix.rst | 77 ++++++++++--------- .../guides/install-guides/yum/grub2.rst | 17 ++++ .../guides/install-guides/yum/install.rst | 1 + docs/source/overview/differentiators.rst | 2 +- docs/source/overview/support_matrix.rst | 38 ++++----- 7 files changed, 100 insertions(+), 59 deletions(-) create mode 100644 docs/source/guides/install-guides/yum/grub2.rst diff --git a/docs/source/advanced/mixed_cluster/building_stateless_images.rst b/docs/source/advanced/mixed_cluster/building_stateless_images.rst index daec246f5..b08f855e0 100644 --- a/docs/source/advanced/mixed_cluster/building_stateless_images.rst +++ b/docs/source/advanced/mixed_cluster/building_stateless_images.rst @@ -9,6 +9,28 @@ In a homogeneous cluster, the management node is the same hardware architecture The issues arises in a heterogeneous cluster, where the management node is running a different level operating system *or* hardware architecture as the compute nodes in which to deploy the image. The ``genimage`` command that builds stateless images depends on various utilities provided by the base operating system and needs to be run on a node with the same hardware architecture and *major* Operating System release as the nodes that will be booted from the image. +When running xCAT >= 2.17 on EL >= 8 based management node with x86_64 architecture, qemu-user-static can be used to cross-build ppc64* and aarch64 osimages. Therefore, you don't need to build images on systems with the target architecture anymore. + +Cross-build ppc64*/aarch64 stateless/statelite image on x86_64 management node +------------------------------------------------------------------------------ + +#. Download qemu-user-static binaries for ppc64le and/or aarch64: :: + + wget https://github.com/multiarch/qemu-user-static/releases/latest/download/qemu-ppc64le-static -P /usr/bin + wget https://github.com/multiarch/qemu-user-static/releases/latest/download/qemu-aarch64-static -P /usr/bin + chmod 755 /usr/bin/qemu-*-static + +#. Configure systemd-binfmt accordingly: :: + + wget https://raw.githubusercontent.com/qemu/qemu/master/scripts/qemu-binfmt-conf.sh + bash qemu-binfmt-conf.sh --systemd 'ALL' --qemu-path '/usr/bin' --qemu-suffix '-static' --persistent 'yes' + systemctl restart systemd-binfmt.service + +#. Make sure ``osimage.osarch`` of your image is set to your target architecture eg. ``ppc64le`` or ``aarch64``. + + Now, ``genimage`` will use systemd-binfmt and the qemu-user-static binary to build the image for the specified architecture. + + Same Operating System, Different Architecture --------------------------------------------- diff --git a/docs/source/advanced/mixed_cluster/index.rst b/docs/source/advanced/mixed_cluster/index.rst index cbe8f293c..1e2cfe4bc 100644 --- a/docs/source/advanced/mixed_cluster/index.rst +++ b/docs/source/advanced/mixed_cluster/index.rst @@ -1,7 +1,7 @@ Mixed Cluster ============= -The concept of mixed cluster support in xCAT is managing cluster which contain different hardware architectures. Most commonly, this is usually a cluster consisting of both Power and x86 systems. +The concept of mixed cluster support in xCAT is managing cluster which contain different hardware architectures. Most commonly, this is usually a cluster consisting of both Power, ARM and x86 systems. .. toctree:: :maxdepth: 2 diff --git a/docs/source/advanced/mixed_cluster/support_matrix.rst b/docs/source/advanced/mixed_cluster/support_matrix.rst index 08f85eb75..8fcbacfd0 100644 --- a/docs/source/advanced/mixed_cluster/support_matrix.rst +++ b/docs/source/advanced/mixed_cluster/support_matrix.rst @@ -1,49 +1,50 @@ Support Matrix ============== -+---------+---------+---------+---------+---------+---------+---------+---------+---------+ -| | RHEL | SLES | RHEL | SLES | Ubuntu | RHEL | SLES | Ubuntu | -| | ppc64 | ppc64 | x86_64 | x86_64 | x86_64 | ppc64le | ppc64le | ppc64el | -| | CN | CN | CN | CN | CN | CN | CN | CN | -+=========+=========+=========+=========+=========+=========+=========+=========+=========+ -| RHEL | | | | | | | | | -| ppc64 | yes | yes | yes | yes | yes | yes | yes | yes | -| MN/SN | | | [1]_ | [1]_ | [1]_ | | | | -+---------+---------+---------+---------+---------+---------+---------+---------+---------+ -| SLES | | | | | | | | | -| ppc64 | yes | yes | yes | yes | yes | yes | yes | yes | -| MN/SN | | | [1]_ | [1]_ | [1]_ | | | | -+---------+---------+---------+---------+---------+---------+---------+---------+---------+ -| RHEL | | | | | | | | | -| x86_64 | yes | yes | yes | yes | yes | yes | yes | yes | -| MN/SN | [4]_ | [4]_ | | | | | | | -+---------+---------+---------+---------+---------+---------+---------+---------+---------+ -| SLES | | | | | | | | | -| x86_64 | yes | yes | yes | yes | yes | yes | yes | yes | -| MN/SN | [4]_ | [4]_ | | | | | | | -+---------+---------+---------+---------+---------+---------+---------+---------+---------+ -| Ubuntu | | | | | | | | | -| x86_64 | yes | yes | yes | yes | yes | yes | yes | yes | -| MN/SN | [5]_ | [5]_ | | | | | | | -+---------+---------+---------+---------+---------+---------+---------+---------+---------+ -| RHEL | | | | | | | | | -| ppc64le | yes | yes | yes | yes | yes | yes | yes | yes | -| MN/SN | [2]_ | [2]_ | | | | | | | -+---------+---------+---------+---------+---------+---------+---------+---------+---------+ -| SLES | | | | | | | | | -| ppc64le | no | no | yes | yes | yes | yes | yes | yes | -| MN/SN | | | | | | | | | -+---------+---------+---------+---------+---------+---------+---------+---------+---------+ -| Ubuntu | | | | | | | | | -| ppc64el | yes | yes | yes | yes | yes | yes | yes | yes | -| MN/SN | [3]_ | [3]_ | | | | | | | -+---------+---------+---------+---------+---------+---------+---------+---------+---------+ ++---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ +| | RHEL | SLES | RHEL | SLES | Ubuntu | RHEL | SLES | Ubuntu | RHEL | SLES | Ubuntu | +| | ppc64 | ppc64 | x86_64 | x86_64 | x86_64 | ppc64le | ppc64le | ppc64el | aarch64 | aarch64 | aarch64 | +| | CN | CN | CN | CN | CN | CN | CN | CN | CN | CN | CN | ++=========+=========+=========+=========+=========+=========+=========+=========+=========+=========+=========+=========+ +| RHEL | | | | | | | | | | | | +| ppc64 | yes | yes | yes | yes | yes | yes | yes | yes | no | no | no | +| MN/SN | | | [1]_ | [1]_ | [1]_ | | | | | | | ++---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ +| SLES | | | | | | | | | | | | +| ppc64 | yes | yes | yes | yes | yes | yes | yes | yes | no | no | no | +| MN/SN | | | [1]_ | [1]_ | [1]_ | | | | | | | ++---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ +| RHEL | | | | | | | | | | | | +| x86_64 | yes | yes | yes | yes | yes | yes | yes | yes | yes | no | no | +| MN/SN | [4]_ | [4]_ | | | | | | | | | | ++---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ +| SLES | | | | | | | | | | | | +| x86_64 | yes | yes | yes | yes | yes | yes | yes | yes | yes | no | no | +| MN/SN | [4]_ | [4]_ | | | | | | | | | | ++---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ +| Ubuntu | | | | | | | | | | | | +| x86_64 | yes | yes | yes | yes | yes | yes | yes | yes | yes | no | no | +| MN/SN | [5]_ | [5]_ | | | | | | | | | | ++---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ +| RHEL | | | | | | | | | | | | +| ppc64le | yes | yes | yes | yes | yes | yes | yes | yes | no | no | no | +| MN/SN | [2]_ | [2]_ | | | | | | | | | | ++---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ +| SLES | | | | | | | | | | | | +| ppc64le | no | no | yes | yes | yes | yes | yes | yes | no | no | no | +| MN/SN | | | | | | | | | | | | ++---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ +| Ubuntu | | | | | | | | | | | | +| ppc64el | yes | yes | yes | yes | yes | yes | yes | yes | no | no | no | +| MN/SN | [3]_ | [3]_ | | | | | | | | | | ++---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ + Notes: * The support statements refers to hardware control, operating system (os) provisioning, and general purpose system management where we do not see any obvious problems with the indicated combination. -* For diskless mixed cluster support, the initial diskless image must be created on a node running the target operating system version and architecture. see :doc:`/advanced/mixed_cluster/building_stateless_images` for more details. +* For xCAT < 2.17 diskless mixed cluster support, the initial diskless image must be created on a node running the target operating system version and architecture. On xCAT >= 2.17 with x86 based management nodes QEMU emulation can be used for different architectures (eg. ppc64 or aarch64) if qemu-user-static is installed and configured via systemd-binfmt. See :doc:`/advanced/mixed_cluster/building_stateless_images` for more details. .. rubric:: Footnotes diff --git a/docs/source/guides/install-guides/yum/grub2.rst b/docs/source/guides/install-guides/yum/grub2.rst new file mode 100644 index 000000000..ff0b498a3 --- /dev/null +++ b/docs/source/guides/install-guides/yum/grub2.rst @@ -0,0 +1,17 @@ +grub2 support for x86_64 and aarch64 +==================================== + +xCAT 2.17 enables grub2 boot support for x86_64 and aarch64 but does not ship the necessary grub2 binaries for both architectures. +If you want to use grub2 for x86_64 or aarch64 you need to download the binaries from some EL OS repository. + +#. Download files from an BaseOS EL repository mirror + + For EL these files are named ``grubx64.efi`` (x86_64) and ``grubaa64.efi`` (aarch64) and usually in ``BaseOS//os/EFI/BOOT``. + For very recent hardware you might need to use a newer grub version. Therefore, it's recommended to use a binary from the latest operationg system release available. + You can use Red Hat Enterprise Linux, AlmaLinux, Rocky Linux or even Fedora repositories to download the grub files. + +#. Copy downloaded files to ``/tftpboot/boot/grub2``: + + * x86_64: ``/tftpboot/boot/grub2/grub2.aarch64`` + * aarch64: ``/tftpboot/boot/grub2/grub2.x86_64`` + diff --git a/docs/source/guides/install-guides/yum/install.rst b/docs/source/guides/install-guides/yum/install.rst index 0396dd0ba..f29815ecc 100644 --- a/docs/source/guides/install-guides/yum/install.rst +++ b/docs/source/guides/install-guides/yum/install.rst @@ -10,3 +10,4 @@ Installing xCAT automatic_install.rst manual_install.rst + grub2.rst diff --git a/docs/source/overview/differentiators.rst b/docs/source/overview/differentiators.rst index 10b27dcc9..2f4fe5e49 100644 --- a/docs/source/overview/differentiators.rst +++ b/docs/source/overview/differentiators.rst @@ -16,7 +16,7 @@ Differentiators * Support Multiple Hardware - IBM Power, IBM Power LE, x86_64 + IBM Power, IBM Power LE, x86_64, aarch64 (alpha support) * Support Multiple Virtualization Infrastructures diff --git a/docs/source/overview/support_matrix.rst b/docs/source/overview/support_matrix.rst index 7e526c614..d458c6167 100644 --- a/docs/source/overview/support_matrix.rst +++ b/docs/source/overview/support_matrix.rst @@ -1,22 +1,22 @@ Operating System & Hardware Support Matrix ========================================== -+-------+-------+-------+-----+-------+--------+--------+--------+ -| | Power | Power | zVM | Power | x86_64 | x86_64 | x86_64 | -| | | LE | | KVM | | KVM | Esxi | -+=======+=======+=======+=====+=======+========+========+========+ -|RHEL | yes | yes | yes | yes | yes | yes | yes | -| | | | | | | | | -+-------+-------+-------+-----+-------+--------+--------+--------+ -|SLES | yes | yes | yes | yes | yes | yes | yes | -| | | | | | | | | -+-------+-------+-------+-----+-------+--------+--------+--------+ -|Ubuntu | no | yes | no | yes | yes | yes | yes | -| | | | | | | | | -+-------+-------+-------+-----+-------+--------+--------+--------+ -|CentOS | no | no | no | no | yes | yes | yes | -| | | | | | | | | -+-------+-------+-------+-----+-------+--------+--------+--------+ -|Windows| no | no | no | no | yes | yes | yes | -| | | | | | | | | -+-------+-------+-------+-----+-------+--------+--------+--------+ ++-------+-------+-------+-----+-------+--------+--------+--------+----------+ +| | Power | Power | zVM | Power | x86_64 | x86_64 | x86_64 | aarch64 | +| | | LE | | KVM | | KVM | Esxi | | ++=======+=======+=======+=====+=======+========+========+========+==========+ +|RHEL | yes | yes | yes | yes | yes | yes | yes | yes | +| | | | | | | | | | ++-------+-------+-------+-----+-------+--------+--------+--------+----------+ +|SLES | yes | yes | yes | yes | yes | yes | yes | no | +| | | | | | | | | | ++-------+-------+-------+-----+-------+--------+--------+--------+----------+ +|Ubuntu | no | yes | no | yes | yes | yes | yes | no | +| | | | | | | | | | ++-------+-------+-------+-----+-------+--------+--------+--------+----------+ +|CentOS | no | no | no | no | yes | yes | yes | no | +| | | | | | | | | | ++-------+-------+-------+-----+-------+--------+--------+--------+----------+ +|Windows| no | no | no | no | yes | yes | yes | no | +| | | | | | | | | | ++-------+-------+-------+-----+-------+--------+--------+--------+----------+ From 597fea5a64694cfbde35eba3f25206c1bcdec8da Mon Sep 17 00:00:00 2001 From: Markus Hilger Date: Fri, 12 Jul 2024 19:43:14 +0200 Subject: [PATCH 13/13] Enable grub2 support for x86 --- perl-xCAT/xCAT/Utils.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perl-xCAT/xCAT/Utils.pm b/perl-xCAT/xCAT/Utils.pm index 1c8ee9d5a..2c510bda9 100644 --- a/perl-xCAT/xCAT/Utils.pm +++ b/perl-xCAT/xCAT/Utils.pm @@ -4828,7 +4828,7 @@ sub lookupNetboot { $ret = "nimol"; } elsif ($imgtype =~ /^Linux$/i) { if ($osarch =~ /^x86_64$/i) { - $ret = "xnba,pxe"; + $ret = "xnba,pxe,grub2"; } elsif ($osarch =~ /^ppc64$/i) { if (($osv =~ /rh/i and $osn < 7) or ($osv =~ /sles/i and ($osn < 11 or ($osn == 11 and $osm < 4)))) { $ret = "yaboot";