From 9c2068e26c404002b31546c92ea2ed05d315254e Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Mon, 14 Sep 2026 07:54:15 -0300 Subject: [PATCH 1/5] test(xcat-core): the Ubuntu netboot initrd carries the whole firmware tree The Ubuntu genimage copies every file under the root image lib/firmware into the netboot initrd. On Ubuntu 26.04 that tree is 666 MB, and grub2 on a pseries node cannot load the initrd it produces. The test runs the firmware step of genimage over a root image whose firmware tree holds files no driver in the initrd asks for, and reads the initrd that comes out. It fails on the present code, which copies the tree whole. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- .../unit/ubuntu_genimage_initrd_firmware.t | 124 ++++++++++++++++++ 1 file changed, 124 insertions(+) create mode 100755 xCAT-test/unit/ubuntu_genimage_initrd_firmware.t diff --git a/xCAT-test/unit/ubuntu_genimage_initrd_firmware.t b/xCAT-test/unit/ubuntu_genimage_initrd_firmware.t new file mode 100755 index 000000000..b51bc6ba1 --- /dev/null +++ b/xCAT-test/unit/ubuntu_genimage_initrd_firmware.t @@ -0,0 +1,124 @@ +#!/usr/bin/env perl +use strict; +use warnings; + +use File::Path qw(make_path); +use File::Spec; +use File::Temp qw(tempdir); +use FindBin; +use Test::More; + +# grub2 on a pseries node cannot load the netboot initrd when it carries the whole Ubuntu +# firmware tree. Run genimage's firmware step over a root image that holds firmware no +# driver in the initrd asks for, and read what the step put in the initrd. + +my $repo_root = File::Spec->rel2abs(File::Spec->catdir($FindBin::Bin, '..', '..')); +my $genimage = File::Spec->catfile( + $repo_root, 'xCAT-server', 'share', 'xcat', 'netboot', 'ubuntu', 'genimage'); +plan skip_all => "genimage not found at $genimage" unless -f $genimage; + +my $src = do { local $/; open my $fh, '<', $genimage or die $!; <$fh> }; + +# The step sits between the loop that copies the initrd files and the copy of the module +# index. Both anchors hold whatever the firmware step itself looks like. +my ($step) = $src =~ m{ + copy_initrd_file\(\$srcpath,\ "/tmp/xcatinitrd\.\$\$/\$_"\);\n + \ {8}\}\n\ {4}\}\n + (.*?) + \n\ {4}if\ \(-d\ "\$rootimg_dir/lib/modules/\$kernelver/"\)\ \{ +}sx; +die "genimage no longer has a firmware step between the initrd file loop and the module index\n" + unless defined $step; + +my ($copy_initrd_file) = $src =~ m{^(sub copy_initrd_file \{.*?\n\}\n)}ms; +die "genimage no longer defines copy_initrd_file\n" unless defined $copy_initrd_file; + +# The helper the step calls once the firmware copy is filtered. A genimage that copies the +# tree whole has no such routine, and the step below then needs none. +my ($helper) = $src =~ m{^(sub initrd_firmware_files \{.*?\n\}\n)}ms; + +my $scratch = tempdir(CLEANUP => 1); +my $rootimg = "$scratch/rootimg"; +my $initrd_dir = "$scratch/initrd"; + +my $rewritten = ($step =~ s{/tmp/xcatinitrd\.\$\$}{$initrd_dir}g); +die "the firmware step no longer writes to /tmp/xcatinitrd.\$\$\n" unless $rewritten; + +# modinfo reports the firmware of a module. Answer for the modules of this root image only. +my $bin = "$scratch/bin"; +make_path($bin); +open(my $fake, '>', "$bin/modinfo") or die $!; +print $fake <<'SH'; +#!/bin/sh +case "$*" in + *mlx5_core*) echo mellanox/fw-a.mfa2 ;; + *bnx2x*) echo bnx2x/bnx2x-e2.fw ;; + *e1000e*) echo intel/absent-from-this-image.bin ;; + *virtio_net*) : ;; + *) exit 1 ;; +esac +SH +close($fake); +chmod 0755, "$bin/modinfo"; +$ENV{PATH} = "$bin:$ENV{PATH}"; + +foreach my $file ( + 'lib/firmware/mellanox/fw-a.mfa2', + 'lib/firmware/bnx2x/bnx2x-e2.fw.zst', + 'lib/firmware/amdgpu/never-asked-for.bin', + 'lib/firmware/qcom/never-asked-for-either.bin', + 'lib/modules/7.0.0/kernel/drivers/net/virtio_net.ko', + 'lib/modules/7.0.0/kernel/drivers/net/mlx5_core.ko', + 'lib/modules/7.0.0/kernel/drivers/net/bnx2x.ko.zst', + 'lib/modules/7.0.0/kernel/drivers/net/e1000e.ko', + 'bin/busybox', + ) +{ + my $full = "$rootimg/$file"; + ($full =~ m{^(.*)/[^/]+$}) and make_path($1); + open(my $fh, '>', $full) or die $!; + print $fh "content of $file\n"; + close($fh); +} +make_path("$initrd_dir/lib/firmware"); + +{ + package Scratch; + use strict; + use warnings; + use File::Basename; + use File::Copy; + use File::Path qw(mkpath); + our $rootimg_dir; + our @filestoadd; + sub xdie { die @_ } +} + +## no critic (BuiltinFunctions::ProhibitStringyEval) +eval "package Scratch;\n$copy_initrd_file\n1" or die $@; +if (defined $helper) { + eval "package Scratch;\n$helper\n1" or die $@; +} + +$Scratch::rootimg_dir = $rootimg; +@Scratch::filestoadd = ( + [ 'lib/modules/7.0.0/kernel/drivers/net/virtio_net.ko', 'lib/virtio_net.ko' ], + [ 'lib/modules/7.0.0/kernel/drivers/net/mlx5_core.ko', 'lib/mlx5_core.ko' ], + [ 'lib/modules/7.0.0/kernel/drivers/net/bnx2x.ko.zst', 'lib/bnx2x.ko' ], + [ 'lib/modules/7.0.0/kernel/drivers/net/e1000e.ko', 'lib/e1000e.ko' ], + [ 'bin/busybox', 'bin/busybox' ], +); +eval "package Scratch;\nno strict 'vars';\n$step\n1" or die $@; + +ok(-e "$initrd_dir/lib/firmware/mellanox/fw-a.mfa2", + 'the initrd keeps the firmware a driver in it asks for'); +ok(-e "$initrd_dir/lib/firmware/bnx2x/bnx2x-e2.fw.zst", + 'a compressed firmware file answers the plain name the driver asks for'); +ok(!-e "$initrd_dir/lib/firmware/amdgpu/never-asked-for.bin", + 'the initrd does not carry firmware no driver in it asks for'); +ok(!-e "$initrd_dir/lib/firmware/qcom/never-asked-for-either.bin", + 'the whole firmware tree does not reach the initrd'); +ok(!-e "$initrd_dir/lib/firmware/intel/absent-from-this-image.bin", + 'a firmware name the root image does not have is left out'); + +done_testing(); From 3fcc781fdd88c05da4572220e01f8703f24c7d79 Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Mon, 14 Sep 2026 07:54:23 -0300 Subject: [PATCH 2/5] fix(xcat-core): grub2 cannot load the Ubuntu 26.04 ppc64el netboot initrd The ubuntu-26.04-ppc64el diskless node never starts its kernel. SLOF reports W3411 and E3406 and falls through to disk. The node fetches the kernel and the initrd in under a minute and fails six minutes later, with no network activity in between: grub2 has the payload and cannot start it. genimage copies the whole lib/firmware tree of the root image into the initrd. That tree is 666 MB on Ubuntu 26.04, which takes the initrd to 719 MB. A 687 MB initrd boots the same kernel on the same node; a 719 MB one does not. The firmware copy now takes only the firmware that the drivers in the initrd ask for, which modinfo reports for each module. The root image keeps its whole tree, so the node that boots is unchanged. ubuntu_genimage_initrd_firmware.t drives the firmware step over a root image whose firmware tree holds files no driver asks for. It fails on the previous commit. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- .../share/xcat/netboot/ubuntu/genimage | 56 ++++++++++++++++++- 1 file changed, 54 insertions(+), 2 deletions(-) diff --git a/xCAT-server/share/xcat/netboot/ubuntu/genimage b/xCAT-server/share/xcat/netboot/ubuntu/genimage index 17005cdb5..9fe0803b4 100755 --- a/xCAT-server/share/xcat/netboot/ubuntu/genimage +++ b/xCAT-server/share/xcat/netboot/ubuntu/genimage @@ -1827,8 +1827,11 @@ EOMS } } - if (-d "$rootimg_dir/lib/firmware/") { - system("cp -r $rootimg_dir/lib/firmware/* /tmp/xcatinitrd.$$/lib/firmware"); + # The whole firmware tree is 666 MB on Ubuntu 26.04, and grub2 on a pseries node cannot + # load an initrd of that size. The initrd only has to reach the network. + foreach my $firmware (initrd_firmware_files($rootimg_dir, \@filestoadd)) { + copy_initrd_file("$rootimg_dir/lib/firmware/$firmware", + "/tmp/xcatinitrd.$$/lib/firmware/$firmware"); } if (-d "$rootimg_dir/lib/modules/$kernelver/") { @@ -1892,6 +1895,55 @@ sub isnetdriver { } } +#------------------------------------------------------------------------------- + +=head3 initrd_firmware_files + + Descriptions: Select the firmware that the drivers in the netboot initrd ask for. + Arguments: + rootimg_dir - the root image directory + filestoadd - reference to the list of files that go into the initrd. A module + is either a plain path or a [ source, destination ] pair. + Returns: the firmware files that exist, as paths below lib/firmware + +=cut + +#------------------------------------------------------------------------------- +sub initrd_firmware_files { + my ($rootimg_dir, $filestoadd) = @_; + + return () unless (-d "$rootimg_dir/lib/firmware"); + + my %wanted; + foreach my $entry (@$filestoadd) { + my $module = ref($entry) ? $entry->[0] : $entry; + next unless ($module =~ /\.ko(?:\.(?:gz|xz|zst))?$/); + next unless (-f "$rootimg_dir/$module"); + + my $out = `modinfo -F firmware "$rootimg_dir/$module" 2>/dev/null`; + if ($?) { + print "Warning: cannot read the firmware of $module\n"; + next; + } + foreach my $name (split(/\n/, $out)) { + $name =~ s/^\s+|\s+$//g; + $wanted{$name} = 1 if (length $name); + } + } + + my @files; + foreach my $name (sort keys %wanted) { + + # Ubuntu keeps the firmware compressed and the kernel asks for the plain name. + foreach my $candidate ($name, "$name.zst", "$name.xz", "$name.gz") { + next unless (-e "$rootimg_dir/lib/firmware/$candidate"); + push @files, $candidate; + last; + } + } + return @files; +} + sub find_rootimg_file { my ($file) = @_; From 3c104421a28fc354a00b90ce27619487dc60960c Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Tue, 15 Sep 2026 21:11:18 -0300 Subject: [PATCH 3/5] fix(xcat-core): ubuntu_genimage_initrd_firmware.t passes when the file it reads is missing ubuntu_genimage_initrd_firmware.t called plan skip_all when xCAT-server/share/xcat/netboot/ubuntu/genimage was absent, so a checkout that lost the file reported 0 tests and exit 0. A test that cannot fail measures nothing. Die instead, which is what makentp_ntp_deps.t already does for setupntp. With xCAT-server/share/xcat/netboot/ubuntu/genimage moved aside the file now exits 2 and prints "genimage not found at "; before this change it exited 0 and printed "1..0 # SKIP genimage not found at ". With the file present the test passes either way. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- xCAT-test/unit/ubuntu_genimage_initrd_firmware.t | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xCAT-test/unit/ubuntu_genimage_initrd_firmware.t b/xCAT-test/unit/ubuntu_genimage_initrd_firmware.t index b51bc6ba1..8408545b9 100755 --- a/xCAT-test/unit/ubuntu_genimage_initrd_firmware.t +++ b/xCAT-test/unit/ubuntu_genimage_initrd_firmware.t @@ -15,7 +15,7 @@ use Test::More; my $repo_root = File::Spec->rel2abs(File::Spec->catdir($FindBin::Bin, '..', '..')); my $genimage = File::Spec->catfile( $repo_root, 'xCAT-server', 'share', 'xcat', 'netboot', 'ubuntu', 'genimage'); -plan skip_all => "genimage not found at $genimage" unless -f $genimage; +die "genimage not found at $genimage\n" unless -f $genimage; my $src = do { local $/; open my $fh, '<', $genimage or die $!; <$fh> }; From 2523a06b1650658a5f0506ab6e679e4913eaef5f Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Thu, 17 Sep 2026 07:37:56 -0300 Subject: [PATCH 4/5] test(xcat-core): the initrd firmware step ignores custom drivers and firmware overrides ubuntu_genimage_initrd_firmware.t drove the firmware step over a root image that holds every module and every firmware file. It covered neither of the two places genimage reads from beside the root image. The test now puts a driver in the custom directory, with its firmware in the root image, and a firmware override under lib/firmware/updates/. Both are red: the step asks modinfo about the module under the root image, where a custom driver is not, and it looks for a firmware name under lib/firmware only, where an override is not. The five assertions that were there stay green. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- .../unit/ubuntu_genimage_initrd_firmware.t | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/xCAT-test/unit/ubuntu_genimage_initrd_firmware.t b/xCAT-test/unit/ubuntu_genimage_initrd_firmware.t index 8408545b9..0d162dd68 100755 --- a/xCAT-test/unit/ubuntu_genimage_initrd_firmware.t +++ b/xCAT-test/unit/ubuntu_genimage_initrd_firmware.t @@ -54,6 +54,7 @@ case "$*" in *mlx5_core*) echo mellanox/fw-a.mfa2 ;; *bnx2x*) echo bnx2x/bnx2x-e2.fw ;; *e1000e*) echo intel/absent-from-this-image.bin ;; + *custom_nic*) echo custom/custom-nic.bin ;; *virtio_net*) : ;; *) exit 1 ;; esac @@ -64,6 +65,8 @@ $ENV{PATH} = "$bin:$ENV{PATH}"; foreach my $file ( 'lib/firmware/mellanox/fw-a.mfa2', + 'lib/firmware/updates/7.0.0/mellanox/fw-a.mfa2', + 'lib/firmware/custom/custom-nic.bin', 'lib/firmware/bnx2x/bnx2x-e2.fw.zst', 'lib/firmware/amdgpu/never-asked-for.bin', 'lib/firmware/qcom/never-asked-for-either.bin', @@ -82,6 +85,16 @@ foreach my $file ( } make_path("$initrd_dir/lib/firmware"); +# A driver the administrator put in the custom directory. genimage takes the module from there +# and not from the root image, so that is the file its firmware has to be read from. +my $customdir = "$scratch/custom"; +my $pathtofiles = "$scratch/pathtofiles"; +make_path("$customdir/lib/modules/7.0.0/kernel/drivers/net"); +make_path($pathtofiles); +open(my $custom, '>', "$customdir/lib/modules/7.0.0/kernel/drivers/net/custom_nic.ko") or die $!; +print $custom "content of a custom driver\n"; +close($custom); + { package Scratch; use strict; @@ -101,11 +114,15 @@ if (defined $helper) { } $Scratch::rootimg_dir = $rootimg; +$Scratch::customdir = $customdir; +$Scratch::pathtofiles = $pathtofiles; +$Scratch::kernelver = '7.0.0'; @Scratch::filestoadd = ( [ 'lib/modules/7.0.0/kernel/drivers/net/virtio_net.ko', 'lib/virtio_net.ko' ], [ 'lib/modules/7.0.0/kernel/drivers/net/mlx5_core.ko', 'lib/mlx5_core.ko' ], [ 'lib/modules/7.0.0/kernel/drivers/net/bnx2x.ko.zst', 'lib/bnx2x.ko' ], [ 'lib/modules/7.0.0/kernel/drivers/net/e1000e.ko', 'lib/e1000e.ko' ], + [ 'lib/modules/7.0.0/kernel/drivers/net/custom_nic.ko', 'lib/custom_nic.ko' ], [ 'bin/busybox', 'bin/busybox' ], ); eval "package Scratch;\nno strict 'vars';\n$step\n1" or die $@; @@ -120,5 +137,9 @@ ok(!-e "$initrd_dir/lib/firmware/qcom/never-asked-for-either.bin", 'the whole firmware tree does not reach the initrd'); ok(!-e "$initrd_dir/lib/firmware/intel/absent-from-this-image.bin", 'a firmware name the root image does not have is left out'); +ok(-e "$initrd_dir/lib/firmware/custom/custom-nic.bin", + 'the firmware of a driver taken from the custom directory reaches the initrd'); +ok(-e "$initrd_dir/lib/firmware/updates/7.0.0/mellanox/fw-a.mfa2", + 'a firmware override under updates/ reaches the initrd'); done_testing(); From 5554b79c2ded72025f6b07cfe7bbaf4ec6e928b6 Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Thu, 17 Sep 2026 07:38:47 -0300 Subject: [PATCH 5/5] fix(xcat-core): the initrd firmware step reads the root image only The step that fills lib/firmware in the Ubuntu netboot initrd asked modinfo about $rootimg_dir/$module and looked a firmware name up under lib/firmware. The copy step beside it takes a module from $customdir or $pathtofiles first, and the kernel looks a firmware name up under updates/, updates/, / and lib/firmware. So a custom driver reached the initrd with no firmware, even when the root image carried it, and a firmware override was left out of the initrd altogether. initrd_firmware_files now takes the module directories the copy step searches and the kernel release. It resolves each module in that order before asking modinfo, and keeps every firmware file that exists in the four directories the kernel searches, so the override still wins on the node. ubuntu_genimage_initrd_firmware.t covers both: its two new cases are red on the commit before this one. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- .../share/xcat/netboot/ubuntu/genimage | 44 +++++++++++++++---- .../unit/ubuntu_genimage_initrd_firmware.t | 3 ++ 2 files changed, 38 insertions(+), 9 deletions(-) diff --git a/xCAT-server/share/xcat/netboot/ubuntu/genimage b/xCAT-server/share/xcat/netboot/ubuntu/genimage index 9fe0803b4..970a97d59 100755 --- a/xCAT-server/share/xcat/netboot/ubuntu/genimage +++ b/xCAT-server/share/xcat/netboot/ubuntu/genimage @@ -1829,7 +1829,9 @@ EOMS # The whole firmware tree is 666 MB on Ubuntu 26.04, and grub2 on a pseries node cannot # load an initrd of that size. The initrd only has to reach the network. - foreach my $firmware (initrd_firmware_files($rootimg_dir, \@filestoadd)) { + foreach my $firmware ( + initrd_firmware_files($rootimg_dir, \@filestoadd, $kernelver, $customdir, $pathtofiles)) + { copy_initrd_file("$rootimg_dir/lib/firmware/$firmware", "/tmp/xcatinitrd.$$/lib/firmware/$firmware"); } @@ -1904,13 +1906,16 @@ sub isnetdriver { rootimg_dir - the root image directory filestoadd - reference to the list of files that go into the initrd. A module is either a plain path or a [ source, destination ] pair. + kernelver - the kernel release, which names one of the firmware directories + module_dirs - directories searched for a module before the root image, in the + same order the copy step searches them Returns: the firmware files that exist, as paths below lib/firmware =cut #------------------------------------------------------------------------------- sub initrd_firmware_files { - my ($rootimg_dir, $filestoadd) = @_; + my ($rootimg_dir, $filestoadd, $kernelver, @module_dirs) = @_; return () unless (-d "$rootimg_dir/lib/firmware"); @@ -1918,9 +1923,18 @@ sub initrd_firmware_files { foreach my $entry (@$filestoadd) { my $module = ref($entry) ? $entry->[0] : $entry; next unless ($module =~ /\.ko(?:\.(?:gz|xz|zst))?$/); - next unless (-f "$rootimg_dir/$module"); - my $out = `modinfo -F firmware "$rootimg_dir/$module" 2>/dev/null`; + # Ask about the module the copy step takes, which is the custom one where there is + # one. A custom driver is not in the root image at all. + my $source; + foreach my $dir (grep { defined && length } @module_dirs, $rootimg_dir) { + next unless (-f "$dir/$module"); + $source = "$dir/$module"; + last; + } + next unless (defined $source); + + my $out = `modinfo -F firmware "$source" 2>/dev/null`; if ($?) { print "Warning: cannot read the firmware of $module\n"; next; @@ -1931,14 +1945,26 @@ sub initrd_firmware_files { } } + # The kernel looks for a firmware name in these directories below lib/firmware, in this + # order, and loads the first file it finds. Keep every one that exists, so an override + # still wins on the node. + my @dirs = ("updates", ""); + if (defined $kernelver and length $kernelver) { + unshift @dirs, "updates/$kernelver"; + splice(@dirs, 2, 0, $kernelver); + } + my @files; foreach my $name (sort keys %wanted) { + foreach my $dir (@dirs) { + my $prefix = length($dir) ? "$dir/" : ""; - # Ubuntu keeps the firmware compressed and the kernel asks for the plain name. - foreach my $candidate ($name, "$name.zst", "$name.xz", "$name.gz") { - next unless (-e "$rootimg_dir/lib/firmware/$candidate"); - push @files, $candidate; - last; + # Ubuntu keeps the firmware compressed and the kernel asks for the plain name. + foreach my $candidate ($name, "$name.zst", "$name.xz", "$name.gz") { + next unless (-e "$rootimg_dir/lib/firmware/$prefix$candidate"); + push @files, "$prefix$candidate"; + last; + } } } return @files; diff --git a/xCAT-test/unit/ubuntu_genimage_initrd_firmware.t b/xCAT-test/unit/ubuntu_genimage_initrd_firmware.t index 0d162dd68..d01242385 100755 --- a/xCAT-test/unit/ubuntu_genimage_initrd_firmware.t +++ b/xCAT-test/unit/ubuntu_genimage_initrd_firmware.t @@ -103,6 +103,9 @@ close($custom); use File::Copy; use File::Path qw(mkpath); our $rootimg_dir; + our $customdir; + our $pathtofiles; + our $kernelver; our @filestoadd; sub xdie { die @_ } }