fix(xcat-core): the Ubuntu Genesis image is built from the build host kernel

dracut copies the kernel, the kernel modules and every command out of the root
it runs in. The Ubuntu Genesis deb was built once, on the build host, so every
Ubuntu release got the build host's kernel. Nothing called the builder at all:
the pipelines converted the EL rpm with alien instead.

builddebs.pl gains --genesis, --genesis-only and --genesis-dist. The Genesis
deb is now built once per codename inside that codename's <codename>-<arch>-sbuild
chroot, the chroots xcat-dep's sbuild-all.pl already creates on the Ubuntu build
host. builddeb-genesis-base takes --expect-codename and stops when the root it
woke up in is a different release, so a build on the build host cannot produce a
codename's image.

builddebs.pl reads the build log through XCAT::BuildUtils::genesis_log_errors and
fails the build on FAILED:, a package apt cannot find and four more lines that a
zero exit status hides. The extracted payload goes through verify-genesis-payload
with the command list read back from the dracut module, plus the DHCP client, the
97xcat hooks and a /lib/modules that holds this chroot's kernel and no other.

The build root gains isc-dhcp-client, ifenslave and util-linux-extra, which
supply dhclient, ifenslave and hwclock.

genesis_deb_per_codename.t, genesis_payload_verification.t and
genesis_ubuntu_build_root.t fail without this change.

Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com>

build-utils/lib/XCAT/BuildUtils.pm

@@ -36,6 +36,8 @@ our @EXPORT_OK = qw(
     rewrite_file write_script read_line
     buildinfo_text
     targetarch_from_target
+    genesis_chroot_name genesis_target_arch genesis_build_plan
+    genesis_log_errors genesis_log_deny_rules
 );
 
 # Both builders echo the commands they run under --verbose.  Set once, after
@@ -491,4 +493,98 @@ sub targetarch_from_target {
     return $parts[-1];
 }
 
+# ------------------------------------------------------------------ Genesis --
+#
+# The Genesis image carries the kernel and the kernel modules of the release it boots,
+# because dracut copies them out of the root it runs in. So the Ubuntu Genesis deb is built
+# once per Ubuntu codename, inside that codename's chroot. A single build on the build host
+# gives every codename the build host's kernel.
+
+# genesis_chroot_name: the schroot chroot that builds the Genesis deb for one codename.
+#
+# Same name xcat-dep's sbuild-all.pl ensure_chroots creates, so both repositories build in
+# the same chroots and neither has to bootstrap a second set.
+sub genesis_chroot_name {
+    my ($codename, $arch) = @_;
+    die "genesis_chroot_name: a codename is required\n"
+        unless defined $codename && length $codename;
+    die "genesis_chroot_name: an architecture is required\n"
+        unless defined $arch && length $arch;
+    return "$codename-$arch-sbuild";
+}
+
+# genesis_target_arch: the directory xCAT reads the Genesis image from, for a deb
+# architecture. mknb reads /opt/xcat/share/xcat/netboot/genesis/<arch>, and that name is
+# the rpm architecture, not the deb one.
+my %GENESIS_TARGET_ARCH = (
+    amd64   => 'x86_64',
+    ppc64el => 'ppc64',
+);
+
+sub genesis_target_arch {
+    my ($arch) = @_;
+    my $target = $GENESIS_TARGET_ARCH{ $arch // '' };
+    die "genesis_target_arch: no Genesis image directory for '" . ($arch // '') . "'\n"
+        unless $target;
+    return $target;
+}
+
+# genesis_build_plan: one Genesis build per codename, for one architecture.
+#
+# The set of codenames comes from the caller, so a pipeline builds exactly the releases it
+# publishes. Returns the codename, the chroot to build it in and the package the build
+# produces, which is what the caller needs to run and to collect.
+sub genesis_build_plan {
+    my ($dists, $arch) = @_;
+    my @dists = @{ $dists || [] };
+    die "genesis_build_plan: at least one codename is required\n" unless @dists;
+    die "genesis_build_plan: an architecture is required\n"
+        unless defined $arch && length $arch;
+
+    my %seen;
+    return map {
+        {
+            codename => $_,
+            arch     => $arch,
+            chroot   => genesis_chroot_name($_, $arch),
+            package  => "xcat-genesis-base-$arch",
+            target   => genesis_target_arch($arch),
+        }
+    } grep { !$seen{$_}++ } @dists;
+}
+
+# genesis_log_deny_rules / genesis_log_errors: what a Genesis build log says when the build
+# failed but the exit status did not.
+#
+# dracut reports a command it cannot install with a FAILED: line and returns 0, and the
+# builder never reads dracut's result. That is how an image with no dhclient was packaged,
+# signed and published by a command that reported success. apt has the same shape: a
+# missing package leaves a diagnostic and a zero status behind a `|| true`. So the log is
+# the gate, not the exit status.
+my @GENESIS_LOG_DENY = (
+    [ qr/\bFAILED:/                    => 'dracut could not install a command' ],
+    [ qr/Cannot find module/           => 'a kernel module the build names is absent' ],
+    [ qr/dracut: Cannot/               => 'dracut refused the request' ],
+    [ qr/command not found/            => 'the build root has no such command' ],
+    [ qr/E: Unable to locate package/  => 'apt has no such package' ],
+    [ qr/Unable to correct problems/   => 'apt could not resolve the build root' ],
+);
+
+sub genesis_log_deny_rules { return @GENESIS_LOG_DENY; }
+
+sub genesis_log_errors {
+    my ($text) = @_;
+    return () unless defined $text && length $text;
+    my @found;
+    for my $line (split /\n/, $text) {
+        for my $rule (@GENESIS_LOG_DENY) {
+            my ($pattern, $why) = @{$rule};
+            next unless $line =~ $pattern;
+            push @found, { line => $line, why => $why };
+            last;
+        }
+    }
+    return @found;
+}
+
 1;

builddebs.pl

@@ -8,9 +8,12 @@
 # The central fact this design rests on: xcat-core debs are Perl. They are byte-identical
 # for every Ubuntu release, so they are built ONCE and the same files are published into
 # every codename. Only xCAT, xCATsn and xCAT-genesis-scripts carry an architecture, and
-# even there the difference is packaging metadata, not compiled output. That is why this
-# needs no sbuild and no per-codename chroot -- unlike xcat-dep, whose compiled packages
-# genuinely differ per release.
+# even there the difference is packaging metadata, not compiled output.
+#
+# The Genesis image is the one exception, and it is why --genesis exists. dracut copies the
+# kernel, the kernel modules and every command out of the root it runs in, so that image
+# genuinely differs per release and is built once per codename inside that codename's
+# sbuild chroot -- the same chroots xcat-dep builds its compiled packages in.
 use strict;
 use warnings;
 use feature 'say';
@@ -38,6 +41,7 @@ use XCAT::BuildUtils qw(
     reprepro_distributions reprepro_options
     lock_id_for take_build_lock sh_quote
     sh sh_or_die usage rewrite_file write_script read_line buildinfo_text
+    genesis_build_plan genesis_log_errors
 );
 
 # The xcat-core packages that ship as debs. xCAT-openbmc-py, xCAT-rmc and xCAT-release
@@ -61,10 +65,14 @@ my @PACKAGES = qw(
 my @DISTS = default_dists();
 
 my %opts;
-my (@cli_packages, @cli_dists);
+my (@cli_packages, @cli_dists, @cli_genesis_dists);
 GetOptions(
     "dist=s@"          => \@cli_dists,
     "package=s@"       => \@cli_packages,
+    "genesis"          => \$opts{genesis},
+    "genesis-only"     => \$opts{genesis_only},
+    "genesis-dist=s@"  => \@cli_genesis_dists,
+    "genesis-arch=s"   => \$opts{genesis_arch},
     "dest=s"           => \$opts{dest},
     "builddir=s"       => \$opts{builddir},
     "release=s"        => \$opts{release},
@@ -83,6 +91,14 @@ $opts{packages} = @cli_packages ? \@cli_packages : \@PACKAGES;
 $opts{dists}    = @cli_dists    ? \@cli_dists    : \@DISTS;
 $opts{gpg_key_name} //= 'xCAT Signing Key';
 
+# The Genesis step is off unless it is asked for, so every run that exists today keeps its
+# behaviour. It takes its own codename list: the Genesis deb is the one package that is not
+# the same file for every release, so the caller says which releases it wants built.
+$opts{genesis} = 1 if $opts{genesis_only};
+$opts{genesis_dists} = @cli_genesis_dists ? \@cli_genesis_dists : $opts{dists};
+die "FATAL: --genesis-dist needs --genesis\n" if @cli_genesis_dists && !$opts{genesis};
+die "FATAL: --genesis-arch needs --genesis\n" if $opts{genesis_arch} && !$opts{genesis};
+
 for my $pkg ($opts{packages}->@*) {
     die "FATAL: unknown package '$pkg'. Known: @PACKAGES\n"
         unless grep { $_ eq $pkg } @PACKAGES;
@@ -265,6 +281,130 @@ sub collect_debs {
     return $moved;
 }
 
+# ----------------------------------------------------------- the Genesis deb --
+#
+# Every other xcat-core deb is Perl and is built once for every release. The Genesis image
+# is not: dracut copies the kernel, the kernel modules and every command out of the root it
+# runs in. Built on the build host, one image serves every codename with the build host's
+# kernel -- which is how Ubuntu management nodes came to install an image built from an EL
+# kernel. So this step builds one image per codename, inside that codename's sbuild chroot.
+#
+# The chroots are the ones xcat-dep's sbuild-all.pl creates on the Ubuntu build host
+# (<codename>-<arch>-sbuild). They hand out disposable overlay sessions, so what the build
+# installs is discarded and the next codename starts from the pristine base.
+
+sub host_deb_arch {
+    my $arch = `dpkg --print-architecture 2>/dev/null` // '';
+    chomp $arch;
+    die "FATAL: dpkg does not report a host architecture\n" unless $arch;
+    return $arch;
+}
+
+# begin_chroot_session: start a disposable schroot session and return its id and its root.
+sub begin_chroot_session {
+    my ($chroot) = @_;
+    my $id = `schroot --begin-session --chroot @{[ sh_quote($chroot) ]} 2>&1` // '';
+    my $rc = $? >> 8;
+    chomp $id;
+    die "FATAL: cannot start a session in chroot '$chroot' (exit $rc): $id\n"
+      . "       sbuild-all.pl ensure_chroots creates it; run it on this host first.\n"
+        if $rc != 0 || $id !~ /\A\S+\z/;
+
+    my $root = `schroot --location -c @{[ sh_quote("session:$id") ]} 2>/dev/null` // '';
+    chomp $root;
+    unless ($root && -d $root) {
+        sh("schroot --end-session -c " . sh_quote("session:$id") . " >/dev/null 2>&1");
+        die "FATAL: schroot reports no location for session:$id\n";
+    }
+    return ($id, $root);
+}
+
+# genesis_build_log_problems: what the log says went wrong when the exit status did not.
+#
+# dracut prints FAILED: for a command it cannot install and exits 0. Reporting the first few
+# offending lines rather than all of them keeps a build console readable; the log file has
+# the rest and is named in the message.
+sub genesis_build_log_problems {
+    my ($logfile) = @_;
+    my $text = -f $logfile ? read_text($logfile) : '';
+    my @errors = genesis_log_errors($text);
+    return '' unless @errors;
+    my $shown = @errors > 10 ? 10 : scalar @errors;
+    my $report = "FATAL: the Genesis build log reports " . scalar(@errors) . " error(s):\n";
+    $report .= "  $_->{line}\n      ($_->{why})\n" for @errors[0 .. $shown - 1];
+    $report .= "  ... " . (@errors - $shown) . " more\n" if @errors > $shown;
+    $report .= "  the whole log is at $logfile\n";
+    return $report;
+}
+
+sub build_one_genesis_deb {
+    my ($step, $pkgdir) = @_;
+    my ($codename, $chroot) = ($step->{codename}, $step->{chroot});
+    say "Building $step->{package} for $codename in $chroot";
+
+    my ($id, $root) = begin_chroot_session($chroot);
+    my $logfile = "$pkgdir/$step->{package}-$codename.buildlog";
+    my $err;
+
+    eval {
+        # The builder needs its own directory, and Version and Release beside it. Copy them
+        # in rather than bind-mount the checkout: the build rewrites debian/control and
+        # debian/changelog, and it must not rewrite them in the tree the pipeline builds from.
+        sh_or_die("mkdir -p " . sh_quote("$root/build/xCAT-genesis-builder"),
+            "FATAL: cannot make the build directory in session:$id\n");
+        sh_or_die("cp -a " . sh_quote("$ROOT/xCAT-genesis-builder") . "/. "
+                . sh_quote("$root/build/xCAT-genesis-builder") . "/",
+            "FATAL: cannot copy xCAT-genesis-builder into session:$id\n");
+        for my $f (qw(Version Release)) {
+            next unless -f "$ROOT/$f";
+            copy("$ROOT/$f", "$root/build/$f")
+                or die "FATAL: cannot copy $f into session:$id: $!\n";
+        }
+        write_text("$root/build/Release", "$RELEASE\n");
+
+        # --expect-codename is the guard that keeps the image and the root together: the
+        # builder stops when the root it woke up in is not the release it was asked for.
+        my $cmd = join ' ',
+            'schroot', '--run-session', '-c', sh_quote("session:$id"), '-u', 'root', '-d', '/',
+            '--', '/bin/bash', '/build/xCAT-genesis-builder/builddeb-genesis-base',
+            '--expect-codename', sh_quote($codename), '--outdir', '/build/out';
+        my $rc = sh("$cmd > " . sh_quote($logfile) . " 2>&1");
+
+        # The log is read whether or not the command failed: a build that exits 0 with
+        # FAILED: lines in its log is the failure this gate exists for.
+        my $problems = genesis_build_log_problems($logfile);
+        if ($rc != 0) {
+            die "FATAL: the Genesis build for $codename failed (exit $rc); log: $logfile\n"
+              . $problems;
+        }
+        die $problems if $problems;
+
+        my @debs = glob("$root/build/out/*.deb");
+        die "FATAL: the Genesis build for $codename produced no .deb; log: $logfile\n"
+            unless @debs;
+        for my $deb (@debs) {
+            my $dest = "$pkgdir/" . basename($deb);
+            copy($deb, $dest) or die "FATAL: cannot collect $deb: $!\n";
+            say "  $dest";
+        }
+        1;
+    } or $err = $@;
+
+    sh("schroot --end-session -c " . sh_quote("session:$id") . " >/dev/null 2>&1");
+    die $err if $err;
+    return;
+}
+
+sub build_genesis_debs {
+    my ($pkgdir) = @_;
+    my $arch = $opts{genesis_arch} || host_deb_arch();
+    my @plan = genesis_build_plan($opts{genesis_dists}, $arch);
+    say "Genesis: @{[ scalar @plan ]} build(s) for $arch: "
+      . join(' ', map { $_->{codename} } @plan);
+    build_one_genesis_deb($_, $pkgdir) for @plan;
+    return scalar @plan;
+}
+
 # ------------------------------------------------------------- apt assembly --
 sub gpg_key_id {
     my ($name) = @_;
@@ -352,16 +492,25 @@ unlink glob("$ROOT/*.deb"), glob("$ROOT/*.buildinfo"), glob("$ROOT/*.changes"),
 say "xcat-core $PKGVER -> $dest";
 say "releases: @{[ join ' ', $opts{dists}->@* ]}";
 
-for my $pkg ($opts{packages}->@*) {
-    for my $arch (deb_package_arches($pkg)) {
-        build_package($pkg, $arch, $pkgdir);
+unless ($opts{genesis_only}) {
+    for my $pkg ($opts{packages}->@*) {
+        for my $arch (deb_package_arches($pkg)) {
+            build_package($pkg, $arch, $pkgdir);
+        }
+        collect_debs($pkg, $pkgdir);
     }
-    collect_debs($pkg, $pkgdir);
 }
 
-my $count = assemble_repo($pkgdir, $repo);
-write_repo_metadata($repo);
-say "published $count package(s) into @{[ scalar $opts{dists}->@* ]} release(s) at $repo";
+build_genesis_debs($pkgdir) if $opts{genesis};
+
+if ($opts{genesis_only}) {
+    say "Genesis debs are in $pkgdir";
+}
+else {
+    my $count = assemble_repo($pkgdir, $repo);
+    write_repo_metadata($repo);
+    say "published $count package(s) into @{[ scalar $opts{dists}->@* ]} release(s) at $repo";
+}
 
 __END__
 
@@ -382,8 +531,15 @@ xcat-core packages are Perl. The same binary serves every Ubuntu release, so eac
 package is built B<once> and the resulting C<.deb> files are published into every
 codename the repository declares. Only C<xCAT>, C<xCATsn> and C<xCAT-genesis-scripts>
 carry an architecture, and there the difference is packaging metadata rather than
-compiled output. Consequently this builder needs no C<sbuild> and no per-codename
-chroot. (xcat-dep is different: its packages are compiled, so it builds per codename.)
+compiled output.
+
+C<xcat-genesis-base> is the exception. dracut copies the kernel, the kernel modules
+and every command out of the root it runs in, so the Genesis image belongs to the
+release that built it. With C<--genesis> this builder makes one image per codename,
+each inside that codename's C<< <codename>-<arch>-sbuild >> schroot -- the chroots
+xcat-dep's C<sbuild-all.pl> creates on the Ubuntu build host. The build refuses to run
+in a root of another release, and it reads its own log: dracut reports a command it
+cannot install with a C<FAILED:> line and still exits 0.
 
 Replaces C<build-ubunturepo>. The GSA upload paths, the C<PROMOTE>/C<PREGA> release
 flows and the C<-d> xcat-dep repository mode were not carried over: publishing is done
@@ -401,6 +557,25 @@ Publish into this release. Repeatable. Defaults to focal, jammy, noble and resol
 
 Build only this package. Repeatable. Defaults to every xcat-core deb package.
 
+=item B<--genesis>
+
+Also build C<xcat-genesis-base-E<lt>archE<gt>>, one C<.deb> per codename, each inside
+that codename's schroot. Off by default.
+
+=item B<--genesis-only>
+
+Build the Genesis debs and nothing else, and assemble no repository. This is what
+xcat-dep needs: it consumes the debs with C<sbuild-all.pl --genesis-deb>.
+
+=item B<--genesis-dist>=I<CODENAME>
+
+Build the Genesis image for this release. Repeatable. Defaults to the C<--dist> list.
+
+=item B<--genesis-arch>=I<ARCH>
+
+Build the Genesis image for this Debian architecture. Defaults to the architecture of
+the build host, because the chroot has to match it.
+
 =item B<--dest>=I<DIR>
 
 Write the build under this directory: packages in C<debs/>, the apt repository in
@@ -442,5 +617,6 @@ This message.
   ./builddebs.pl
   ./builddebs.pl --dist noble --package perl-xCAT
   ./builddebs.pl --dest /srv/out --gpg-sign --gpg-home /keys/xcat-gpg-home
+  ./builddebs.pl --genesis-only --genesis-dist jammy --genesis-dist noble --dest /srv/out
 
 =cut

xCAT-genesis-builder/builddeb-genesis-base

@@ -1,13 +1,36 @@
 #!/bin/bash
-# Build xcat-genesis-base .deb package natively on Ubuntu.
-# Must run as root on an Ubuntu system (22.04, 24.04, or 26.04).
-# Parallel to buildrpm for EL targets.
+# Build the xcat-genesis-base .deb for ONE Ubuntu codename.
+#
+# dracut copies the kernel, the kernel modules and every command out of the root it runs
+# in, so this must run inside a root of the target codename. builddebs.pl --genesis starts
+# it in that codename's <codename>-<arch>-sbuild chroot, one build per codename.
+# --expect-codename is what stops a run on the build host: a single build there gives every
+# Ubuntu release the build host's kernel, which is how the EL-built image reached Ubuntu
+# management nodes in the first place.
+#
+# Parallel to xCAT-genesis-base.spec, which does the same for EL.
 
 set -euo pipefail
 
 DIR=$(readlink -f "$(dirname "$0")")
+OS_RELEASE=${OS_RELEASE:-/etc/os-release}
+expect_codename=""
+outdir=""
+
+while [ $# -gt 0 ]; do
+    case "$1" in
+        --expect-codename) expect_codename=${2:-}; shift 2 ;;
+        --expect-codename=*) expect_codename=${1#*=}; shift ;;
+        --outdir)          outdir=${2:-}; shift 2 ;;
+        --outdir=*)        outdir=${1#*=}; shift ;;
+        -h|--help)
+            echo "usage: builddeb-genesis-base [--expect-codename <codename>] [--outdir <dir>]"
+            exit 0 ;;
+        *) echo "ERROR: unknown argument: $1" >&2; exit 2 ;;
+    esac
+done
+
 BUILDARCH=$(dpkg --print-architecture)
-TRIPLET=$(dpkg-architecture -qDEB_HOST_MULTIARCH)
 
 case "$BUILDARCH" in
     amd64)   TARCH=x86_64 ;;
@@ -15,9 +38,36 @@ case "$BUILDARCH" in
     *)       echo "ERROR: unsupported architecture: $BUILDARCH" >&2; exit 1 ;;
 esac
 
+# The Genesis debs carry the architecture in the package name, so the packages an upgrade has
+# to displace carry it too. 2.19 renames the ppc64 debs to ppc64el; dpkg keeps the old package,
+# and its copy of the same files, unless the new one replaces it by name.
+rewrite_control() {
+    local control=$1 arch=$2 superseded
+    case "$arch" in
+        ppc64el) superseded="xcat-genesis-ppc64, xcat-genesis-base-ppc64" ;;
+        *)       superseded="xcat-genesis-$arch" ;;
+    esac
+    sed -i -e "s/xcat-genesis-base-amd64/xcat-genesis-base-$arch/g" \
+           -e "s/xcat-genesis-scripts-amd64/xcat-genesis-scripts-$arch/g" \
+           -e "s/xcat-genesis-amd64/$superseded/g" "$control"
+}
+
 VERSION=$(cat "$DIR/../Version" 2>/dev/null || echo "2.18.0")
 RELEASE=$(cat "$DIR/../Release" 2>/dev/null || echo "snap$(date +%Y%m%d%H%M)")
-CODENAME=$(. /etc/os-release && echo "$VERSION_CODENAME")
+CODENAME=$(. "$OS_RELEASE" && echo "${VERSION_CODENAME:-}")
+
+if [ -z "$CODENAME" ]; then
+    echo "ERROR: $OS_RELEASE names no VERSION_CODENAME" >&2
+    exit 1
+fi
+
+# The image is only valid for the release whose kernel it carries. Refuse to build a
+# codename's image anywhere but in that codename's root.
+if [ -n "$expect_codename" ] && [ "$expect_codename" != "$CODENAME" ]; then
+    echo "ERROR: this root is $CODENAME, not $expect_codename." >&2
+    echo "       The image would carry the $CODENAME kernel and boot under $expect_codename." >&2
+    exit 1
+fi
 
 echo "Building xcat-genesis-base for $BUILDARCH ($TARCH) on Ubuntu $CODENAME"
 
@@ -30,6 +80,7 @@ REQUIRED_PACKAGES="
     nfs-common rpcbind pciutils usbutils parted
     dosfstools e2fsprogs lvm2 mdadm net-tools
     bc psmisc rsync wget cpio
+    isc-dhcp-client ifenslave util-linux-extra
     dpkg-dev debhelper fakeroot devscripts vim-tiny
 "
 if [ "$BUILDARCH" = "amd64" ]; then
@@ -40,6 +91,9 @@ echo "Installing build dependencies..."
 apt-get update -qq
 apt-get install -y --no-install-recommends $REQUIRED_PACKAGES
 
+# dpkg-architecture comes from dpkg-dev, which the line above installs.
+TRIPLET=$(dpkg-architecture -qDEB_HOST_MULTIARCH)
+
 # Set up dracut module
 if [ -d /usr/lib/dracut/modules.d ]; then
     DRACUT_PARENT=/usr/lib/dracut/modules.d
@@ -71,6 +125,8 @@ if [ "$BUILDARCH" != "amd64" ]; then
     sed -i '/efibootmgr dmidecode/d' "$DRACUTMODDIR/module-setup.sh"
 fi
 
+# linux-image-generic pulls exactly one kernel into this root, and this root belongs to
+# $CODENAME, so this is the target release's kernel.
 KERNELVERSION=$(ls -1 /lib/modules | sort -V | tail -n 1)
 if [ -z "$KERNELVERSION" ]; then
     echo "ERROR: no kernel modules found in /lib/modules" >&2
@@ -123,24 +179,53 @@ if [ ! -e "$KERNEL_IMAGE" ]; then
     for candidate in \
         "/boot/vmlinux-$KERNELVERSION" \
         "/usr/lib/modules/$KERNELVERSION/vmlinuz" \
-        "/lib/modules/$KERNELVERSION/vmlinuz" \
-        "$(find /usr/lib/modules/"$KERNELVERSION" -maxdepth 2 -name 'vmlinuz*' -o -name 'vmlinux*' 2>/dev/null | head -n 1)" \
-        "$(find /lib/modules/"$KERNELVERSION" -maxdepth 2 -name 'vmlinuz*' -o -name 'vmlinux*' 2>/dev/null | head -n 1)" \
-        "$(ls -1 /boot/vmlinuz-* /boot/vmlinux-* 2>/dev/null | sort -V | tail -n 1)"
+        "/lib/modules/$KERNELVERSION/vmlinuz"
     do
-        if [ -n "$candidate" ] && [ -e "$candidate" ]; then
+        if [ -e "$candidate" ]; then
             KERNEL_IMAGE="$candidate"
             break
         fi
     done
 fi
 if [ ! -e "$KERNEL_IMAGE" ]; then
-    echo "ERROR: cannot find kernel image" >&2
+    echo "ERROR: cannot find the image of kernel $KERNELVERSION" >&2
     exit 1
 fi
 echo "Adding kernel $KERNEL_IMAGE"
 cp "$KERNEL_IMAGE" "$GENESIS_ROOT/kernel"
 
+# dracut_install reports a missing command and returns, so a hole reaches the .deb with
+# nothing in the log but one line. Read the commands back from the module and check them
+# against the payload, the way xCAT-genesis-base.spec does for EL.
+bash "$DIR/verify-genesis-payload" --commands-from "$DRACUTMODDIR/module-setup.sh" "$GENESIS_FS" \
+    usr/sbin/dhclient bin/sh sbin/xcatroot sbin/dhclient-script etc/rsyslog.conf
+
+# What the verifier cannot know: that this image belongs to this chroot's kernel. An image
+# built against another root's /lib/modules boots and then finds no driver for its NIC.
+if [ ! -d "$GENESIS_FS/lib/modules/$KERNELVERSION" ]; then
+    echo "ERROR: the image carries no /lib/modules/$KERNELVERSION" >&2
+    ls -1 "$GENESIS_FS/lib/modules" 2>/dev/null >&2 || true
+    exit 1
+fi
+if [ -z "$(find "$GENESIS_FS/lib/modules/$KERNELVERSION" -name '*.ko*' -print -quit)" ]; then
+    echo "ERROR: /lib/modules/$KERNELVERSION in the image holds no kernel module" >&2
+    exit 1
+fi
+for stray in "$GENESIS_FS"/lib/modules/*; do
+    [ -d "$stray" ] || continue
+    if [ "$(basename "$stray")" != "$KERNELVERSION" ]; then
+        echo "ERROR: the image carries $(basename "$stray"), not the $KERNELVERSION of this root" >&2
+        exit 1
+    fi
+done
+
+# The 97xcat hooks are what makes the image xCAT's. dracut drops a module whose check()
+# fails without a word, and the image then boots to a plain dracut shell.
+if [ -z "$(find "$GENESIS_FS" -path '*/hooks/cmdline/*xcat-cmdline.sh' -print -quit)" ]; then
+    echo "ERROR: the image carries no 97xcat cmdline hook" >&2
+    exit 1
+fi
+
 find "$GENESIS_TMPDIR" -type c -delete
 
 # Stage for dpkg-buildpackage
@@ -148,8 +233,9 @@ rm -rf "$DIR/opt"
 cp -a "$GENESIS_TMPDIR/opt" "$DIR/"
 
 # Adjust control file for target arch
-sed -i "s/xcat-genesis-base-amd64/xcat-genesis-base-$BUILDARCH/g" "$DIR/debian/control"
-sed -i "s/xcat-genesis-scripts-amd64/xcat-genesis-scripts-$BUILDARCH/g" "$DIR/debian/control"
+rewrite_control "$DIR/debian/control" "$BUILDARCH"
+# debian/dirs names the image directory, which is the rpm architecture.
+echo "/opt/xcat/share/xcat/netboot/genesis/$TARCH/" > "$DIR/debian/dirs"
 
 PKG_VERSION="${VERSION}-${RELEASE}~${CODENAME}"
 rm -f "$DIR/debian/changelog"
@@ -162,5 +248,12 @@ echo "Building .deb package..."
 cd "$DIR"
 dpkg-buildpackage -rfakeroot -uc -us -b
 
-echo "Build complete. .deb files:"
-ls -la "$DIR/../"xcat-genesis-base*.deb 2>/dev/null || echo "Check parent directory for .deb files"
+if [ -n "$outdir" ]; then
+    mkdir -p "$outdir"
+    mv "$DIR/../"xcat-genesis-base-*_*.deb "$outdir/"
+    echo "Build complete. .deb files in $outdir:"
+    ls -la "$outdir"
+else
+    echo "Build complete. .deb files:"
+    ls -la "$DIR/../"xcat-genesis-base*.deb 2>/dev/null || echo "Check parent directory for .deb files"
+fi

xCAT-genesis-builder/dracut_105/ubuntu/module-setup.sh

@@ -53,7 +53,26 @@ install() {
     dracut_install mount.nfs sshd vi reboot lspci parted screen mkfs mkfs.ext4 mkfs.btrfs
     #dracut_install libvirtd /usr/share/libvirt/cpu_map.xml /usr/bin/qemu-img /usr/libexec/qemu-kvm
     dracut_install mkswap df ifenslave ssh-keygen scp clear
+    # getdestiny makes its request file with mktemp. Without it the node reports no
+    # destiny, so xcatd never moves nodelist.status past powering-on.
+    dracut_install mktemp
     dracut_install dhclient lldpad
+
+    # OpenSSH 9.8 moved the per-connection work into sshd-session, which sshd execs by
+    # absolute path. Without it every connection to Genesis is refused.
+    for _sshd_helper in \
+        /usr/libexec/openssh/sshd-session \
+        /usr/libexec/openssh/sshd-auth \
+        /usr/lib/openssh/sshd-session \
+        /usr/lib/openssh/sshd-auth
+    do
+        _dracut_install_opt "$_sshd_helper"
+    done
+
+    # tmux exits under the C locale, and the image carries no locale data of its own.
+    for _lc_file in /usr/lib/locale/C.utf8/LC_*; do
+        _dracut_install_opt "$_lc_file"
+    done
     _dracut_install_opt "/lib/$TRIPLET/libnss_dns.so.2"
     dracut_install poweroff hwclock date /usr/share/terminfo/x/xterm /usr/share/terminfo/s/screen /etc/nsswitch.conf /etc/services
     dracut_install /usr/sbin/rsyslogd /etc/protocols umount /usr/bin/dpkg

xCAT-genesis-builder/dracut_105/ubuntu/xcat-cmdline.sh

@@ -2,13 +2,29 @@
 root=1
 rootok=1
 netroot=xcat
+
+# The image ships the C.UTF-8 locale only. tmux refuses to start under the C locale.
+export LC_ALL=C.UTF-8
+
+# screen exits when the image carries no usable terminal. doxcat is the whole of Genesis, so
+# it must run whether or not the multiplexer starts. Prints screen or direct.
+xcat_console_mode() {
+    if screen -ln -d -m -S xcatprobe true >/dev/null 2>&1; then
+        screen -S xcatprobe -X quit >/dev/null 2>&1
+        echo screen
+    else
+        echo direct
+    fi
+}
 clear
 echo PS1="'"'[xCAT Genesis running on \H \w]\$ '"'" > /.bashrc
 echo PS1="'"'[xCAT Genesis running on \H \w]\$ '"'" > /.bash_profile
 mkdir -p /etc/ssh
 mkdir -p /var/tmp/
 mkdir -p /var/empty/sshd
-sed -i '/^root:x/d' /etc/passwd
+# dracut writes this entry itself, with an empty password field unless the image is
+# built --hostonly. Match the user name only.
+sed -i '/^root:/d' /etc/passwd
 cat >>/etc/passwd <<"__ENDL"
 root:x:0:0::/:/bin/bash
 sshd:x:30:30:SSH User:/var/empty/sshd:/sbin/nologin
@@ -39,10 +55,13 @@ mkdir -p /var/lib/dhclient/
 mkdir -p /var/log
 ip link set lo up
 echo '127.0.0.1 localhost' >> /etc/hosts
-if grep -q console=ttyS /proc/cmdline; then
+XCAT_CONSOLE_MODE="$(xcat_console_mode)"
+if [ "$XCAT_CONSOLE_MODE" = "screen" ]; then
+    if grep -q console=ttyS /proc/cmdline; then
         while :; do sleep 1; screen -S console -ln screen -x doxcat </dev/tty1 &>/dev/tty1; clear &>/dev/tty1 ; done &
+    fi
+    while :; do screen -ln < /dev/tty2 &> /dev/tty2 ; done &
 fi
-while :; do screen -ln < /dev/tty2 &> /dev/tty2 ; done &
 
 # The section below is just for System P LE hardware discovery
 
@@ -87,4 +106,8 @@ elif [[ ${ARCH} =~ x86_64 ]]; then
     done
 fi
 
-while :; do screen -dr doxcat || screen -S doxcat -L -ln doxcat; done
+if [ "$XCAT_CONSOLE_MODE" = "screen" ]; then
+    while :; do screen -dr doxcat || screen -S doxcat -L -ln doxcat; done
+else
+    while :; do doxcat; sleep 5; done
+fi

xCAT-genesis-builder/verify-genesis-payload

@@ -0,0 +1,122 @@
+#!/bin/bash
+#
+# verify-genesis-payload [--commands-from <module-setup.sh>] <payload-root> [required-path ...]
+#
+# dracut_install() reports a missing binary and returns, so the module install function keeps
+# going and the image ships without it. Four such holes reached a release: no dhclient, no
+# openssl, no sshd-session and no UTF-8 locale. Check the extracted payload before it becomes
+# an rpm.
+#
+# Paths given on the command line are relative to <payload-root>. --commands-from reads back
+# what the dracut module installs: a bare command name is looked for in the four binary
+# directories, an absolute path under <payload-root> itself. The caller adds what only it
+# knows (the DHCP client is not the same package on every release); the rules below come from
+# the payload itself.
+
+set -u
+
+commands_from=""
+while [ $# -gt 0 ]; do
+    case "$1" in
+        --commands-from)
+            commands_from=${2:-}
+            shift 2 || true
+            ;;
+        --commands-from=*)
+            commands_from=${1#*=}
+            shift
+            ;;
+        *)
+            break
+            ;;
+    esac
+done
+
+payload=${1:-}
+if [ -z "$payload" ] || [ ! -d "$payload" ]; then
+    echo "verify-genesis-payload: not a payload directory: ${payload:-<empty>}" >&2
+    exit 2
+fi
+shift
+
+missing=""
+
+# have PATH: true when the payload carries PATH as a file, following the usr-merge symlinks
+# the image ships (/sbin -> usr/sbin).
+have() {
+    [ -e "$payload/$1" ]
+}
+
+require() {
+    local path=$1 why=$2
+    have "$path" || missing="$missing
+  $path ($why)"
+}
+
+for path in "$@"; do
+    require "$path" "required by the build"
+done
+
+# The dracut module names every command and every data file Genesis needs. A name the build
+# root does not supply installs nothing and says nothing, so read the names back and check
+# each one. Names under a condition are release-dependent, so only the top level of install()
+# counts.
+if [ -n "$commands_from" ]; then
+    if [ ! -r "$commands_from" ]; then
+        echo "verify-genesis-payload: cannot read $commands_from" >&2
+        exit 2
+    fi
+    commands=$(awk '
+        /^install\(\)/     { in_install = 1; next }
+        in_install && /^}/ { in_install = 0 }
+        in_install && /^    dracut_install / {
+            sub(/#.*/, "")
+            sub(/^    dracut_install /, "")
+            print
+        }' "$commands_from" | tr ' \t' '\n\n' | grep -v '^$' | grep -v '^-' | sort -u)
+    if [ -z "$commands" ]; then
+        echo "verify-genesis-payload: no command name read from $commands_from" >&2
+        exit 2
+    fi
+    for want in $commands; do
+        case "$want" in
+            # dracut_install installs an absolute path at that same path, so read it back
+            # under the payload root. Dropping these let an image with no /usr/bin/awk pass.
+            /*) have "${want#/}" || missing="$missing
+  $want (installed by $commands_from)"
+                ;;
+            *)  have "bin/$want" || have "sbin/$want" \
+                    || have "usr/bin/$want" || have "usr/sbin/$want" \
+                    || missing="$missing
+  $want (installed by $commands_from)"
+                ;;
+        esac
+    done
+fi
+
+require usr/sbin/sshd "Genesis is reached over ssh"
+require usr/bin/mktemp "getdestiny makes its request file with it"
+
+# OpenSSH 9.8 split the per-connection work into sshd-session, which sshd execs by absolute
+# path. EL9 carries OpenSSH 9.9, so an image with sshd alone refuses every connection.
+if have usr/sbin/sshd && grep -qa 'sshd-session' "$payload/usr/sbin/sshd" 2>/dev/null; then
+    if ! have usr/libexec/openssh/sshd-session && ! have usr/lib/openssh/sshd-session; then
+        missing="$missing
+  usr/libexec/openssh/sshd-session (this sshd execs it for every connection)"
+    fi
+fi
+
+# tmux exits under the C locale. The hook falls back to running doxcat directly, so this is
+# not fatal to booting, but a Genesis shell without tmux loses the console attach.
+if have usr/bin/tmux && ! have usr/lib/locale/C.utf8/LC_CTYPE; then
+    missing="$missing
+  usr/lib/locale/C.utf8/LC_CTYPE (tmux refuses to start without a UTF-8 locale)"
+fi
+
+if [ -n "$missing" ]; then
+    echo "verify-genesis-payload: $payload is incomplete:$missing" >&2
+    exit 1
+fi
+
+echo "verify-genesis-payload: $payload is complete"
+exit 0