diff --git a/build-utils/lib/XCAT/BuildUtils.pm b/build-utils/lib/XCAT/BuildUtils.pm index 5eb897799..c5f588e52 100644 --- a/build-utils/lib/XCAT/BuildUtils.pm +++ b/build-utils/lib/XCAT/BuildUtils.pm @@ -36,6 +36,9 @@ our @EXPORT_OK = qw( rewrite_file write_script read_line buildinfo_text targetarch_from_target + genesis_chroot_name genesis_target_arch genesis_build_plan + genesis_log_errors genesis_log_deny_rules deb_belongs_to_dist + genesis_dists genesis_dist_reason ); # Both builders echo the commands they run under --verbose. Set once, after @@ -690,4 +693,135 @@ sub targetarch_from_target { return $parts[-1]; } +# ------------------------------------------------------------------ Genesis -- +# +# The Genesis image carries the kernel and the kernel modules of the release it boots, +# because dracut copies them out of the root it runs in. So the Ubuntu Genesis deb is built +# once per Ubuntu codename, inside that codename's chroot. A single build on the build host +# gives every codename the build host's kernel. + +# genesis_chroot_name: the schroot chroot that builds the Genesis deb for one codename. +# +# Same name xcat-dep's sbuild-all.pl ensure_chroots creates, so both repositories build in +# the same chroots and neither has to bootstrap a second set. +sub genesis_chroot_name { + my ($codename, $arch) = @_; + die "genesis_chroot_name: a codename is required\n" + unless defined $codename && length $codename; + die "genesis_chroot_name: an architecture is required\n" + unless defined $arch && length $arch; + return "$codename-$arch-sbuild"; +} + +# genesis_target_arch: the directory xCAT reads the Genesis image from, for a deb +# architecture. mknb reads /opt/xcat/share/xcat/netboot/genesis/, and that name is +# the rpm architecture, not the deb one. +my %GENESIS_TARGET_ARCH = ( + amd64 => 'x86_64', + ppc64el => 'ppc64', +); + +sub genesis_target_arch { + my ($arch) = @_; + my $target = $GENESIS_TARGET_ARCH{ $arch // '' }; + die "genesis_target_arch: no Genesis image directory for '" . ($arch // '') . "'\n" + unless $target; + return $target; +} + +# genesis_build_plan: one Genesis build per codename, for one architecture. +# +# The set of codenames comes from the caller, so a pipeline builds exactly the releases it +# publishes. +sub genesis_build_plan { + my ($dists, $arch) = @_; + my @dists = @{ $dists || [] }; + die "genesis_build_plan: at least one codename is required\n" unless @dists; + die "genesis_build_plan: an architecture is required\n" + unless defined $arch && length $arch; + + my %seen; + return map { + { + codename => $_, + arch => $arch, + chroot => genesis_chroot_name($_, $arch), + package => "xcat-genesis-base-$arch", + target => genesis_target_arch($arch), + } + } grep { !$seen{$_}++ } @dists; +} + +# genesis_log_deny_rules / genesis_log_errors: what a Genesis build log says when the build +# failed but the exit status did not. +# +# dracut reports a command it cannot install with a FAILED: line and returns 0. apt has the +# same shape: a missing package leaves a diagnostic and a zero status behind a `|| true`. So +# the log is the gate, not the exit status. +my @GENESIS_LOG_DENY = ( + [ qr/\bFAILED:/ => 'dracut could not install a command' ], + [ qr/Cannot find module/ => 'a kernel module the build names is absent' ], + [ qr/dracut: Cannot/ => 'dracut refused the request' ], + [ qr/command not found/ => 'the build root has no such command' ], + [ qr/E: Unable to locate package/ => 'apt has no such package' ], + [ qr/Unable to correct problems/ => 'apt could not resolve the build root' ], +); + +sub genesis_log_deny_rules { return @GENESIS_LOG_DENY; } + +# The package built once per codename. It is the only one: it carries the kernel and the kernel +# modules of the root that built it. +my $GENESIS_IMAGE_DEB = qr{\Axcat-genesis-base-}; + +# Releases whose stock chroot cannot build that package, and why. focal ships debhelper 12 and +# xCAT-genesis-base declares debhelper-compat (= 13), so sbuild stops on the build dependencies +# before dracut runs. +my %GENESIS_DIST_UNSUPPORTED = ( + focal => 'debhelper 12 cannot satisfy debhelper-compat (= 13)', +); + +# genesis_dists: the releases of @dists a Genesis image can be built on. +sub genesis_dists { + my (@dists) = @_; + return grep { !exists $GENESIS_DIST_UNSUPPORTED{$_} } @dists; +} + +# genesis_dist_reason: why a release was left out, or undef. +sub genesis_dist_reason { return $GENESIS_DIST_UNSUPPORTED{ $_[0] // '' }; } + +# deb_belongs_to_dist: whether a built .deb may be published into one release. +# +# Almost every xcat-core deb is Architecture:all and the same file serves every release, so the +# answer is yes. The Genesis image is not: it is built per codename and carries that codename in +# its version (2.19.0-snap...~noble). Publishing all three into every suite lets apt serve the +# newest, which is the image of another release. +# +# Only that package is asked. A ~ in a version is Debian's prerelease separator before it is +# anything else, and --release takes whatever the caller gives it, so reading every ~ as a +# codename drops a whole `--release 1~rc1` build from every suite. +sub deb_belongs_to_dist { + my ($deb, $dist) = @_; + return 1 unless defined $deb && defined $dist && $dist ne ''; + my $base = basename($deb); + my ($name) = $base =~ /\A([^_]+)_/; + return 1 unless defined $name && $name =~ $GENESIS_IMAGE_DEB; + return 1 unless $base =~ /_[^_]*~([A-Za-z0-9.]+)_[^_]*\.deb\z/; + return $1 eq $dist ? 1 : 0; +} + +sub genesis_log_errors { + my ($text) = @_; + return () unless defined $text && length $text; + my @found; + for my $line (split /\n/, $text) { + for my $rule (@GENESIS_LOG_DENY) { + my ($pattern, $why) = @{$rule}; + next unless $line =~ $pattern; + push @found, { line => $line, why => $why }; + last; + } + } + return @found; +} + 1; diff --git a/builddebs.pl b/builddebs.pl index 4c53ca722..d27791dfa 100755 --- a/builddebs.pl +++ b/builddebs.pl @@ -8,9 +8,10 @@ # The central fact this design rests on: xcat-core debs are Perl. They are byte-identical # for every Ubuntu release, so they are built ONCE and the same files are published into # every codename. Only xCAT, xCATsn and xCAT-genesis-scripts carry an architecture, and -# even there the difference is packaging metadata, not compiled output. That is why this -# needs no sbuild and no per-codename chroot -- unlike xcat-dep, whose compiled packages -# genuinely differ per release. +# even there the difference is packaging metadata, not compiled output. +# +# The Genesis image is the one exception. --genesis builds it once per codename, inside +# that codename's sbuild chroot. use strict; use warnings; use feature 'say'; @@ -38,6 +39,8 @@ use XCAT::BuildUtils qw( reprepro_distributions reprepro_options lock_id_for take_build_lock sh_quote sh sh_or_die usage rewrite_file write_script read_line buildinfo_text + genesis_build_plan genesis_log_errors deb_belongs_to_dist + genesis_dists genesis_dist_reason ); # The xcat-core packages that ship as debs. xCAT-openbmc-py, xCAT-rmc and xCAT-release @@ -61,10 +64,14 @@ my @PACKAGES = qw( my @DISTS = default_dists(); my %opts; -my (@cli_packages, @cli_dists); +my (@cli_packages, @cli_dists, @cli_genesis_dists); GetOptions( "dist=s@" => \@cli_dists, "package=s@" => \@cli_packages, + "genesis" => \$opts{genesis}, + "genesis-only" => \$opts{genesis_only}, + "genesis-dist=s@" => \@cli_genesis_dists, + "genesis-arch=s" => \$opts{genesis_arch}, "dest=s" => \$opts{dest}, "builddir=s" => \$opts{builddir}, "release=s" => \$opts{release}, @@ -83,6 +90,23 @@ $opts{packages} = @cli_packages ? \@cli_packages : \@PACKAGES; $opts{dists} = @cli_dists ? \@cli_dists : \@DISTS; $opts{gpg_key_name} //= 'xCAT Signing Key'; +# The Genesis step is off unless it is asked for, so today's runs keep their behaviour. +$opts{genesis} = 1 if $opts{genesis_only}; +# A release named on the command line is built as asked. The default list is the one the rest +# of the build uses, and not every release on it can build the image, so those are dropped and +# named rather than failing the run on its first codename. +if (@cli_genesis_dists) { + $opts{genesis_dists} = \@cli_genesis_dists; +} else { + $opts{genesis_dists} = [ genesis_dists($opts{dists}->@*) ]; + for my $dist ($opts{dists}->@*) { + my $why = genesis_dist_reason($dist) or next; + say "genesis: leaving out $dist -- $why"; + } +} +die "FATAL: --genesis-dist needs --genesis\n" if @cli_genesis_dists && !$opts{genesis}; +die "FATAL: --genesis-arch needs --genesis\n" if $opts{genesis_arch} && !$opts{genesis}; + for my $pkg ($opts{packages}->@*) { die "FATAL: unknown package '$pkg'. Known: @PACKAGES\n" unless grep { $_ eq $pkg } @PACKAGES; @@ -268,6 +292,132 @@ sub collect_debs { return $moved; } +# ----------------------------------------------------------- the Genesis deb -- +# +# dracut copies the kernel, the kernel modules and every command out of the root it runs in, +# so the Genesis image belongs to the release that built it. One build on the build host +# serves every codename with the build host's kernel. This step builds one image per +# codename, inside that codename's sbuild chroot. +# +# The chroots are the ones xcat-dep's sbuild-all.pl creates on the Ubuntu build host +# (--sbuild). A session is a disposable overlay, so the next codename starts +# from the pristine base. + +# Where the build runs inside the chroot. A directory of its own at the chroot root, because +# every other candidate is shared: /build and /opt/xcat-ci-shared are bind mounts the sbuild +# chroots give to every session. +my $GENESIS_STAGE = '/xcat-genesis-build'; + +sub host_deb_arch { + my $arch = `dpkg --print-architecture 2>/dev/null` // ''; + chomp $arch; + die "FATAL: dpkg does not report a host architecture\n" unless $arch; + return $arch; +} + +# begin_chroot_session: start a disposable schroot session and return its id and its root. +sub begin_chroot_session { + my ($chroot) = @_; + my $id = `schroot --begin-session --chroot @{[ sh_quote($chroot) ]} 2>&1` // ''; + my $rc = $? >> 8; + chomp $id; + die "FATAL: cannot start a session in chroot '$chroot' (exit $rc): $id\n" + . " sbuild-all.pl ensure_chroots creates it; run it on this host first.\n" + if $rc != 0 || $id !~ /\A\S+\z/; + + my $root = `schroot --location -c @{[ sh_quote("session:$id") ]} 2>/dev/null` // ''; + chomp $root; + unless ($root && -d $root) { + sh("schroot --end-session -c " . sh_quote("session:$id") . " >/dev/null 2>&1"); + die "FATAL: schroot reports no location for session:$id\n"; + } + return ($id, $root); +} + +# genesis_build_log_problems: what the log says went wrong when the exit status did not. +# +# Report the first few offending lines only, so a build console stays readable. The message +# names the log file that has the rest. +sub genesis_build_log_problems { + my ($logfile) = @_; + my $text = -f $logfile ? read_text($logfile) : ''; + my @errors = genesis_log_errors($text); + return '' unless @errors; + my $shown = @errors > 10 ? 10 : scalar @errors; + my $report = "FATAL: the Genesis build log reports " . scalar(@errors) . " error(s):\n"; + $report .= " $_->{line}\n ($_->{why})\n" for @errors[0 .. $shown - 1]; + $report .= " ... " . (@errors - $shown) . " more\n" if @errors > $shown; + $report .= " the whole log is at $logfile\n"; + return $report; +} + +sub build_one_genesis_deb { + my ($step, $pkgdir) = @_; + my ($codename, $chroot) = ($step->{codename}, $step->{chroot}); + say "Building $step->{package} for $codename in $chroot"; + + my ($id, $root) = begin_chroot_session($chroot); + my $logfile = "$pkgdir/$step->{package}-$codename.buildlog"; + my $err; + + eval { + # Copy the builder in rather than bind-mount the checkout: the build rewrites + # debian/control and debian/changelog, and it must not rewrite them in the tree the + # pipeline builds from. + my $stage = "$root$GENESIS_STAGE"; + sh_or_die("rm -rf " . sh_quote($stage) . " && mkdir -p " + . sh_quote("$stage/xCAT-genesis-builder"), + "FATAL: cannot make the build directory in session:$id\n"); + sh_or_die("cp -a " . sh_quote("$ROOT/xCAT-genesis-builder") . "/. " + . sh_quote("$stage/xCAT-genesis-builder") . "/", + "FATAL: cannot copy xCAT-genesis-builder into session:$id\n"); + copy("$ROOT/Version", "$stage/Version") + or die "FATAL: cannot copy Version into session:$id: $!\n"; + write_text("$stage/Release", "$RELEASE\n"); + + # --expect-codename is the guard that keeps the image and the root together: the + # builder stops when the root it woke up in is not the release it was asked for. + my $cmd = join ' ', + 'schroot', '--run-session', '-c', sh_quote("session:$id"), '-u', 'root', '-d', '/', + '--', '/bin/bash', "$GENESIS_STAGE/xCAT-genesis-builder/builddeb-genesis-base", + '--expect-codename', sh_quote($codename), '--outdir', "$GENESIS_STAGE/out"; + my $rc = sh("$cmd > " . sh_quote($logfile) . " 2>&1"); + + # The log is read whether or not the command failed: dracut exits 0 with FAILED: + # lines in its log. + my $problems = genesis_build_log_problems($logfile); + if ($rc != 0) { + die "FATAL: the Genesis build for $codename failed (exit $rc); log: $logfile\n" + . $problems; + } + die $problems if $problems; + + my @debs = glob("$root$GENESIS_STAGE/out/*.deb"); + die "FATAL: the Genesis build for $codename produced no .deb; log: $logfile\n" + unless @debs; + for my $deb (@debs) { + my $dest = "$pkgdir/" . basename($deb); + copy($deb, $dest) or die "FATAL: cannot collect $deb: $!\n"; + say " $dest"; + } + 1; + } or $err = $@; + + sh("schroot --end-session -c " . sh_quote("session:$id") . " >/dev/null 2>&1"); + die $err if $err; + return; +} + +sub build_genesis_debs { + my ($pkgdir) = @_; + my $arch = $opts{genesis_arch} || host_deb_arch(); + my @plan = genesis_build_plan($opts{genesis_dists}, $arch); + say "Genesis: @{[ scalar @plan ]} build(s) for $arch: " + . join(' ', map { $_->{codename} } @plan); + build_one_genesis_deb($_, $pkgdir) for @plan; + return scalar @plan; +} + # ------------------------------------------------------------- apt assembly -- sub gpg_key_id { my ($name) = @_; @@ -304,6 +454,9 @@ sub assemble_repo { for my $deb (@debs) { # A release that predates an architecture must not be handed its packages. next if basename($deb) =~ /_(\w+)\.deb\z/ && $1 ne 'all' && !$ok{$1}; + # Nor an image built for another release: the Genesis deb carries the codename it + # was built on, because it carries that release's kernel. + next unless deb_belongs_to_dist($deb, $dist); sh_or_die("cd " . sh_quote($repodir) . " && reprepro -b ./ includedeb " . sh_quote($dist) . ' ' . sh_quote($deb), "FATAL: reprepro could not add $deb to $dist\n"); @@ -369,16 +522,25 @@ unlink glob("$ROOT/*.deb"), glob("$ROOT/*.buildinfo"), glob("$ROOT/*.changes"), say "xcat-core $PKGVER -> $dest"; say "releases: @{[ join ' ', $opts{dists}->@* ]}"; -for my $pkg ($opts{packages}->@*) { - for my $arch (deb_package_arches($pkg)) { - build_package($pkg, $arch, $pkgdir); +unless ($opts{genesis_only}) { + for my $pkg ($opts{packages}->@*) { + for my $arch (deb_package_arches($pkg)) { + build_package($pkg, $arch, $pkgdir); + } + collect_debs($pkg, $pkgdir); } - collect_debs($pkg, $pkgdir); } -my $count = assemble_repo($pkgdir, $repo); -write_repo_metadata($repo); -say "published $count package(s) into @{[ scalar $opts{dists}->@* ]} release(s) at $repo"; +build_genesis_debs($pkgdir) if $opts{genesis}; + +if ($opts{genesis_only}) { + say "Genesis debs are in $pkgdir"; +} +else { + my $count = assemble_repo($pkgdir, $repo); + write_repo_metadata($repo); + say "published $count package(s) into @{[ scalar $opts{dists}->@* ]} release(s) at $repo"; +} __END__ @@ -399,8 +561,15 @@ xcat-core packages are Perl. The same binary serves every Ubuntu release, so eac package is built B and the resulting C<.deb> files are published into every codename the repository declares. Only C, C and C carry an architecture, and there the difference is packaging metadata rather than -compiled output. Consequently this builder needs no C and no per-codename -chroot. (xcat-dep is different: its packages are compiled, so it builds per codename.) +compiled output. + +C is the exception. dracut copies the kernel, the kernel modules +and every command out of the root it runs in, so the Genesis image belongs to the +release that built it. With C<--genesis> this builder makes one image per codename, +each inside that codename's C<< --sbuild >> schroot -- the chroots +xcat-dep's C creates on the Ubuntu build host. The build refuses to run +in a root of another release, and it reads its own log: dracut reports a command it +cannot install with a C line and still exits 0. Replaced C, removed in 2.19. The GSA upload paths, the C/C release flows and the C<-d> xcat-dep repository mode were not carried over: publishing is done @@ -418,6 +587,25 @@ Publish into this release. Repeatable. Defaults to focal, jammy, noble and resol Build only this package. Repeatable. Defaults to every xcat-core deb package. +=item B<--genesis> + +Also build CarchE>, one C<.deb> per codename, each inside +that codename's schroot. Off by default. + +=item B<--genesis-only> + +Build the Genesis debs and nothing else, and assemble no repository. This is what +xcat-dep needs: it consumes the debs with C. + +=item B<--genesis-dist>=I + +Build the Genesis image for this release. Repeatable. Defaults to the C<--dist> list. + +=item B<--genesis-arch>=I + +Build the Genesis image for this Debian architecture. Defaults to the architecture of +the build host, because the chroot has to match it. + =item B<--dest>=I Write the build under this directory: packages in C, the apt repository in @@ -459,5 +647,6 @@ This message. ./builddebs.pl ./builddebs.pl --dist noble --package perl-xCAT ./builddebs.pl --dest /srv/out --gpg-sign --gpg-home /keys/xcat-gpg-home + ./builddebs.pl --genesis-only --genesis-dist jammy --genesis-dist noble --dest /srv/out =cut diff --git a/buildrpms.pl b/buildrpms.pl index 42b87bb7a..5e46bd20e 100755 --- a/buildrpms.pl +++ b/buildrpms.pl @@ -349,6 +349,9 @@ sub buildsources_genesis_base($) { # %install runs this against the extracted payload before it becomes an rpm. cp "xCAT-genesis-builder/verify-genesis-payload", "$staging_root/verify-genesis-payload"; + make_path("$staging_root/lib/XCAT"); + cp "xCAT-genesis-builder/lib/XCAT/GenesisPayload.pm", + "$staging_root/lib/XCAT/GenesisPayload.pm"; unlink $support_tarball if -f $support_tarball; sh_or_die(qq(tar --sort=name --owner=0 --group=0 --mtime="\@$SOURCE_DATE_EPOCH" -cjf "$support_tarball" -C "$staging_parent" xCAT-genesis-base-build-support), diff --git a/docs/source/developers/guides/code/builds.rst b/docs/source/developers/guides/code/builds.rst index 021428aae..183910417 100644 --- a/docs/source/developers/guides/code/builds.rst +++ b/docs/source/developers/guides/code/builds.rst @@ -63,6 +63,20 @@ ppc64el, and every release the repository serves declares the architecture; ``xCAT-genesis-scripts`` keeps the two architectures it has control files for, because riscv64 Genesis ships as an OpenEmbedded package instead. +The Genesis image is the exception, and it is off unless it is asked for:: + + ./builddebs.pl --genesis-only --genesis-dist jammy --genesis-dist noble + +``dracut`` copies the kernel, the kernel modules and every command out of the root +it runs in, so the image belongs to the release that built it. ``--genesis`` builds +one image per ``--genesis-dist`` codename inside that codename's +``--sbuild`` schroot -- the chroots xcat-dep's ``sbuild-all.pl`` +creates on the Ubuntu build host -- for the architecture of the build host. It +refuses to run in a root of another release, it fails the build on an error in the +log even when the exit status is 0, and it checks the extracted payload against the +commands the dracut module installs. ``--genesis-only`` builds the images and +nothing else, which is what xcat-dep consumes with ``--genesis-deb``. + Helpers shared by both builders live in ``build-utils/lib/XCAT/BuildUtils.pm``. ``buildcore.sh`` builds the architecture specific packages (``xCAT``, ``xCATsn``, diff --git a/xCAT-genesis-builder/builddeb-genesis-base b/xCAT-genesis-builder/builddeb-genesis-base index 5b9bc9141..41b0696d5 100755 --- a/xCAT-genesis-builder/builddeb-genesis-base +++ b/xCAT-genesis-builder/builddeb-genesis-base @@ -1,13 +1,35 @@ #!/bin/bash -# Build xcat-genesis-base .deb package natively on Ubuntu. -# Must run as root on an Ubuntu system (22.04, 24.04, or 26.04). -# Parallel to buildrpm for EL targets. +# Build the xcat-genesis-base .deb for ONE Ubuntu codename. +# +# dracut copies the kernel, the kernel modules and every command out of the root it runs +# in, so this must run inside a root of the target codename. builddebs.pl --genesis starts +# it in that codename's --sbuild chroot, one build per codename. +# --expect-codename is what stops a run on the build host: a single build there gives every +# Ubuntu release the build host's kernel. +# +# Parallel to xCAT-genesis-base.spec, which does the same for EL. set -euo pipefail DIR=$(readlink -f "$(dirname "$0")") +OS_RELEASE=${OS_RELEASE:-/etc/os-release} +expect_codename="" +outdir="" + +while [ $# -gt 0 ]; do + case "$1" in + --expect-codename) expect_codename=${2:-}; shift 2 ;; + --expect-codename=*) expect_codename=${1#*=}; shift ;; + --outdir) outdir=${2:-}; shift 2 ;; + --outdir=*) outdir=${1#*=}; shift ;; + -h|--help) + echo "usage: builddeb-genesis-base [--expect-codename ] [--outdir ]" + exit 0 ;; + *) echo "ERROR: unknown argument: $1" >&2; exit 2 ;; + esac +done + BUILDARCH=$(dpkg --print-architecture) -TRIPLET=$(dpkg-architecture -qDEB_HOST_MULTIARCH) case "$BUILDARCH" in amd64) TARCH=x86_64 ;; @@ -31,44 +53,45 @@ rewrite_control() { VERSION=$(cat "$DIR/../Version" 2>/dev/null || echo "2.18.0") RELEASE=$(cat "$DIR/../Release" 2>/dev/null || echo "snap$(date +%Y%m%d%H%M)") -CODENAME=$(. /etc/os-release && echo "$VERSION_CODENAME") +CODENAME=$(. "$OS_RELEASE" && echo "${VERSION_CODENAME:-}") + +if [ -z "$CODENAME" ]; then + echo "ERROR: $OS_RELEASE names no VERSION_CODENAME" >&2 + exit 1 +fi + +# The image is only valid for the release whose kernel it carries. Refuse to build a +# codename's image anywhere but in that codename's root. +if [ -n "$expect_codename" ] && [ "$expect_codename" != "$CODENAME" ]; then + echo "ERROR: this root is $CODENAME, not $expect_codename." >&2 + echo " The image would carry the $CODENAME kernel and boot under $expect_codename." >&2 + exit 1 +fi echo "Building xcat-genesis-base for $BUILDARCH ($TARCH) on Ubuntu $CODENAME" export DEBIAN_FRONTEND=noninteractive -REQUIRED_PACKAGES=" - dracut linux-image-generic - ipmitool lldpad ethtool iproute2 kexec-tools screen - openssh-server openssh-client rsyslog chrony - nfs-common rpcbind pciutils usbutils parted - dosfstools e2fsprogs lvm2 mdadm net-tools - bc psmisc rsync wget cpio - isc-dhcp-client ifenslave - dpkg-dev debhelper fakeroot devscripts vim-tiny -" -if [ "$BUILDARCH" = "amd64" ]; then - REQUIRED_PACKAGES="$REQUIRED_PACKAGES dmidecode efibootmgr" -fi - -# Keep only the packages this release actually has. hwclock moved out of util-linux into -# util-linux-extra, so focal and jammy have no such package and naming it there fails the -# whole install. util-linux only Suggests it, and --no-install-recommends is passed, so the -# releases that did split it must name it. -optional_packages() { - local package - for package in "$@"; do - if apt-cache show "$package" >/dev/null 2>&1; then - echo "$package" - fi - done -} - echo "Installing build dependencies..." apt-get update -qq -REQUIRED_PACKAGES="$REQUIRED_PACKAGES $(optional_packages util-linux-extra)" + +# XCAT::GenesisBuildRoot holds the package list, so the unit tests can call it. +REQUIRED_PACKAGES=$(perl -I"$DIR/lib" -MXCAT::GenesisBuildRoot=required_packages \ + -e 'print "$_\n" for required_packages(@ARGV)' "$BUILDARCH" "$CODENAME") apt-get install -y --no-install-recommends $REQUIRED_PACKAGES +# dpkg-architecture comes from dpkg-dev, which the line above installs. +TRIPLET=$(dpkg-architecture -qDEB_HOST_MULTIARCH) + +# ncurses-base put its terminfo entries under /lib/terminfo before 24.04, and both dracut's own +# terminfo module and the 97xcat module ask for /usr/share/terminfo. On jammy the build stopped +# on /usr/share/terminfo/l/linux. Give this build root the path they ask for; the chroot is +# thrown away when the build ends. +if [ -d /lib/terminfo ]; then + mkdir -p /usr/share/terminfo + cp -an /lib/terminfo/. /usr/share/terminfo/ 2>/dev/null || true +fi + # Set up dracut module if [ -d /usr/lib/dracut/modules.d ]; then DRACUT_PARENT=/usr/lib/dracut/modules.d @@ -100,6 +123,8 @@ if [ "$BUILDARCH" != "amd64" ]; then sed -i '/efibootmgr dmidecode/d' "$DRACUTMODDIR/module-setup.sh" fi +# linux-image-generic pulls exactly one kernel into this root, and this root belongs to +# $CODENAME, so this is the target release's kernel. KERNELVERSION=$(ls -1 /lib/modules | sort -V | tail -n 1) if [ -z "$KERNELVERSION" ]; then echo "ERROR: no kernel modules found in /lib/modules" >&2 @@ -152,24 +177,53 @@ if [ ! -e "$KERNEL_IMAGE" ]; then for candidate in \ "/boot/vmlinux-$KERNELVERSION" \ "/usr/lib/modules/$KERNELVERSION/vmlinuz" \ - "/lib/modules/$KERNELVERSION/vmlinuz" \ - "$(find /usr/lib/modules/"$KERNELVERSION" -maxdepth 2 -name 'vmlinuz*' -o -name 'vmlinux*' 2>/dev/null | head -n 1)" \ - "$(find /lib/modules/"$KERNELVERSION" -maxdepth 2 -name 'vmlinuz*' -o -name 'vmlinux*' 2>/dev/null | head -n 1)" \ - "$(ls -1 /boot/vmlinuz-* /boot/vmlinux-* 2>/dev/null | sort -V | tail -n 1)" + "/lib/modules/$KERNELVERSION/vmlinuz" do - if [ -n "$candidate" ] && [ -e "$candidate" ]; then + if [ -e "$candidate" ]; then KERNEL_IMAGE="$candidate" break fi done fi if [ ! -e "$KERNEL_IMAGE" ]; then - echo "ERROR: cannot find kernel image" >&2 + echo "ERROR: cannot find the image of kernel $KERNELVERSION" >&2 exit 1 fi echo "Adding kernel $KERNEL_IMAGE" cp "$KERNEL_IMAGE" "$GENESIS_ROOT/kernel" +# dracut_install reports a missing command and returns, so a hole reaches the .deb with +# nothing in the log but one line. Read the commands back from the module and check them +# against the payload, the way xCAT-genesis-base.spec does for EL. +bash "$DIR/verify-genesis-payload" --commands-from "$DRACUTMODDIR/module-setup.sh" "$GENESIS_FS" \ + usr/sbin/dhclient bin/sh sbin/xcatroot sbin/dhclient-script etc/rsyslog.conf + +# What the verifier cannot know: that this image belongs to this chroot's kernel. An image +# built against another root's /lib/modules boots and then finds no driver for its NIC. +if [ ! -d "$GENESIS_FS/lib/modules/$KERNELVERSION" ]; then + echo "ERROR: the image carries no /lib/modules/$KERNELVERSION" >&2 + ls -1 "$GENESIS_FS/lib/modules" 2>/dev/null >&2 || true + exit 1 +fi +if [ -z "$(find "$GENESIS_FS/lib/modules/$KERNELVERSION" -name '*.ko*' -print -quit)" ]; then + echo "ERROR: /lib/modules/$KERNELVERSION in the image holds no kernel module" >&2 + exit 1 +fi +for stray in "$GENESIS_FS"/lib/modules/*; do + [ -d "$stray" ] || continue + if [ "$(basename "$stray")" != "$KERNELVERSION" ]; then + echo "ERROR: the image carries $(basename "$stray"), not the $KERNELVERSION of this root" >&2 + exit 1 + fi +done + +# The 97xcat hooks are what makes the image xCAT's. dracut drops a module whose check() +# fails without a word, and the image then boots to a plain dracut shell. +if [ -z "$(find "$GENESIS_FS" -path '*/hooks/cmdline/*xcat-cmdline.sh' -print -quit)" ]; then + echo "ERROR: the image carries no 97xcat cmdline hook" >&2 + exit 1 +fi + find "$GENESIS_TMPDIR" -type c -delete # Stage for dpkg-buildpackage @@ -178,6 +232,11 @@ cp -a "$GENESIS_TMPDIR/opt" "$DIR/" # Adjust control file for target arch rewrite_control "$DIR/debian/control" "$BUILDARCH" +# debian/dirs names the image directory, which is the rpm architecture. +echo "/opt/xcat/share/xcat/netboot/genesis/$TARCH/" > "$DIR/debian/dirs" + +# dch reads debian/control from the current directory, not from the file it writes. +cd "$DIR" PKG_VERSION="${VERSION}-${RELEASE}~${CODENAME}" rm -f "$DIR/debian/changelog" @@ -187,8 +246,14 @@ dch --create --package "xcat-genesis-base-$BUILDARCH" \ "Native Ubuntu build on $CODENAME $BUILDARCH" echo "Building .deb package..." -cd "$DIR" dpkg-buildpackage -rfakeroot -uc -us -b -echo "Build complete. .deb files:" -ls -la "$DIR/../"xcat-genesis-base*.deb 2>/dev/null || echo "Check parent directory for .deb files" +if [ -n "$outdir" ]; then + mkdir -p "$outdir" + mv "$DIR/../"xcat-genesis-base-*_*.deb "$outdir/" + echo "Build complete. .deb files in $outdir:" + ls -la "$outdir" +else + echo "Build complete. .deb files:" + ls -la "$DIR/../"xcat-genesis-base*.deb 2>/dev/null || echo "Check parent directory for .deb files" +fi diff --git a/xCAT-genesis-builder/lib/XCAT/GenesisBuildRoot.pm b/xCAT-genesis-builder/lib/XCAT/GenesisBuildRoot.pm new file mode 100644 index 000000000..58c981a3f --- /dev/null +++ b/xCAT-genesis-builder/lib/XCAT/GenesisBuildRoot.pm @@ -0,0 +1,88 @@ +package XCAT::GenesisBuildRoot; + +# builddeb-genesis-base runs this module inside an Ubuntu build root, where only perl-base is +# installed. Use core modules only. +use strict; +use warnings; +use Exporter 'import'; + +our @EXPORT_OK = qw(required_packages apt_carries); + +my @BASE_PACKAGES = qw( + dracut linux-image-generic + ipmitool lldpad ethtool iproute2 kexec-tools screen + openssh-server openssh-client rsyslog chrony + nfs-common rpcbind pciutils usbutils parted + dosfstools e2fsprogs lvm2 mdadm net-tools + bc psmisc rsync wget cpio + isc-dhcp-client ifenslave + systemd-sysv hwdata btrfs-progs netcat-openbsd iputils-ping fdisk ncurses-term + dpkg-dev debhelper fakeroot devscripts vim-tiny +); + +# Two commands the image needs changed package between releases: nslookup left dnsutils for +# bind9-dnsutils in 22.04, and hwclock left util-linux for util-linux-extra in 23.04. +my @RENAMED_PACKAGES = ([qw(bind9-dnsutils dnsutils)], [qw(util-linux-extra util-linux)]); + +#------------------------------------------------------------------------------- + +=head3 apt_carries + + Descriptions: ask apt-cache whether the release carries a package. + Arguments: + $package: the package name + Returns: + 1 when apt-cache knows the package, 0 when it does not + +=cut + +#------------------------------------------------------------------------------- +sub apt_carries { + my ($package) = @_; + return system('sh', '-c', 'apt-cache show "$1" >/dev/null 2>&1', 'sh', $package) == 0 + ? 1 : 0; +} + +#------------------------------------------------------------------------------- + +=head3 required_packages + + Descriptions: list the packages the Genesis build root needs. + For a renamed package, apt says which name the release carries, so the list does + not branch on the codename. 26.04 moved the backward-compatibility zone names that + the dracut module installs out of tzdata into tzdata-legacy. + Arguments: + $arch: the dpkg architecture (amd64, ppc64el) + $codename: the release name, used in the error message + $carries: optional code ref. It takes a package name and returns true when the + release carries that package. The default is apt_carries. + Returns: + the package names, in install order. + Dies with "ERROR: carries none of these packages: ..." when the release + carries neither name of a renamed package. + +=cut + +#------------------------------------------------------------------------------- +sub required_packages { + my ($arch, $codename, $carries) = @_; + $carries ||= \&apt_carries; + + my @packages = @BASE_PACKAGES; + push @packages, qw(dmidecode efibootmgr) if $arch eq 'amd64'; + + RENAMED: + for my $alternatives (@RENAMED_PACKAGES) { + for my $package (@$alternatives) { + if ($carries->($package)) { + push @packages, $package; + next RENAMED; + } + } + die "ERROR: $codename carries none of these packages: @$alternatives\n"; + } + push @packages, 'tzdata-legacy' if $carries->('tzdata-legacy'); + return @packages; +} + +1; diff --git a/xCAT-genesis-builder/lib/XCAT/GenesisPayload.pm b/xCAT-genesis-builder/lib/XCAT/GenesisPayload.pm new file mode 100644 index 000000000..135bd925e --- /dev/null +++ b/xCAT-genesis-builder/lib/XCAT/GenesisPayload.pm @@ -0,0 +1,196 @@ +package XCAT::GenesisPayload; + +# verify-genesis-payload runs this module in the rpm %install of xCAT-genesis-base and inside +# the Ubuntu build root, where only perl-base is installed. Use core modules only. +use strict; +use warnings; +use Exporter 'import'; + +our @EXPORT_OK = qw(module_commands missing_paths check_payload main); + +my $ME = 'verify-genesis-payload'; + +#------------------------------------------------------------------------------- + +=head3 module_commands + + Descriptions: read back the names a dracut module installs. + Only the top level of install() counts. A name under a condition is + release-dependent, so the caller names it as a required path instead. + An option to dracut_install (a word that starts with "-") is not a name. + Arguments: + $module_setup: the path of the module-setup.sh + Returns: + the names, sorted, each one once. A name that starts with "/" is an + absolute path; any other name is a command. + Dies with "verify-genesis-payload: cannot read " when the file + cannot be read, and with "verify-genesis-payload: no command name read + from " when install() names nothing. + +=cut + +#------------------------------------------------------------------------------- +sub module_commands { + my ($module_setup) = @_; + open(my $fh, '<', $module_setup) + or die "$ME: cannot read $module_setup\n"; + + my ($in_install, %names); + while (my $line = <$fh>) { + if ($line =~ /^install\(\)/) { $in_install = 1; next } + $in_install = 0 if $in_install && $line =~ /^}/; + next unless $in_install && $line =~ s/^ dracut_install //; + $line =~ s/#.*//s; + $names{$_} = 1 for grep { length && !/^-/ } split /\s+/, $line; + } + close($fh); + + die "$ME: no command name read from $module_setup\n" unless %names; + return sort keys %names; +} + +#------------------------------------------------------------------------------- + +=head3 missing_paths + + Descriptions: list what a Genesis payload lacks. + dracut_install reports a missing binary and returns, so the image can + ship without it. This check runs on the extracted payload before it is + packaged. + Arguments: + $have: code ref. It takes a path relative to the payload root and + returns true when the payload carries it. + %opt: + required: paths relative to the payload root that the caller needs + commands: names from module_commands. A bare command is looked + for in bin, sbin, usr/bin and usr/sbin; an absolute + path is looked for under the payload root. + source: the module the commands came from, for the message + sshd: the content of usr/sbin/sshd, or undef without one + Returns: + one " ()" string per missing item, in the order checked. + An empty list means the payload is complete. + +=cut + +#------------------------------------------------------------------------------- +sub missing_paths { + my ($have, %opt) = @_; + my @missing; + + for my $path (@{ $opt{required} || [] }) { + push @missing, "$path (required by the build)" unless $have->($path); + } + + for my $want (@{ $opt{commands} || [] }) { + my $found = + $want =~ m{^/(.*)} + ? $have->($1) + : scalar(grep { $have->("$_/$want") } qw(bin sbin usr/bin usr/sbin)); + push @missing, "$want (installed by $opt{source})" unless $found; + } + + push @missing, "usr/sbin/sshd (Genesis is reached over ssh)" + unless $have->('usr/sbin/sshd'); + push @missing, "usr/bin/mktemp (getdestiny makes its request file with it)" + unless $have->('usr/bin/mktemp'); + + # OpenSSH 9.8 split the per-connection work into sshd-session, which sshd execs by + # absolute path. EL9 carries OpenSSH 9.9, so an image with sshd alone refuses every + # connection. + if (index($opt{sshd} // '', 'sshd-session') >= 0 + && !$have->('usr/libexec/openssh/sshd-session') + && !$have->('usr/lib/openssh/sshd-session')) + { + push @missing, + "usr/libexec/openssh/sshd-session (this sshd execs it for every connection)"; + } + + # tmux exits under the C locale. The hook then runs doxcat directly, but a Genesis + # shell without tmux loses the console attach. + if ($have->('usr/bin/tmux') && !$have->('usr/lib/locale/C.utf8/LC_CTYPE')) { + push @missing, + "usr/lib/locale/C.utf8/LC_CTYPE (tmux refuses to start without a UTF-8 locale)"; + } + return @missing; +} + +#------------------------------------------------------------------------------- + +=head3 check_payload + + Descriptions: the command line of verify-genesis-payload, without the output: + [--commands-from ] [required-path ...] + Arguments: + @args: the command line + Returns: + ($status, $message). $status is the exit status: 0 when the payload is + complete, 1 when it lacks something, 2 on a usage error. $message is + the text to print, one line or a list of missing items. + +=cut + +#------------------------------------------------------------------------------- +sub check_payload { + my @args = @_; + my $commands_from = ''; + while (@args) { + if ($args[0] eq '--commands-from') { + shift @args; + $commands_from = shift(@args) // ''; + } elsif ($args[0] =~ /^--commands-from=(.*)/s) { + $commands_from = $1; + shift @args; + } else { + last; + } + } + + my $payload = shift(@args) // ''; + if ($payload eq '' || !-d $payload) { + return (2, "$ME: not a payload directory: " . ($payload eq '' ? '' : $payload) . "\n"); + } + + my @commands; + if ($commands_from ne '') { + @commands = eval { module_commands($commands_from) }; + return (2, $@) if $@; + } + + my $sshd; + if (open(my $fh, '<:raw', "$payload/usr/sbin/sshd")) { + local $/; + $sshd = <$fh> // ''; + close($fh); + } + my @missing = missing_paths(sub { -e "$payload/$_[0]" }, + required => \@args, + commands => \@commands, + source => $commands_from, + sshd => $sshd); + return (1, "$ME: $payload is incomplete:" . join('', map { "\n $_" } @missing) . "\n") + if @missing; + return (0, "$ME: $payload is complete\n"); +} + +#------------------------------------------------------------------------------- + +=head3 main + + Descriptions: run check_payload and print its message: on STDOUT when the + payload is complete, on STDERR otherwise. + Arguments: + @args: the command line + Returns: + the exit status from check_payload + +=cut + +#------------------------------------------------------------------------------- +sub main { + my ($status, $message) = check_payload(@_); + print { $status ? *STDERR : *STDOUT } $message; + return $status; +} + +1; diff --git a/xCAT-genesis-builder/verify-genesis-payload b/xCAT-genesis-builder/verify-genesis-payload index 1f44898c1..d5d0369b0 100755 --- a/xCAT-genesis-builder/verify-genesis-payload +++ b/xCAT-genesis-builder/verify-genesis-payload @@ -6,115 +6,9 @@ # going and the image ships without it. Check the extracted payload before it is packaged. # # Paths given on the command line are relative to . --commands-from reads back -# what the dracut module installs: a bare command name is looked for in the four binary -# directories, an absolute path under itself. The caller adds what only it -# knows (the DHCP client is not the same package on every release); the rules below come from -# the payload itself. +# what the dracut module installs. XCAT::GenesisPayload, in lib/ beside this script, holds +# the rules, so the unit tests call the same code. +# +# Exit status: 0 complete, 1 something missing, 2 usage error. -set -u - -commands_from="" -while [ $# -gt 0 ]; do - case "$1" in - --commands-from) - commands_from=${2:-} - shift 2 || true - ;; - --commands-from=*) - commands_from=${1#*=} - shift - ;; - *) - break - ;; - esac -done - -payload=${1:-} -if [ -z "$payload" ] || [ ! -d "$payload" ]; then - echo "verify-genesis-payload: not a payload directory: ${payload:-}" >&2 - exit 2 -fi -shift - -missing="" - -# have PATH: true when the payload carries PATH as a file, following the usr-merge symlinks -# the image ships (/sbin -> usr/sbin). -have() { - [ -e "$payload/$1" ] -} - -require() { - local path=$1 why=$2 - have "$path" || missing="$missing - $path ($why)" -} - -for path in "$@"; do - require "$path" "required by the build" -done - -# The dracut module names every command and every data file Genesis needs. A name the build -# root does not supply installs nothing and says nothing, so read the names back and check -# each one. Names under a condition are release-dependent, so only the top level of install() -# counts. -if [ -n "$commands_from" ]; then - if [ ! -r "$commands_from" ]; then - echo "verify-genesis-payload: cannot read $commands_from" >&2 - exit 2 - fi - commands=$(awk ' - /^install\(\)/ { in_install = 1; next } - in_install && /^}/ { in_install = 0 } - in_install && /^ dracut_install / { - sub(/#.*/, "") - sub(/^ dracut_install /, "") - print - }' "$commands_from" | tr ' \t' '\n\n' | grep -v '^$' | grep -v '^-' | sort -u) - if [ -z "$commands" ]; then - echo "verify-genesis-payload: no command name read from $commands_from" >&2 - exit 2 - fi - for want in $commands; do - case "$want" in - # dracut_install installs an absolute path at that same path, so read it back - # under the payload root. Dropping these let an image with no /usr/bin/awk pass. - /*) have "${want#/}" || missing="$missing - $want (installed by $commands_from)" - ;; - *) have "bin/$want" || have "sbin/$want" \ - || have "usr/bin/$want" || have "usr/sbin/$want" \ - || missing="$missing - $want (installed by $commands_from)" - ;; - esac - done -fi - -require usr/sbin/sshd "Genesis is reached over ssh" -require usr/bin/mktemp "getdestiny makes its request file with it" - -# OpenSSH 9.8 split the per-connection work into sshd-session, which sshd execs by absolute -# path. EL9 carries OpenSSH 9.9, so an image with sshd alone refuses every connection. -if have usr/sbin/sshd && grep -qa 'sshd-session' "$payload/usr/sbin/sshd" 2>/dev/null; then - if ! have usr/libexec/openssh/sshd-session && ! have usr/lib/openssh/sshd-session; then - missing="$missing - usr/libexec/openssh/sshd-session (this sshd execs it for every connection)" - fi -fi - -# tmux exits under the C locale. The hook falls back to running doxcat directly, so this is -# not fatal to booting, but a Genesis shell without tmux loses the console attach. -if have usr/bin/tmux && ! have usr/lib/locale/C.utf8/LC_CTYPE; then - missing="$missing - usr/lib/locale/C.utf8/LC_CTYPE (tmux refuses to start without a UTF-8 locale)" -fi - -if [ -n "$missing" ]; then - echo "verify-genesis-payload: $payload is incomplete:$missing" >&2 - exit 1 -fi - -echo "verify-genesis-payload: $payload is complete" -exit 0 +exec perl -I"$(dirname "$0")/lib" -MXCAT::GenesisPayload=main -e 'exit main(@ARGV)' -- "$@" diff --git a/xCAT-test/unit/genesis_deb_per_codename.t b/xCAT-test/unit/genesis_deb_per_codename.t new file mode 100644 index 000000000..9b732500d --- /dev/null +++ b/xCAT-test/unit/genesis_deb_per_codename.t @@ -0,0 +1,175 @@ +#!/usr/bin/env perl +# The Genesis image carries the kernel and the kernel modules of the root that built it, so +# builddebs.pl --genesis builds one image per codename, in that codename's chroot. Each +# assertion here reads the value the code returns. +use strict; +use warnings; + +use File::Path qw(make_path); +use File::Slurper qw(read_text write_text); +use File::Temp qw(tempdir); +use FindBin; +use lib "$FindBin::Bin/../lib"; +use lib "$FindBin::Bin/../../build-utils/lib"; +use Test::More; + +use XCAT::Test::File qw(repo_path); + +require XCAT::BuildUtils; + +my @WANTED = qw(genesis_chroot_name genesis_target_arch genesis_build_plan genesis_log_errors); +for my $sub (@WANTED) { + ok(XCAT::BuildUtils->can($sub), "XCAT::BuildUtils provides $sub"); +} +unless (scalar(grep { XCAT::BuildUtils->can($_) } @WANTED) == scalar @WANTED) { + diag('builddebs.pl has no Genesis step: the per-codename build does not exist yet'); + done_testing(); + exit; +} + +# --- one build per codename, in that codename's chroot --------------------------------- +my @dists = qw(jammy noble resolute); +my @plan = XCAT::BuildUtils::genesis_build_plan(\@dists, 'amd64'); + +is(scalar @plan, scalar @dists, 'one Genesis build per codename'); +is_deeply([ map { $_->{codename} } @plan ], \@dists, 'the plan keeps the codename order'); +is_deeply([ map { $_->{chroot} } @plan ], + [ 'jammy-amd64-sbuild', 'noble-amd64-sbuild', 'resolute-amd64-sbuild' ], + 'each codename builds in its own sbuild chroot'); +is_deeply([ map { $_->{package} } @plan ], + [ ('xcat-genesis-base-amd64') x 3 ], + 'every codename produces the same package name'); + +my @ppc = XCAT::BuildUtils::genesis_build_plan(['noble'], 'ppc64el'); +is($ppc[0]{chroot}, 'noble-ppc64el-sbuild', 'the architecture selects the chroot'); +is($ppc[0]{package}, 'xcat-genesis-base-ppc64el', 'the architecture is in the package name'); +is($ppc[0]{target}, 'ppc64', 'ppc64el reads its image from the ppc64 directory'); +is(XCAT::BuildUtils::genesis_target_arch('amd64'), 'x86_64', + 'amd64 reads its image from the x86_64 directory'); + +is(scalar(() = XCAT::BuildUtils::genesis_build_plan([qw(noble noble)], 'amd64')), 1, + 'a repeated codename does not build twice'); + +ok(!eval { XCAT::BuildUtils::genesis_build_plan([], 'amd64'); 1 }, + 'a plan with no codename is an error'); +ok(!eval { XCAT::BuildUtils::genesis_target_arch('riscv64'); 1 }, + 'an architecture with no Genesis image directory is an error'); + +# --- a per-codename image reaches only its own suite -------------------------------------- +ok(XCAT::BuildUtils->can('deb_belongs_to_dist'), + 'XCAT::BuildUtils decides which suite a deb belongs to'); +if (XCAT::BuildUtils->can('deb_belongs_to_dist')) { + my $noble = 'xcat-genesis-base-amd64_2.19.0-snap202609121200~noble_all.deb'; + ok(XCAT::BuildUtils::deb_belongs_to_dist($noble, 'noble'), + 'the noble image is published into noble'); + ok(!XCAT::BuildUtils::deb_belongs_to_dist($noble, 'jammy'), + 'the noble image is not published into jammy'); + # Everything else in xcat-core is the same file for every release. + ok(XCAT::BuildUtils::deb_belongs_to_dist('perl-xcat_2.19.0-snap1_all.deb', 'jammy'), + 'a deb with no codename in its version reaches every suite'); + ok(XCAT::BuildUtils::deb_belongs_to_dist('xcat_2.19.0-snap1_amd64.deb', 'resolute'), + 'an architecture deb reaches every suite'); + + # A `~` in a version is not a codename. Debian uses it for a prerelease, and --release + # takes whatever the caller gives it, so `--release 1~rc1` puts one in every package + # name. Reading it as a codename drops the whole build from every suite, and the run + # still reports the packages it published. + for my $dist (qw(focal jammy noble resolute)) { + ok(XCAT::BuildUtils::deb_belongs_to_dist('xcat-client_2.19.0-1~rc1_all.deb', $dist), + "a prerelease version reaches $dist"); + } + ok(XCAT::BuildUtils::deb_belongs_to_dist('xcat-genesis-scripts-amd64_2.19.0-1~rc1_amd64.deb', 'noble'), + 'a prerelease Genesis SCRIPTS deb reaches every suite: only the image is per codename'); + + # And the rule the exclusion exists for still holds under a prerelease version. + ok(XCAT::BuildUtils::deb_belongs_to_dist('xcat-genesis-base-amd64_2.19.0-1~rc1~noble_amd64.deb', 'noble'), + 'a prerelease Genesis image reaches its own suite'); + ok(!XCAT::BuildUtils::deb_belongs_to_dist('xcat-genesis-base-amd64_2.19.0-1~rc1~noble_amd64.deb', 'jammy'), + 'and no other'); +} + +# --- the releases a Genesis image can be built on ------------------------------------------ +# A plain --genesis run takes the release list the rest of the build uses. focal is on it and +# cannot build the package: it ships debhelper 12.10 and xCAT-genesis-base declares +# debhelper-compat (= 13), so sbuild stops on the build dependencies before dracut runs and +# the run ends on its first release. +ok(XCAT::BuildUtils->can('genesis_dists'), + 'XCAT::BuildUtils says which releases a Genesis image can be built on'); +if (XCAT::BuildUtils->can('genesis_dists')) { + is_deeply([ XCAT::BuildUtils::genesis_dists(XCAT::BuildUtils::default_dists()) ], + [qw(jammy noble resolute)], + 'the default plan leaves out the release whose chroot cannot build the package'); + is_deeply([ XCAT::BuildUtils::genesis_dists(qw(jammy noble)) ], [qw(jammy noble)], + 'a list with none of them is unchanged'); + is_deeply([ XCAT::BuildUtils::genesis_dists('focal') ], [], + 'a list of only that release plans nothing'); +} + +# --- the log guard --------------------------------------------------------------------- +# +# dracut prints FAILED: for a command it cannot install and exits 0. +my $dracut_log = <<'LOG'; +Installing build dependencies... +dracut: Executing: /usr/bin/dracut --compress gzip -m xcat base -N -f /tmp/genesis.rfs 6.8.0-45-generic +dracut-install: ERROR: installing 'dhclient' +dracut: FAILED: /usr/lib/dracut/dracut-install -D /var/tmp/dracut.XXXX -a dhclient +dracut: *** Creating initramfs image file '/tmp/genesis.rfs' done *** +Extracting initramfs... +LOG + +my @errors = XCAT::BuildUtils::genesis_log_errors($dracut_log); +ok(scalar @errors, 'a dracut log with a FAILED: line is an error'); +like($errors[0]{line}, qr/FAILED:/, 'the offending line is reported'); +ok(length $errors[0]{why}, 'the reason is named'); + +is_deeply([ XCAT::BuildUtils::genesis_log_errors(<<'LOG') ], [], 'a clean build log is not an error'); +Installing build dependencies... +dracut: *** Creating initramfs image file '/tmp/genesis.rfs' done *** +Extracting initramfs... +dpkg-deb: building package 'xcat-genesis-base-amd64' +LOG + +for my $case ( + [ 'E: Unable to locate package isc-dhcp-client' => 'a package apt cannot find' ], + [ 'dracut: Cannot find module directory /lib/modules/6.8.0' => 'a module directory dracut cannot find' ], + [ '/build/builddeb-genesis-base: line 9: dch: command not found' => 'a command the build root lacks' ], + [ 'E: Unable to correct problems, you have held broken packages.' => 'a build root apt cannot resolve' ], + ) +{ + my ($line, $what) = @{$case}; + ok(scalar XCAT::BuildUtils::genesis_log_errors("before\n$line\nafter\n"), + "the log guard catches $what"); +} + +is_deeply([ XCAT::BuildUtils::genesis_log_errors(undef) ], [], 'no log is not an error'); + +# --- the builder refuses a root of another release ------------------------------------- +my $builder = repo_path('xCAT-genesis-builder/builddeb-genesis-base'); +if (!-f $builder) { + fail('xCAT-genesis-builder/builddeb-genesis-base is missing'); + done_testing(); + exit; +} + +my $tmp = tempdir(CLEANUP => 1); +mkdir "$tmp/bin"; +# dpkg is shadowed so the guard is tested on any host, and so the test cannot reach apt. +write_text("$tmp/bin/dpkg", "#!/bin/sh\necho amd64\n"); +chmod 0755, "$tmp/bin/dpkg"; + +sub build_in_a_root_of { + my ($codename, $expected) = @_; + write_text("$tmp/os-release", "ID=ubuntu\nVERSION_CODENAME=$codename\n"); + my $err = "$tmp/err"; + my $cmd = sprintf('PATH=%s:$PATH OS_RELEASE=%s /bin/bash %s --expect-codename %s >/dev/null 2>%s', + "'$tmp/bin'", "'$tmp/os-release'", "'$builder'", "'$expected'", "'$err'"); + system('/bin/bash', '-c', $cmd); + return ($? >> 8, -f $err ? read_text($err) : ''); +} + +my ($rc, $err) = build_in_a_root_of('jammy', 'noble'); +isnt($rc, 0, 'the builder refuses to build noble in a jammy root'); +like($err, qr/jammy/, 'the message names the root it woke up in'); +like($err, qr/noble/, 'the message names the release that was asked for'); + +done_testing(); diff --git a/xCAT-test/unit/genesis_payload_verification.t b/xCAT-test/unit/genesis_payload_verification.t index d1334b24b..9e4871b4c 100644 --- a/xCAT-test/unit/genesis_payload_verification.t +++ b/xCAT-test/unit/genesis_payload_verification.t @@ -1,186 +1,185 @@ #!/usr/bin/env perl -# Drive verify-genesis-payload against payload trees that each leave out one thing the image -# needs. +# XCAT::GenesisPayload decides whether an extracted Genesis payload is complete. +# verify-genesis-payload calls it in the EL spec and in the Ubuntu builder. Each payload +# below leaves out one thing the image needs. use strict; use warnings; +use File::Basename qw(dirname); use File::Path qw(make_path); -use File::Slurper qw(read_text write_text); +use File::Slurper qw(write_text); use File::Temp qw(tempdir); use FindBin; use lib "$FindBin::Bin/../lib"; +use lib "$FindBin::Bin/../../xCAT-genesis-builder/lib"; use Test::More; -use XCAT::Test::File qw(repo_path); +use XCAT::GenesisPayload qw(module_commands missing_paths check_payload); -my $verifier = repo_path('xCAT-genesis-builder/verify-genesis-payload'); -plan skip_all => 'verify-genesis-payload not found' unless -f $verifier; -plan tests => 22; - -my $tmpdir = tempdir(CLEANUP => 1); -my $module_seq = 0; +my $OPENSSH_99 = "OpenSSH_9.9p1\n/usr/libexec/openssh/sshd-session\n"; +my $OPENSSH_80 = "OpenSSH_8.0p1\n"; # A complete payload: OpenSSH 9.9 sshd plus its session helper, tmux plus a UTF-8 locale. -my $good = build_payload(sshd_execs_session => 1, session_helper => 1, tmux => 1, locale => 1, dhclient => 1, mktemp => 1); -my ($rc, $err) = run($good, 'usr/sbin/dhclient'); -is($rc, 0, 'a complete payload passes') or diag($err); +my @COMPLETE = qw( + usr/sbin/sshd usr/libexec/openssh/sshd-session usr/bin/tmux + usr/lib/locale/C.utf8/LC_CTYPE usr/sbin/dhclient usr/bin/mktemp + usr/bin/awk etc/services usr/bin/openssl usr/bin/wget usr/bin/tar +); + +# The payload carries exactly the paths it is given. +sub carries { + my %present = map { $_ => 1 } @_; + return sub { $present{ $_[0] } }; +} +sub all_but { my %gone = map { $_ => 1 } @_; return carries(grep { !$gone{$_} } @COMPLETE) } + +is_deeply([ missing_paths(carries(@COMPLETE), sshd => $OPENSSH_99, required => ['usr/sbin/dhclient']) ], + [], 'a complete payload passes'); # doxcat calls dhclient with ISC flags. dhclient.conf and dhclient-script are not enough. -my $nodhcp = build_payload(sshd_execs_session => 1, session_helper => 1, tmux => 1, locale => 1, dhclient => 0, mktemp => 1); -($rc, $err) = run($nodhcp, 'usr/sbin/dhclient'); -isnt($rc, 0, 'a payload without dhclient fails'); -like($err, qr{usr/sbin/dhclient}, 'the missing dhclient is named'); +is_deeply([ missing_paths(all_but('usr/sbin/dhclient'), sshd => $OPENSSH_99, required => ['usr/sbin/dhclient']) ], + ['usr/sbin/dhclient (required by the build)'], + 'a payload without dhclient fails and names it'); # sshd 9.9 execs /usr/libexec/openssh/sshd-session for every connection. -my $nohelper = build_payload(sshd_execs_session => 1, session_helper => 0, tmux => 1, locale => 1, dhclient => 1, mktemp => 1); -($rc, $err) = run($nohelper, 'usr/sbin/dhclient'); -isnt($rc, 0, 'a payload whose sshd execs sshd-session but does not ship it fails'); -like($err, qr{sshd-session}, 'the missing sshd-session is named'); - -# OpenSSH 8 does not use the helper, so el8 must still pass without it. -my $openssh8 = build_payload(sshd_execs_session => 0, session_helper => 0, tmux => 1, locale => 1, dhclient => 1, mktemp => 1); -($rc, $err) = run($openssh8, 'usr/sbin/dhclient'); -is($rc, 0, 'an OpenSSH 8 payload passes without sshd-session') or diag($err); +is_deeply([ missing_paths(all_but('usr/libexec/openssh/sshd-session'), sshd => $OPENSSH_99) ], + ['usr/libexec/openssh/sshd-session (this sshd execs it for every connection)'], + 'a payload whose sshd execs sshd-session but does not ship it fails'); # tmux without a UTF-8 locale is what stopped doxcat from ever running. -my $nolocale = build_payload(sshd_execs_session => 1, session_helper => 1, tmux => 1, locale => 0, dhclient => 1, mktemp => 1); -($rc, $err) = run($nolocale, 'usr/sbin/dhclient'); -isnt($rc, 0, 'a payload with tmux and no UTF-8 locale fails'); -like($err, qr{C\.utf8}, 'the missing locale is named'); +is_deeply([ missing_paths(all_but('usr/lib/locale/C.utf8/LC_CTYPE'), sshd => $OPENSSH_99) ], + ['usr/lib/locale/C.utf8/LC_CTYPE (tmux refuses to start without a UTF-8 locale)'], + 'a payload with tmux and no UTF-8 locale fails'); # getdestiny makes its request file with mktemp. -my $nomktemp = build_payload(sshd_execs_session => 1, session_helper => 1, tmux => 1, locale => 1, dhclient => 1, mktemp => 0); -($rc, $err) = run($nomktemp, 'usr/sbin/dhclient'); -isnt($rc, 0, 'a payload without mktemp fails'); -like($err, qr{usr/bin/mktemp}, 'the missing mktemp is named'); +is_deeply([ missing_paths(all_but('usr/bin/mktemp'), sshd => $OPENSSH_99) ], + ['usr/bin/mktemp (getdestiny makes its request file with it)'], + 'a payload without mktemp fails'); + +# Genesis is reached over ssh. +is_deeply([ missing_paths(all_but('usr/sbin/sshd')) ], + ['usr/sbin/sshd (Genesis is reached over ssh)'], + 'a payload without sshd fails'); + +is_deeply([ missing_paths(carries(grep({ $_ ne 'usr/libexec/openssh/sshd-session' } @COMPLETE), + 'usr/lib/openssh/sshd-session'), sshd => $OPENSSH_99) ], + [], 'the Debian path of sshd-session counts'); + +# OpenSSH 8 does not use the helper, so el8 must still pass without it. +is_deeply([ missing_paths(all_but('usr/libexec/openssh/sshd-session'), sshd => $OPENSSH_80) ], + [], 'an OpenSSH 8 payload passes without sshd-session'); + +is_deeply([ missing_paths(all_but('usr/lib/locale/C.utf8/LC_CTYPE', 'usr/bin/tmux'), sshd => $OPENSSH_99) ], + [], 'a payload without tmux needs no locale'); # dracut_install reports a missing binary and returns, so every name the dracut module -# installs has to be checked against the payload. -my $module = write_module_setup([qw(openssl wget tar)]); -my $full = build_payload(sshd_execs_session => 1, session_helper => 1, tmux => 1, locale => 1, - dhclient => 1, mktemp => 1, commands => [qw(openssl wget tar)]); -($rc, $err) = run_with_commands($module, $full); -is($rc, 0, 'a payload carrying every command the module names passes') or diag($err); +# installs has to be checked against the payload. A name starting with "/" is installed at +# that same path; the rest are commands. +my $tmpdir = tempdir(CLEANUP => 1); +my $module = "$tmpdir/module-setup.sh"; +write_text($module, <<'SH'); +#!/bin/bash -my $noopenssl = build_payload(sshd_execs_session => 1, session_helper => 1, tmux => 1, locale => 1, - dhclient => 1, mktemp => 1, commands => [qw(wget tar)]); -($rc, $err) = run_with_commands($module, $noopenssl); -isnt($rc, 0, 'a payload without openssl fails'); -like($err, qr/openssl/, 'the missing openssl is named'); +install() { + dracut_install -o openssl wget tar # a trailing comment + dracut_install /usr/bin/awk /etc/services + if command -v dhclient >/dev/null 2>&1; then + dracut_install dhclient + fi +} -# dracut_install installs an absolute path at that same path, so a name starting with "/" is a -# command the payload must carry. doxcat, getdestiny and the firmware wrappers all run awk. -my $noawk = build_payload(sshd_execs_session => 1, session_helper => 1, tmux => 1, locale => 1, - dhclient => 1, mktemp => 1, commands => [qw(openssl wget tar)], absent => ['usr/bin/awk']); -($rc, $err) = run_with_commands($module, $noawk); -isnt($rc, 0, 'a payload without the absolute path /usr/bin/awk fails'); -like($err, qr{/usr/bin/awk}, 'the missing /usr/bin/awk is named'); - -# The module names data files by absolute path too. Genesis resolves service names with -# /etc/services. -my $noservices = build_payload(sshd_execs_session => 1, session_helper => 1, tmux => 1, locale => 1, - dhclient => 1, mktemp => 1, commands => [qw(openssl wget tar)], absent => ['etc/services']); -($rc, $err) = run_with_commands($module, $noservices); -isnt($rc, 0, 'a payload without the absolute path /etc/services fails'); -like($err, qr{/etc/services}, 'the missing /etc/services is named'); +installkernel() { + dracut_install notacommand +} +SH +my @commands = module_commands($module); # The DHCP client is release-dependent, so the module installs it inside a conditional. Those # names are not the contract; the spec passes the one it wants as a required path. -my $conditional = write_module_setup(['wget'], ['dhclient']); -my $nodhclient = build_payload(sshd_execs_session => 1, session_helper => 1, tmux => 1, locale => 1, - dhclient => 0, mktemp => 1, commands => ['wget']); -($rc, $err) = run_with_commands($conditional, $nodhclient); -is($rc, 0, 'a name installed under a condition is not required') or diag($err); +is_deeply(\@commands, [qw(/etc/services /usr/bin/awk openssl tar wget)], + 'the top-level names of install() are read back, without options, comments, conditionals + or other functions'); -# A module the verifier cannot read names for covers nothing, so say so instead of passing. +my %names = (commands => \@commands, source => $module, sshd => $OPENSSH_99); +is_deeply([ missing_paths(carries(@COMPLETE), %names) ], [], + 'a payload carrying every command the module names passes'); +is_deeply([ missing_paths(carries(grep({ $_ ne 'usr/bin/wget' } @COMPLETE), 'sbin/wget'), %names) ], + [], 'a command under sbin counts as present'); + +is_deeply([ missing_paths(all_but('usr/bin/openssl'), %names) ], + ["openssl (installed by $module)"], + 'a payload without openssl fails and names it'); + +# doxcat, getdestiny and the firmware wrappers all run awk. +is_deeply([ missing_paths(all_but('usr/bin/awk'), %names) ], + ["/usr/bin/awk (installed by $module)"], + 'a payload without the absolute path /usr/bin/awk fails'); + +# Genesis resolves service names with /etc/services. +is_deeply([ missing_paths(all_but('etc/services'), %names) ], + ["/etc/services (installed by $module)"], + 'a payload without the absolute path /etc/services fails'); + +# A module the verifier cannot read names from covers nothing, so say so instead of passing. my $unparsable = "$tmpdir/module-setup-unparsable.sh"; write_text($unparsable, "#!/bin/bash\nsetup() {\n dracut_install wget\n}\n"); -($rc, $err) = run_with_commands($unparsable, $full); -is($rc, 2, 'a module the verifier finds no command names in is a usage error'); -like($err, qr/command name/, 'the empty command list is named'); +ok(!eval { module_commands($unparsable); 1 }, 'a module with no install() names is refused'); +is($@, "verify-genesis-payload: no command name read from $unparsable\n", + 'the empty command list is named'); -($rc, $err) = run_with_commands("$tmpdir/no-such-module", $full); -is($rc, 2, 'a module file that cannot be read is a usage error'); +ok(!eval { module_commands("$tmpdir/no-such-module"); 1 }, 'an unreadable module is refused'); +is($@, "verify-genesis-payload: cannot read $tmpdir/no-such-module\n", + 'the unreadable module is named'); -($rc, $err) = run("$tmpdir/does-not-exist"); -is($rc >> 0, 2, 'a missing payload directory is a usage error'); +# --- the command line reads a real payload tree --------------------------------------------- +my $good = payload_tree(@COMPLETE); +write_text("$good/usr/sbin/sshd", $OPENSSH_99); +my $nodhcp = payload_tree(grep { $_ ne 'usr/sbin/dhclient' } @COMPLETE); +my $nohelper = payload_tree(grep { $_ ne 'usr/libexec/openssh/sshd-session' } @COMPLETE); +write_text("$nohelper/usr/sbin/sshd", $OPENSSH_99); + +is_deeply([ check_payload($good, 'usr/sbin/dhclient') ], + [ 0, "verify-genesis-payload: $good is complete\n" ], + 'a complete payload exits 0 and says so'); +is_deeply([ check_payload($nodhcp, 'usr/sbin/dhclient') ], + [ 1, "verify-genesis-payload: $nodhcp is incomplete:\n" + . " usr/sbin/dhclient (required by the build)\n" ], + 'an incomplete payload exits 1 and lists what is missing'); +is_deeply([ check_payload($nohelper) ], + [ 1, "verify-genesis-payload: $nohelper is incomplete:\n" + . " usr/libexec/openssh/sshd-session (this sshd execs it for every connection)\n" ], + 'the command line reads usr/sbin/sshd to decide on sshd-session'); +is_deeply([ check_payload('--commands-from', $module, $nodhcp) ], + [ 0, "verify-genesis-payload: $nodhcp is complete\n" ], + '--commands-from does not require a name installed under a condition'); +my $noopenssl = payload_tree(grep { $_ ne 'usr/bin/openssl' } @COMPLETE); +is_deeply([ check_payload("--commands-from=$module", $noopenssl) ], + [ 1, "verify-genesis-payload: $noopenssl is incomplete:\n" + . " openssl (installed by $module)\n" ], + '--commands-from= checks the names of the module'); +is_deeply([ check_payload('--commands-from', $unparsable, $good) ], + [ 2, "verify-genesis-payload: no command name read from $unparsable\n" ], + 'a module with no command names is a usage error'); +is_deeply([ check_payload('--commands-from', "$tmpdir/no-such-module", $good) ], + [ 2, "verify-genesis-payload: cannot read $tmpdir/no-such-module\n" ], + 'a module file that cannot be read is a usage error'); +is_deeply([ check_payload("$tmpdir/does-not-exist") ], + [ 2, "verify-genesis-payload: not a payload directory: $tmpdir/does-not-exist\n" ], + 'a missing payload directory is a usage error'); +is_deeply([ check_payload() ], + [ 2, "verify-genesis-payload: not a payload directory: \n" ], + 'no payload directory at all is a usage error'); + +done_testing(); #--- -# build_payload: make a payload tree with the pieces the verifier reasons about. +# payload_tree: a payload directory that carries the given paths as empty files. #--- -sub build_payload { - my (%opt) = @_; +sub payload_tree { my $root = tempdir(DIR => $tmpdir, CLEANUP => 1); - make_path("$root/usr/sbin", "$root/usr/bin", "$root/usr/libexec/openssh"); - write_text("$root/usr/sbin/sshd", - $opt{sshd_execs_session} - ? "OpenSSH_9.9p1\n/usr/libexec/openssh/sshd-session\n" - : "OpenSSH_8.0p1\n"); - write_text("$root/usr/libexec/openssh/sshd-session", "helper\n") if $opt{session_helper}; - write_text("$root/usr/bin/tmux", "tmux\n") if $opt{tmux}; - if ($opt{locale}) { - make_path("$root/usr/lib/locale/C.utf8"); - write_text("$root/usr/lib/locale/C.utf8/LC_CTYPE", "ctype\n"); - } - write_text("$root/usr/sbin/dhclient", "dhclient\n") if $opt{dhclient}; - write_text("$root/usr/bin/mktemp", "mktemp\n") if $opt{mktemp}; - write_text("$root/usr/bin/$_", "$_\n") for @{ $opt{commands} || [] }; - - # The module written by write_module_setup names these two by absolute path. - my %absent = map { $_ => 1 } @{ $opt{absent} || [] }; - for my $path (qw(usr/bin/awk etc/services)) { - next if $absent{$path}; - my ($dir) = $path =~ m{^(.*)/}; - make_path("$root/$dir"); - write_text("$root/$path", "$path\n"); + for my $path (@_) { + make_path(dirname("$root/$path")); + write_text("$root/$path", ''); } return $root; } - -#--- -# run: run the verifier and return its exit status and stderr. -#--- -sub run { - my ($root, @required) = @_; - my $errfile = "$tmpdir/err.$$"; - my $cmd = join ' ', map { "'$_'" } ($verifier, $root, @required); - system("/bin/bash $cmd >/dev/null 2>$errfile"); - my $status = $? >> 8; - my $err = -f $errfile ? read_text($errfile) : ''; - unlink $errfile; - return ($status, $err); -} - -#--- -# write_module_setup: a dracut module whose install() names commands at the top level, and -# optionally more inside a conditional. -#--- -sub write_module_setup { - my ($top, $conditional) = @_; - my $path = "$tmpdir/module-setup." . ++$module_seq . ".sh"; - my $text = "#!/bin/bash\n\ninstall() {\n"; - $text .= " dracut_install " . join(' ', @$top) . " # a trailing comment\n"; - $text .= " dracut_install /usr/bin/awk /etc/services\n"; - if ($conditional) { - $text .= " if command -v " . $conditional->[0] . " >/dev/null 2>&1; then\n"; - $text .= " dracut_install " . join(' ', @$conditional) . "\n"; - $text .= " fi\n"; - } - $text .= "}\n"; - write_text($path, $text); - return $path; -} - -#--- -# run_with_commands: run the verifier with the command list read back from a dracut module. -#--- -sub run_with_commands { - my ($module, $root) = @_; - my $errfile = "$tmpdir/err.commands.$$"; - my $cmd = join ' ', map { "'$_'" } ($verifier, '--commands-from', $module, $root); - system("/bin/bash $cmd >/dev/null 2>$errfile"); - my $status = $? >> 8; - my $err = -f $errfile ? read_text($errfile) : ''; - unlink $errfile; - return ($status, $err); -} diff --git a/xCAT-test/unit/genesis_ubuntu_build_root.t b/xCAT-test/unit/genesis_ubuntu_build_root.t index fb598ccd9..ce314c925 100755 --- a/xCAT-test/unit/genesis_ubuntu_build_root.t +++ b/xCAT-test/unit/genesis_ubuntu_build_root.t @@ -3,113 +3,89 @@ # mandatory. dracut_install reports a missing command and returns 0, so a hole in the image # does not fail the build. # -# The mandatory list comes from RUNNING the module: module-setup.sh is sourced with -# dracut_install shadowed, _dracut_install_opt neutralised, and install() called. The -# package list comes from evaluating the REQUIRED_PACKAGES assignment in the build script. +# XCAT::GenesisBuildRoot::required_packages lists the packages of the build root, and +# XCAT::GenesisPayload::module_commands reads the mandatory commands from the module, as +# verify-genesis-payload does. The refusal of a root of another release is in +# genesis_deb_per_codename.t. use strict; use warnings; +use File::Slurper qw(read_text); use File::Temp qw(tempdir); use FindBin; use lib "$FindBin::Bin/../lib"; +use lib "$FindBin::Bin/../../xCAT-genesis-builder/lib"; use Test::More; +use XCAT::GenesisBuildRoot qw(required_packages); +use XCAT::GenesisPayload qw(module_commands); use XCAT::Test::File qw(repo_path); -my $builder = repo_path('xCAT-genesis-builder/builddeb-genesis-base'); -my $module = repo_path('xCAT-genesis-builder/dracut_105/ubuntu/module-setup.sh'); -plan skip_all => 'builddeb-genesis-base not found' unless -f $builder; -plan skip_all => 'ubuntu module-setup.sh not found' unless -f $module; -plan tests => 9; +my $builder = repo_path('xCAT-genesis-builder/builddeb-genesis-base'); +my $module = repo_path('xCAT-genesis-builder/dracut_105/ubuntu/module-setup.sh'); -# Mandatory commands a minimal Ubuntu server root does NOT already provide, and the package -# that supplies each one on every release xCAT builds for. -my %PACKAGE_FOR = ( - dhclient => 'isc-dhcp-client', - ifenslave => 'ifenslave', +# Mandatory commands a minimal Ubuntu server root does NOT already provide, and the packages +# that supply each one. hwclock has two names: it left util-linux for util-linux-extra in +# 23.04. +my %PACKAGES_FOR = ( + dhclient => ['isc-dhcp-client'], + ifenslave => ['ifenslave'], + hwclock => [ 'util-linux-extra', 'util-linux' ], ); -# hwclock is not in that list because the package that carries it moved. Measured on the -# four Ubuntu management nodes: focal and jammy have it in util-linux, which is essential -# and always in the build root, and no util-linux-extra exists to install; noble and -# resolute have it in util-linux-extra. util-linux only Suggests that package, and this -# build passes --no-install-recommends, so the releases that split it must name it and the -# releases that did not must not. - -my %mandatory = map { $_ => 1 } mandatory_commands($module); -my @packages = required_packages($builder); - -for my $command (sort keys %PACKAGE_FOR) { - ok($mandatory{$command}, "the Ubuntu dracut module installs '$command' unconditionally"); - ok(scalar(grep { $_ eq $PACKAGE_FOR{$command} } @packages), - "the build root installs $PACKAGE_FOR{$command}, which provides '$command'"); +# A release carries exactly the names in its list. +sub release { + my %carried = map { $_ => 1 } @_; + return sub { $carried{ $_[0] } }; } +my $NOBLE = release(qw(bind9-dnsutils dnsutils util-linux-extra util-linux tzdata-legacy)); +my $FOCAL = release(qw(dnsutils util-linux)); + +my @noble = required_packages('amd64', 'noble', $NOBLE); +is_deeply([ @noble[ -5 .. -1 ] ], + [qw(dmidecode efibootmgr bind9-dnsutils util-linux-extra tzdata-legacy)], + 'amd64 adds dmidecode and efibootmgr, then the first name the release carries'); + +my @focal = required_packages('amd64', 'focal', $FOCAL); +is_deeply([ @focal[ -2 .. -1 ] ], [qw(dnsutils util-linux)], + 'a release without the new names gets dnsutils and util-linux, and no tzdata-legacy'); + +my @ppc = required_packages('ppc64el', 'noble', $NOBLE); +is_deeply([ @ppc[ -3 .. -1 ] ], [qw(bind9-dnsutils util-linux-extra tzdata-legacy)], + 'ppc64el gets neither dmidecode nor efibootmgr'); +is_deeply([ @ppc[ 0 .. $#ppc - 3 ] ], [ @noble[ 0 .. $#noble - 5 ] ], + 'ppc64el and amd64 share the base packages'); + +my @asked; +required_packages('amd64', 'noble', sub { push @asked, $_[0]; $NOBLE->($_[0]) }); +is_deeply(\@asked, [qw(bind9-dnsutils util-linux-extra tzdata-legacy)], + 'apt is asked for the older name only when the newer one is absent'); + +ok(!eval { required_packages('amd64', 'oddball', release('bind9-dnsutils')); 1 }, + 'a release that carries no hwclock package fails'); +is($@, "ERROR: oddball carries none of these packages: util-linux-extra util-linux\n", + 'the failure names the release and the missing alternatives'); + +# --- every mandatory command has a package in the build root ----------------------------- +# An absolute path is a data file, not a command. +my %mandatory = map { $_ => 1 } grep { !m{^/} } module_commands($module); +my %packages = map { $_ => 1 } @noble; +for my $command (sort keys %PACKAGES_FOR) { + my @provider = @{ $PACKAGES_FOR{$command} }; + ok($mandatory{$command}, "the Ubuntu dracut module installs '$command' unconditionally"); + ok(scalar(grep { $packages{$_} } @provider), + "the build root installs @{[ join ' or ', @provider ]}, which provides '$command'"); +} +ok(grep({ $_ eq 'util-linux' } @focal), 'a release before 23.04 gets hwclock from util-linux'); # doxcat asks dhclient for the provisioning lease. -ok($mandatory{dhclient} && scalar(grep { $_ eq 'isc-dhcp-client' } @packages), +ok($mandatory{dhclient} && $packages{'isc-dhcp-client'}, 'the Genesis image can obtain a DHCP lease'); -ok($mandatory{hwclock}, "the Ubuntu dracut module installs 'hwclock' unconditionally"); +# The build cannot run here, so the call to the payload gate is read from the script. The +# gate itself is exercised by genesis_payload_verification.t. +like(read_text($builder), + qr{^bash "\$DIR/verify-genesis-payload" --commands-from "\$DRACUTMODDIR/module-setup\.sh" "\$GENESIS_FS"}m, + 'builddeb-genesis-base verifies the payload it packages against the module'); -# Naming a package apt cannot locate fails the whole install, and the script runs under -# set -e, so an unconditional util-linux-extra stops the build on focal and jammy. -ok(!scalar(grep { $_ eq 'util-linux-extra' } @packages), - 'the unconditional list does not name util-linux-extra'); - -# What the script does instead: keep a package only where apt has a candidate for it. -{ - is_deeply(optional_packages($builder, 'util-linux-extra', 0), ['util-linux-extra'], - 'a release that carries util-linux-extra installs it'); - is_deeply(optional_packages($builder, 'util-linux-extra', 1), [], - 'a release without it installs nothing in its place'); -} - -# An absolute path in the install() output is a data file, not a command. -sub mandatory_commands { - my ($path) = @_; - my $dir = tempdir(CLEANUP => 1); - my $driver = "$dir/collect.sh"; - open my $fh, '>', $driver or die "$driver: $!"; - print $fh <<"BASH"; -dracut_install() { printf '%s\\n' "\$\@"; } -instmods() { :; } -inst_multiple() { :; } -inst() { :; } -dpkg-architecture() { echo x86_64-linux-gnu; } -. '$path' -# _dracut_install_opt installs only what the build root already has. Neutralise it after -# sourcing, so its commands stay out of the mandatory set. -_dracut_install_opt() { :; } -install -BASH - close $fh; - my @out = qx{bash '$driver' 2>/dev/null}; - die("running install() from $path produced nothing") unless @out; - my %seen; - my @names = grep { !$seen{$_}++ } grep { length && !m{^/} } map { chomp; $_ } @out; - die("install() from $path named no bare commands") unless @names; - return @names; -} - -# Run the script's own selector with apt-cache shadowed, so the decision is exercised -# rather than read. $rc is what the shadow returns: 0 for a release that has the package. -sub optional_packages { - my ($path, $package, $rc) = @_; - my $text = do { open my $fh, '<', $path or die "$path: $!"; local $/; <$fh> }; - my ($block) = $text =~ /^(optional_packages\(\)\s*\{.*?^\})/ms; - BAIL_OUT("no optional_packages() in $path") unless $block; - my $out = qx{bash -c 'set -u; apt-cache() { return $rc; }; $block; optional_packages $package' 2>/dev/null}; - return [ grep { length } split /\s+/, ($out // '') ]; -} - -# Evaluate the assignment rather than parse it, so the list is the value the script uses. -sub required_packages { - my ($path) = @_; - my $text = do { open my $fh, '<', $path or die "$path: $!"; local $/; <$fh> }; - my ($block) = $text =~ /^(REQUIRED_PACKAGES="[^"]*")/ms; - die("no REQUIRED_PACKAGES assignment in $path") unless $block; - my $out = qx{bash -c 'set -u; $block; printf "%s\\n" \$REQUIRED_PACKAGES' 2>/dev/null}; - my @packages = grep { length } split /\s+/, ($out // ''); - die("REQUIRED_PACKAGES in $path evaluated to nothing") unless @packages; - return @packages; -} +done_testing();