diff --git a/BuildUtils.pm b/BuildUtils.pm index c99005f..3753608 100644 --- a/BuildUtils.pm +++ b/BuildUtils.pm @@ -37,7 +37,7 @@ our @EXPORT_OK = qw( supported_arches is_supported_arch chroot_name chroot_sources_list chroot_is_disposable chroot_build_script chroot_build_timeout - control_field genesis_deb_control + control_field genesis_debs_for_codename deb_field deb_version deb_hash cross_copy_genesis_deb build_deb_in_chroot ); @@ -476,53 +476,6 @@ sub control_field { return undef; } -# genesis_deb_control: build the DEBIAN/control text for the cross-arch-converted xcat-genesis-base -# deb, PRESERVING the maintained packaging's semantics (Depends/Breaks/Replaces/Section/Priority) -# instead of hand-rolling a bare 5-field control (the bug in build-dep-debs.sh flagged by review -# concern #2). $maintained is the text of xCAT-genesis-builder/debian/control (or undef when it -# cannot be located — then a minimal-but-honest control is produced and the caller should warn). -# $pkgname is e.g. xcat-genesis-base-ppc64el, $version the deb version, $arch 'all'. Pure/testable. -sub genesis_deb_control { - my ($maintained, $pkgname, $version, $arch) = @_; - $arch ||= 'all'; - my %f = ( - Package => $pkgname, - Version => $version, - Architecture => $arch, - Section => 'admin', - Priority => 'optional', - Maintainer => 'xCAT ', - ); - if (defined $maintained && $maintained ne '') { - for my $k (qw(Section Priority Maintainer Depends Pre-Depends Recommends - Suggests Breaks Replaces Conflicts Provides)) { - my $v = control_field($maintained, $k); - $f{$k} = $v if defined $v && $v ne ''; - } - my $desc = control_field($maintained, 'Description'); - $f{Description} = $desc if defined $desc && $desc ne ''; - } - $f{Description} ||= 'xCAT Genesis netboot image (converted from the rpm for cross-arch netboot)'; - # ${misc:Depends} is a debhelper substitution var that only resolves during a real dpkg build; - # in a hand-assembled control it would ship literally, so drop it from a preserved Depends. - for my $k (qw(Depends Pre-Depends Recommends Suggests)) { - next unless defined $f{$k}; - $f{$k} =~ s/\$\{[^}]+\}//g; - $f{$k} =~ s/^[,\s]+|[,\s]+$//g; - $f{$k} =~ s/\s*,\s*,\s*/, /g; - delete $f{$k} if $f{$k} eq ''; - } - my @order = qw(Package Version Section Priority Architecture Maintainer - Pre-Depends Depends Recommends Suggests Breaks Replaces Conflicts - Provides Description); - my $out = ''; - for my $k (@order) { - next unless defined $f{$k} && $f{$k} ne ''; - $out .= "$k: $f{$k}\n"; - } - return $out; -} - # --------------------------------------------------------------------------------------------------- # Built-.deb inspection + cross-arch genesis provisioning (filesystem; tested with real dpkg-deb). # --------------------------------------------------------------------------------------------------- @@ -596,9 +549,30 @@ sub deb_hash { # Idempotent; content is compared by deb_hash so a stale same-name deb is refreshed rather than # mistaken for up to date. $sign is an optional coderef($deb_path) invoked on each copied deb; pass # undef to skip. Mirrors MockBuildUtils::cross_copy_genesis for the apt world. +# $codename, when given, narrows the set to the image built for that release -- see +# genesis_debs_for_codename. +# genesis_debs_for_codename: the Genesis debs that belong to ONE Ubuntu release. +# +# The Genesis image carries the kernel of the root that built it, so xcat-core builds one deb per +# codename and stamps the codename into the version (2.19.0-snap...~noble). Staging all of them into +# every suite publishes three images per suite, and apt serves the newest -- the image of another +# release. A deb with no codename in its version predates the native build and serves every release. +sub genesis_debs_for_codename { + my ($debs, $codename) = @_; + my @debs = @{ $debs || [] }; + return @debs unless @debs && defined $codename && $codename ne ''; + my $marked = qr/_[^_]*~[A-Za-z0-9.]+_[^_]*\.deb\z/; + return @debs unless grep { basename($_) =~ $marked } @debs; + return grep { + my $base = basename($_); + $base =~ /_[^_]*~\Q$codename\E_[^_]*\.deb\z/ || $base !~ $marked; + } @debs; +} + sub cross_copy_genesis_deb { - my ($from, $to, $arch, $sign) = @_; - my @src = glob("$from/xcat-genesis-base-$arch\_*.deb"); + my ($from, $to, $arch, $sign, $codename) = @_; + my @src = genesis_debs_for_codename( + [ glob("$from/xcat-genesis-base-$arch\_*.deb") ], $codename); return 0 if !@src; my %want = map { basename($_) => $_ } @src; my @existing = glob("$to/xcat-genesis-base-$arch\_*.deb"); diff --git a/sbuild-all.pl b/sbuild-all.pl index 80044e3..b3bc92d 100755 --- a/sbuild-all.pl +++ b/sbuild-all.pl @@ -5,7 +5,7 @@ # * mk-dep-chroots.sh -> the "ensure chroots" phase (auto-initializes per-codename sbuild chroots # on first run; idempotent). # * build-dep-debs.sh -> the per-package build phase (drives each /sbuild.pl in the matching -# chroot) + the metadata-preserving genesis phase. +# chroot) + the genesis-ingest phase. # * build-apt-repo.sh -> the apt-repo assembly + signing phase (in Perl, focal supported). # # Design (mirrors mockbuild-all.pl + fixes the PR #63 review): @@ -22,9 +22,9 @@ # arch-specific compiled deps (concern #3). # 4. Any required chroot / package / artifact failure, or any version-pin mismatch, fails the whole # run non-zero (concern #4). -# 5. The genesis-base deb keeps the maintained Debian packaging semantics -- a native deb is ingested -# as-is when provided; a converted rpm keeps the maintained control (Depends/Breaks/Replaces) and -# maintainer scripts (concern #2). +# 5. The genesis-base debs are built by xcat-core, one per Ubuntu codename, and are ingested here +# as they are. Each is staged into the suite it was built for: the image carries the kernel of +# the root that built it, so one image cannot serve several releases. use strict; use warnings; use Cwd qw(abs_path); @@ -53,12 +53,11 @@ use BuildUtils qw(sh_quote print_step version_matches required_pkgs read_manifes codename_to_version known_codenames supported_arches is_supported_arch chroot_name chroot_sources_list chroot_is_disposable - control_field genesis_deb_control + control_field deb_field deb_version deb_hash cross_copy_genesis_deb); my $script_dir = abs_path(dirname(__FILE__)); my $repo_root = $script_dir; -my $xcat_src = "$repo_root/../xcat-core"; my $output_root = ''; my $apt_dir = ''; my $manifest = ''; @@ -108,10 +107,7 @@ my $genesis_release_checksums; # its verified SHA256SUMS, read once at sta # They are Architecture:all and identical for all suites, so a per-suite copy would multiply hundreds # of megabytes by the number of codenames for no gain. my $GENESIS_POOL_RELATIVE = 'pool/main/xcat-genesis-openembedded'; -my @genesis_debs; # native xcat-genesis-base- deb(s): path or URL (preferred) -my $genesis_rpm = ''; # fallback: native-arch genesis rpm to convert -my $genesis_rpm_ppc = ''; # fallback: cross-arch ppc genesis rpm to convert (amd64 host) -my $require_ppc_genesis = 0; +my @genesis_debs; # native xcat-genesis-base- deb(s): path or URL # File-scoped exclusive run-lock handle. MUST be file-scoped (not a lexical inside a block) so the # flock lives for the WHOLE process -- a lexical would close the FH and release the lock early. # Seconds to wait for a concurrent publisher before giving up (--publish-lock-wait). Long by default: @@ -136,7 +132,6 @@ my %PKG_DIR = ( # mockbuild-all.pl), plus the apt/sbuild-specific options this orchestrator adds. my %DEST = ( 'repo-root' => \$repo_root, - 'xcat-source' => \$xcat_src, 'output' => \$output_root, # alias of --output-root 'output-root' => \$output_root, 'manifest' => \$manifest, @@ -161,9 +156,6 @@ my %DEST = ( 'gpg-key-id' => \$gpg_key_id, 'genesis-release' => \$genesis_release, 'genesis-deb' => \@genesis_debs, - 'genesis-rpm' => \$genesis_rpm, - 'genesis-rpm-ppc' => \$genesis_rpm_ppc, - 'require-ppc-genesis' => \$require_ppc_genesis, ); my %spec; # option-spec-string => destination ref for my $s (standard_options()) { @@ -180,9 +172,6 @@ $spec{'gpg-key-id=s'} = \$gpg_key_id; $spec{'parallel-targets=i'} = \$parallel_targets; $spec{'build-timeout=i'} = \$build_timeout; # per-package wall-clock bound (0 = unbounded) $spec{'genesis-deb=s'} = \@genesis_debs; -$spec{'genesis-rpm=s'} = \$genesis_rpm; -$spec{'genesis-rpm-ppc=s'} = \$genesis_rpm_ppc; -$spec{'require-ppc-genesis!'} = \$require_ppc_genesis; $spec{'install-deps!'} = \$install_deps; # make this host able to run at all, then exit $spec{'publish!'} = \$publish; # run the finalization (assemble+sign+gate+tarball) $spec{'publish-lock-wait=i'} = \$PUBLISH_LOCK_WAIT; # seconds to queue behind another publisher @@ -221,7 +210,6 @@ if ($install_deps) { # Configuration # --------------------------------------------------------------------------------------------------- $repo_root = abs_path($repo_root); -$xcat_src = abs_path($xcat_src) if -d $xcat_src; $manifest ||= "$repo_root/debs-manifest.conf"; $arch ||= `dpkg --print-architecture 2>/dev/null`; chomp $arch; $arch ||= 'amd64'; @@ -369,7 +357,6 @@ $ENV{XCAT_DEP_BUILD_TIMEOUT} = $build_timeout if defined $build_timeout; print_step('Configuration'); print " repo-root: $repo_root\n"; -print " xcat-source: $xcat_src\n"; print " arch: $arch\n"; print " dists: @dist_list\n"; print " manifest: $manifest\n"; @@ -677,49 +664,13 @@ sub build_deps { } # --------------------------------------------------------------------------------------------------- -# Phase: genesis-base deb (concern #2: preserve maintained packaging; native ingest preferred) +# Phase: genesis-base deb (built natively by xcat-core, one image per Ubuntu codename) +# +# The Genesis image carries the kernel of the root that built it. xcat-core's builddebs.pl --genesis +# builds one deb per codename inside that codename's chroot; this phase only ingests them and stages +# each one into the suite it belongs to. The rpm->deb conversion that came before it gave every +# Ubuntu release the EL kernel, so it is gone. # --------------------------------------------------------------------------------------------------- -# maintained_genesis_control($arch): the maintained xCAT-genesis-builder/debian/control text, with the -# arch-specific package/relationship names remapped to $arch (the tree carries the amd64 control). -sub maintained_genesis_control { - my ($a) = @_; - my $f = "$xcat_src/xCAT-genesis-builder/debian/control"; - return undef unless -f $f; - local $/; open my $fh, '<', $f or return undef; my $t = <$fh>; close $fh; - # The tree carries the amd64 control; any other arch is the same text with the arch renamed. - $t =~ s/amd64/$a/g if $a ne 'amd64'; - return $t; -} -# convert_genesis_rpm($rpm, $pkgname, $arch, $outdir): rpm2cpio-extract the noarch genesis rpm and -# repackage as a .deb whose DEBIAN/control PRESERVES the maintained Depends/Breaks/Replaces and whose -# maintainer scripts (postinst/prerm/preinst/postrm) are copied from the maintained debian/ -- so the -# converted deb keeps the install/upgrade semantics the bare 5-field shim dropped (concern #2). -sub convert_genesis_rpm { - my ($rpm, $pkgname, $a, $outdir) = @_; - my $work = tempdir(CLEANUP => 1); - my $get = ($rpm =~ m{^https?://}) - ? "curl -fsSL " . sh_quote($rpm) . " | rpm2cpio" - : "rpm2cpio " . sh_quote($rpm); - run("cd $work && $get | cpio -idm --quiet"); - my $ver = `rpm -qp --qf '%{VERSION}-%{RELEASE}' ${\ sh_quote($rpm)} 2>/dev/null`; chomp $ver; - $ver ||= "2.18.0-snap$snap_ts"; - $ver =~ s/\.(el|fc)\d+.*$//; # drop the EL dist tag from the rpm Release - my $pkgd = "$work/pkg"; make_path("$pkgd/DEBIAN", "$pkgd/opt/xcat"); - run("cp -a $work/opt/xcat/. $pkgd/opt/xcat/ 2>/dev/null || true", nofail => 1); - my $control = genesis_deb_control(maintained_genesis_control($a), $pkgname, $ver, 'all'); - if (!$dry_run) { - open my $fh, '>', "$pkgd/DEBIAN/control" or die "write control: $!\n"; print $fh $control; close $fh; - # preserve maintainer scripts from the maintained packaging (install/upgrade behavior) - my $mdeb = "$xcat_src/xCAT-genesis-builder/debian"; - for my $s (qw(postinst preinst postrm prerm)) { - next unless -f "$mdeb/$s"; - copy("$mdeb/$s", "$pkgd/DEBIAN/$s"); chmod 0755, "$pkgd/DEBIAN/$s"; - } - } - make_path($outdir); - run("dpkg-deb --build " . sh_quote($pkgd) . " " . sh_quote("$outdir/${pkgname}_${ver}_all.deb")); - return "$outdir/${pkgname}_${ver}_all.deb"; -} # genesis_in_manifest(): whether the legacy Genesis deb belongs to this run at all. It is named # per target in the manifest, and riscv64 does not name it: its Genesis is the OpenEmbedded package # published once into the shared pool. Without this, a plain --arch riscv64 run reaches @@ -747,35 +698,12 @@ sub build_genesis { print " ingested native genesis deb: $base\n"; $produced_native = 1 if $base =~ /^\Q$native_arch_pkg\E_/; } - # 2) else convert the native-arch rpm (metadata-preserving) - if (!$produced_native) { - if ($genesis_rpm) { - print " converting native-arch genesis rpm -> deb (preserving control + scripts)\n"; - convert_genesis_rpm($genesis_rpm, $native_arch_pkg, $arch, $gen); - } elsif (!@genesis_debs) { - die "FATAL: no native genesis for $arch: pass --genesis-deb (preferred) or --genesis-rpm\n"; - } - } - # 3) cross-arch ppc genesis on the amd64 host (#7610): convert the ppc rpm if given - if ($arch eq 'amd64') { - my $have_ppc = grep { basename($_) =~ /^xcat-genesis-base-ppc64el_/ } glob("$gen/*.deb"); - if (!$have_ppc && $genesis_rpm_ppc) { - print " converting cross-arch ppc64el genesis rpm -> deb (#7610)\n"; - convert_genesis_rpm($genesis_rpm_ppc, 'xcat-genesis-base-ppc64el', 'ppc64el', $gen); - $have_ppc = 1; - } - if (!$have_ppc) { - my $msg = "no ppc64el genesis (pass --genesis-deb/--genesis-rpm-ppc): an amd64 MN cannot " - . "netboot ppc nodes (#7610)"; - die "FATAL: $msg\n" if $require_ppc_genesis; - warn "WARN: $msg\n"; - } - } - # stage the arch:all genesis deb(s) into every codename (this host's arch subdir; the cross-arch - # ppc genesis produced on the amd64 host rides in the amd64 subdir and is picked up by assemble). - # Use BuildUtils::cross_copy_genesis_deb -- the tested, hash-based, stale-dropping copier -- once - # per genesis package-arch present in $gen (the native-arch one, plus the cross-converted ppc64el - # one on the amd64 host). It refreshes a stale same-name deb by content and is idempotent. + die "FATAL: no Genesis deb for $arch: pass --genesis-deb.\n" + . " xcat-core builds them with `builddebs.pl --genesis-only --genesis-dist `.\n" + unless $produced_native || $dry_run; + # Stage each suite's OWN image. cross_copy_genesis_deb is the tested, hash-based, stale-dropping + # copier; the codename narrows it to the deb built on that release. A deb with no codename in its + # version serves every suite, which is how a package published before the native build is reused. my %gen_arches; for my $d (glob("$gen/*.deb")) { $gen_arches{$1}++ if basename($d) =~ /^xcat-genesis-base-([a-z0-9]+)_/; @@ -788,7 +716,10 @@ sub build_genesis { } make_path($dst); for my $ga (sort keys %gen_arches) { - my $n = cross_copy_genesis_deb($gen, $dst, $ga, undef); + my $n = cross_copy_genesis_deb($gen, $dst, $ga, undef, $cn); + die "FATAL: no xcat-genesis-base-$ga image built for $cn: xcat-core builds one per\n" + . " codename, so --genesis-dist must name every release this run publishes.\n" + unless $n || glob("$dst/xcat-genesis-base-$ga\_*.deb"); print " staged xcat-genesis-base-$ga -> $cn/$arch ($n newly copied)\n"; } } @@ -1391,7 +1322,10 @@ sbuild-all.pl - build, validate, sign and assemble the xcat-dep Ubuntu/Debian ap # STEP 1 -- per arch, on that arch's build host: build + validate into staging (does NOT publish): sbuild-all.pl --arch amd64 --dists "focal jammy noble resolute" \ - --xcat-source ../xcat-core --genesis-rpm + --genesis-deb \ + --genesis-deb \ + --genesis-deb \ + --genesis-deb sbuild-all.pl --arch ppc64el --dists "focal jammy noble resolute" --skip-genesis sbuild-all.pl --arch riscv64 --dists "focal jammy noble resolute" @@ -1404,7 +1338,7 @@ sbuild-all.pl - build, validate, sign and assemble the xcat-dep Ubuntu/Debian ap sbuild-all.pl --target noble-amd64 ... # equivalent single-target form # single host, build AND publish in one go (add --publish explicitly): - sbuild-all.pl --arch amd64 --dists noble --genesis-rpm --publish --expect-arch amd64 \ + sbuild-all.pl --arch amd64 --dists noble --genesis-deb --publish --expect-arch amd64 \ --gpg-sign --gpg-key-id --gpg-home # verify an already-published tree out of band (signatures checked by DEFAULT): @@ -1472,10 +1406,11 @@ failure (see L). =item Genesis -Produces the C deb: a native deb is ingested as-is when provided -(C<--genesis-deb>); otherwise the rpm is converted while B -(Depends/Breaks/Replaces) and maintainer scripts. The amd64 host also converts the cross-arch -ppc64el genesis (issue #7610) unless C<--require-ppc-genesis> gates it. Skipped with C<--skip-genesis>. +Ingests the C debs named by C<--genesis-deb> and stages each one into the +suite it was built for. The Genesis image carries the kernel of the root that built it, so +xcat-core builds one deb per Ubuntu codename (CcodenameE>) and stamps the codename into the version. A run that publishes a codename +with no image for it stops. Skipped with C<--skip-genesis>. An B is a separate, verified input published by C<--genesis-release>; it is not built here. @@ -1542,9 +1477,9 @@ Build a single target; the arch must match C<--arch>. Per-target manifest. Default: C<< /debs-manifest.conf >>. -=item B<--repo-root> / B<--xcat-source> C +=item B<--repo-root> C -xcat-dep root (default: the script's dir) / xcat-core root (for the maintained genesis packaging). +xcat-dep root (default: the script's dir). =item B<--output-root> / B<--apt-dir> C @@ -1558,15 +1493,8 @@ build hosts). =item B<--genesis-deb> C -Native C deb to ingest (repeatable; preferred over conversion). - -=item B<--genesis-rpm> / B<--genesis-rpm-ppc> C - -Native-arch genesis rpm to convert / cross-arch ppc genesis rpm to convert on amd64 (issue #7610). - -=item B<--require-ppc-genesis> - -Make a missing ppc64el genesis fatal (default: warn). +An C deb that xcat-core built. Repeatable: pass one per codename, and one per +architecture on a host that stages another architecture's image. =item B<--genesis-release> C diff --git a/t/sbuild-all.t b/t/sbuild-all.t index acbc304..c5e4558 100644 --- a/t/sbuild-all.t +++ b/t/sbuild-all.t @@ -20,7 +20,7 @@ use BuildUtils qw(install_deps_packages install_deps_command missing_perl_module supported_arches is_supported_arch codename_to_version version_to_codename known_codenames chroot_name chroot_sources_list chroot_is_disposable chroot_build_script - control_field genesis_deb_control + control_field deb_field deb_version deb_hash cross_copy_genesis_deb build_deb_in_chroot); @@ -88,41 +88,6 @@ is(chroot_name('noble', 'amd64'), 'noble-amd64-sbuild', 'chroot_name shape'); is(control_field($ctrl, 'Replaces'), undef, 'absent field -> undef'); } -# ---- genesis_deb_control: PRESERVE the maintained packaging semantics (concern #2) -------------- -{ - # The real xCAT-genesis-builder/debian/control fields that the bare 5-field shim used to drop. - my $maintained = <<'CTRL'; -Source: xcat-genesis-base-amd64 -Section: admin -Priority: optional -Maintainer: xCAT - -Package: xcat-genesis-base-amd64 -Architecture: all -Depends: ${misc:Depends} -Replaces: xcat-genesis-amd64 -Breaks: xcat-genesis-amd64, xcat-genesis-scripts-amd64 (<< 2.13.10) -Description: xCAT Genesis netboot image - base platform. -CTRL - my $c = genesis_deb_control($maintained, 'xcat-genesis-base-amd64', '2.18.0-snap1', 'all'); - like($c, qr/^Package: xcat-genesis-base-amd64$/m, 'Package set'); - like($c, qr/^Version: 2\.18\.0-snap1$/m, 'Version set'); - like($c, qr/^Architecture: all$/m, 'Architecture set'); - like($c, qr/^Replaces: xcat-genesis-amd64$/m, 'Replaces PRESERVED (was dropped by the shim)'); - like($c, qr/^Breaks: xcat-genesis-amd64, xcat-genesis-scripts-amd64 \(<< 2\.13\.10\)$/m, - 'Breaks PRESERVED with its version constraint'); - unlike($c, qr/\$\{misc:Depends\}/, 'unresolved ${misc:Depends} substvar dropped (would ship literal)'); - like($c, qr/^Maintainer: xCAT /m, 'Maintainer preserved'); -} -# With no maintained control available, an honest minimal control is still produced. -{ - my $c = genesis_deb_control(undef, 'xcat-genesis-base-ppc64el', '2.18.0-snap1', 'all'); - like($c, qr/^Package: xcat-genesis-base-ppc64el$/m, 'minimal control still names the package'); - like($c, qr/^Architecture: all$/m, 'minimal control still arch:all'); - unlike($c, qr/^Replaces:/m, 'no Replaces invented when the maintained control is absent'); -} - # ---- verify_repo_packages: PURE completeness decision (no I/O; manifest = source of truth) ------- { my %req = ('ipmitool-xcat' => '1.8.18', 'goconserver' => '0.3.3', 'xcat-genesis-base' => '*');