From f96ec6b8d5bc4644ec3e7d2de9fe30b037f804e6 Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Thu, 16 Jul 2026 15:00:23 -0300 Subject: [PATCH 01/55] feat(mockbuild-all): add --build-number CD version bump Append .snap. to every xcat-dep package spec Release so each CD run publishes a fresh, monotonic NVR (deploy's additive rsync is a no-op on an unchanged NVR). Applied before any child builder; genesis-base lives under xcat-core and is untouched, keeping it in lockstep with the deployed core's genesis-scripts. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- mockbuild-all.pl | 51 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/mockbuild-all.pl b/mockbuild-all.pl index f1be0b0..7702970 100755 --- a/mockbuild-all.pl +++ b/mockbuild-all.pl @@ -28,6 +28,11 @@ my $parallel_targets = 1; # 1 = serial (default; safe). 0/auto = all EL target my $max_parallel = 0; # 0/auto = host nproc: global cap on concurrent mock builds (all targets) my $run_id = ''; my $build_timestamp; +# CD version bump: when set, every xcat-dep package spec's Release gets a +# ".snap." suffix so each pipeline run publishes a +# fresh, monotonic NVR (deploy's additive rsync is a no-op otherwise). NOT applied +# to xCAT-genesis-base (built from xcat-core, kept in lockstep with genesis-scripts). +my $build_number; my $skip_install = 0; my $skip_build = 0; my $skip_xcat_dep = 0; @@ -64,6 +69,7 @@ GetOptions( 'max-parallel=i' => \$max_parallel, 'run-id=s' => \$run_id, 'build-timestamp=i' => \$build_timestamp, + 'build-number=i' => \$build_number, 'skip-install!' => \$skip_install, 'skip-build!' => \$skip_build, 'skip-xcat-dep!' => \$skip_xcat_dep, @@ -99,6 +105,18 @@ if ($run_id eq '') { $run_id = strftime('%Y%m%d-%H%M%S', gmtime($SOURCE_DATE_EPOCH)); } +# CD version bump. Rewrite every xcat-dep package spec's Release line in this +# (freshly-checked-out, git-clean) tree so the built rpms carry a fresh, monotonic +# NVR each run. Runs BEFORE any child builder is invoked. genesis-base lives under +# xcat-core, not $repo_root, so it is untouched (stays in lockstep with the deployed +# core's genesis-scripts). +my $RELEASE_BUMP = ''; +if (defined $build_number) { + die "--build-number must be a non-negative integer\n" if $build_number < 0; + $RELEASE_BUMP = strftime('.snap%Y%m%d%H%M', gmtime($SOURCE_DATE_EPOCH)) . ".$build_number"; + bump_dep_release_suffix($repo_root, $RELEASE_BUMP); +} + # Single output base for every NFS-shared write. Two hosts build in parallel on one NFS by # passing distinct --output paths. --output-root/--repo-dep, if given, override the derived # values. Default keeps the historical layout so existing callers are unaffected. @@ -191,6 +209,39 @@ die "FATAL: $tgt_fail target(s) failed\n" if $tgt_fail; print_step('All targets completed'); exit 0; +# Append $suffix (e.g. ".snap202607161200.57") to the Release: line of every xcat-dep +# package spec under $root, so the CD build stamps a fresh, monotonic NVR. Idempotent: +# a spec already carrying this exact suffix is left alone (so a re-run in the same tree +# does not double-stamp). Preserves any %{?dist}/%{?distver} macro already on the line. +sub bump_dep_release_suffix { + my ($root, $suffix) = @_; + my $qs = quotemeta($suffix); + my @specs; + find(sub { push @specs, $File::Find::name if /\.spec$/ && -f $_ }, $root); + my $n = 0; + for my $spec (sort @specs) { + open my $in, '<', $spec or die "open $spec: $!\n"; + my @lines = <$in>; + close $in; + my $changed = 0; + for my $line (@lines) { + next unless $line =~ /^Release:\s*\S/; + next if $line =~ /$qs\s*$/; # already stamped this run + $line =~ s/(^Release:\s*\S+)/$1$suffix/; + $changed = 1; + last; # only the first Release: line + } + next unless $changed; + open my $out, '>', $spec or die "open> $spec: $!\n"; + print {$out} @lines; + close $out; + $n++; + } + print "Release bump '$suffix' applied to $n spec(s) under $root\n"; + die "FATAL: --build-number given but no spec Release lines were bumped under $root\n" + if $n == 0; +} + # Build a single target into its own build-output/ tree and return # { repo_dir, rel }. Everything below through the summary is per-target work. sub build_one_target { From 8c337407f953056ffe7e05aef638b45cdaaa1527 Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Thu, 16 Jul 2026 15:03:15 -0300 Subject: [PATCH 02/55] fix(mockbuild-all): make release bump idempotent + concurrency-safe Two parallel per-arch builds share the NFS source tree; the second finds every spec already stamped with the identical .snap. suffix. Treat all-already-stamped as the normal idempotent case (fail only when NO spec has a Release: line at all), and write specs atomically (temp + rename) so a concurrent arch never reads a torn spec. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- mockbuild-all.pl | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/mockbuild-all.pl b/mockbuild-all.pl index 7702970..104cb40 100755 --- a/mockbuild-all.pl +++ b/mockbuild-all.pl @@ -218,28 +218,37 @@ sub bump_dep_release_suffix { my $qs = quotemeta($suffix); my @specs; find(sub { push @specs, $File::Find::name if /\.spec$/ && -f $_ }, $root); - my $n = 0; + my ($with_release, $bumped, $already) = (0, 0, 0); for my $spec (sort @specs) { open my $in, '<', $spec or die "open $spec: $!\n"; my @lines = <$in>; close $in; - my $changed = 0; + my ($has_release, $changed) = (0, 0); for my $line (@lines) { next unless $line =~ /^Release:\s*\S/; - next if $line =~ /$qs\s*$/; # already stamped this run + $has_release = 1; + if ($line =~ /$qs\s*$/) { last } # already stamped (idempotent / concurrent arch) $line =~ s/(^Release:\s*\S+)/$1$suffix/; $changed = 1; last; # only the first Release: line } + $with_release++ if $has_release; + $already++ if $has_release && !$changed; next unless $changed; - open my $out, '>', $spec or die "open> $spec: $!\n"; + # atomic write (temp + rename) so a concurrent per-arch build on the shared NFS tree never + # sees a torn spec; identical suffix -> identical content, so last-writer-wins is safe. + my $tmp = "$spec.bump.$$"; + open my $out, '>', $tmp or die "open> $tmp: $!\n"; print {$out} @lines; close $out; - $n++; + rename $tmp, $spec or die "rename $tmp -> $spec: $!\n"; + $bumped++; } - print "Release bump '$suffix' applied to $n spec(s) under $root\n"; - die "FATAL: --build-number given but no spec Release lines were bumped under $root\n" - if $n == 0; + print "Release bump '$suffix': $bumped newly stamped, $already already stamped, of $with_release spec(s) with a Release line under $root\n"; + # Only a genuine "no dep specs at all" is fatal. All-already-stamped is the expected idempotent + # case (re-run in the same tree, or the other arch bumped first) -- NOT an error. + die "FATAL: --build-number given but NO spec carried a Release: line under $root (wrong tree?)\n" + if $with_release == 0; } # Build a single target into its own build-output/ tree and return From 1abfdf5428a5a9f35c818d54418baa8a561c4a5b Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Thu, 16 Jul 2026 16:30:49 -0300 Subject: [PATCH 03/55] fix(elilo): use tracked prebuilt payload on EL8 too; require elilo-xcat EL8's gnu-efi-devel places elf_x86_64_efi.lds where elilo's Makefile can't find it, so elilo failed to compile on EL8 -- yet xCAT hard-requires elilo-xcat on every arch, so the whole dep repo became uninstallable. Reuse the same tracked prebuilt elilo-x64.efi (SOURCE4) that ppc64le already uses (elilo-x64.efi is a noarch artifact). Also add elilo-xcat to assert_required_deps so a missing elilo fails the build loudly instead of surfacing later as a dnf depsolve error. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- elilo/elilo-xcat.spec | 17 ++++++++++++----- mockbuild-all.pl | 5 ++++- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/elilo/elilo-xcat.spec b/elilo/elilo-xcat.spec index 3cbd284..b8a9156 100644 --- a/elilo/elilo-xcat.spec +++ b/elilo/elilo-xcat.spec @@ -19,10 +19,17 @@ Patch1: elilo-xcat.patch Patch2: elilo-big-bzimage-limit.patch Patch3: elilo-gnu-efi-strncpy-conflict.patch Source4: elilo-xcat-3.14-6.noarch.rpm +# Ship the tracked prebuilt EFI payload (SOURCE4) instead of compiling on targets where the +# gnu-efi toolchain layout does not support elilo's build: ppc64le (no x86 EFI toolchain) and +# EL8 (gnu-efi-devel places elf_x86_64_efi.lds under a path elilo's Makefile does not find). +# elilo-x64.efi is a noarch artifact, so the prebuilt is byte-identical to the compiled one. +%if "%{_host_cpu}" == "ppc64le" || 0%{?rhel} == 8 +%global use_prebuilt 1 +%endif BuildRequires: gcc BuildRequires: make BuildRequires: cpio -%if "%{_host_cpu}" != "ppc64le" +%if ! 0%{?use_prebuilt} BuildRequires: gnu-efi BuildRequires: gnu-efi-devel %endif @@ -48,8 +55,8 @@ sed -i 's|^GNUEFILIB[[:space:]]*=.*|GNUEFILIB = /usr/lib64|' Make.defaults sed -i 's|^EFILIB[[:space:]]*=.*|EFILIB = /usr/lib64|' Make.defaults sed -i 's|^EFICRT0[[:space:]]*=.*|EFICRT0 = /usr/lib|' Make.defaults %endif -%if "%{_host_cpu}" == "ppc64le" -# On ppc64le, reuse the prebuilt EFI payload from the tracked noarch package. +%if 0%{?use_prebuilt} +# Reuse the prebuilt EFI payload from the tracked noarch package (ppc64le / EL8). mkdir -p prebuilt rpm2cpio %{SOURCE4} | (cd prebuilt && cpio -idm --quiet) test -f prebuilt/tftpboot/xcat/elilo-x64.efi @@ -59,7 +66,7 @@ test -f prebuilt/tftpboot/xcat/elilo-x64.efi rm -rf %{buildroot} -%if "%{_host_cpu}" != "ppc64le" +%if ! 0%{?use_prebuilt} make %endif @@ -67,7 +74,7 @@ make %install mkdir -p %{buildroot}/tftpboot/xcat -%if "%{_host_cpu}" == "ppc64le" +%if 0%{?use_prebuilt} cp prebuilt/tftpboot/xcat/elilo-x64.efi %{buildroot}/tftpboot/xcat/elilo-x64.efi %else cp elilo.efi %{buildroot}/tftpboot/xcat/elilo-x64.efi diff --git a/mockbuild-all.pl b/mockbuild-all.pl index 104cb40..f3659a9 100755 --- a/mockbuild-all.pl +++ b/mockbuild-all.pl @@ -931,7 +931,10 @@ sub assert_required_deps { # xCAT Requires all of these on every arch, and every one of them builds natively on every # arch (the noarch deps -- grub2-xcat, xnba-undi -- just repackage committed artifacts), so # a self-sufficient per-arch build produces the whole set with no cross-arch import. - my @req = qw(ipmitool-xcat syslinux-xcat grub2-xcat xnba-undi + # elilo-xcat is noarch but xCAT hard-requires it (Requires: elilo-xcat >= 3.14-6) on EVERY arch, + # so a missing elilo makes the whole dep repo uninstallable -- it MUST be required here, not + # silently tolerated (it builds from a tracked prebuilt on ppc64le/EL8, compiled elsewhere). + my @req = qw(elilo-xcat ipmitool-xcat syslinux-xcat grub2-xcat xnba-undi perl-IO-Stty perl-HTTP-Async perl-Net-HTTPS-NB); push @req, 'xCAT-genesis-base' unless $skip_genesis; my @missing = grep { !have_rpm($dir, $_) } @req; From df28d848d362e59c904b9d740f5032fbee9d460d Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Thu, 16 Jul 2026 17:06:17 -0300 Subject: [PATCH 04/55] fix(elilo): set use_prebuilt via %ifarch + %if (drop || that older rpm mis-evaluates) EL8/EL9 mock chroots run an older rpm that does not evaluate an || between a string compare and an arithmetic test like EL10's rpm, so use_prebuilt stayed unset on el9-ppc and the gnu-efi BuildRequires (absent on ppc) broke the build. Use two independent %ifarch ppc64le / %if 0%{?rhel}==8 blocks instead. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- elilo/elilo-xcat.spec | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/elilo/elilo-xcat.spec b/elilo/elilo-xcat.spec index b8a9156..f9e6070 100644 --- a/elilo/elilo-xcat.spec +++ b/elilo/elilo-xcat.spec @@ -23,7 +23,13 @@ Source4: elilo-xcat-3.14-6.noarch.rpm # gnu-efi toolchain layout does not support elilo's build: ppc64le (no x86 EFI toolchain) and # EL8 (gnu-efi-devel places elf_x86_64_efi.lds under a path elilo's Makefile does not find). # elilo-x64.efi is a noarch artifact, so the prebuilt is byte-identical to the compiled one. -%if "%{_host_cpu}" == "ppc64le" || 0%{?rhel} == 8 +# Separate %ifarch / %if (NOT a single `... || ...` expression): the older rpm inside the EL8/EL9 +# mock chroot does not evaluate an `||` between a string compare and an arithmetic test the way +# EL10's rpm does, which silently left use_prebuilt unset and pulled in the gnu-efi BuildRequires. +%ifarch ppc64le +%global use_prebuilt 1 +%endif +%if 0%{?rhel} == 8 %global use_prebuilt 1 %endif BuildRequires: gcc From 552e82b87730fe0f2f2be4bee4a62bee3ab1e8a0 Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Thu, 16 Jul 2026 17:10:45 -0300 Subject: [PATCH 05/55] fix(elilo): match both ppc64le and powerpc64le for prebuilt (alma reports powerpc64le) The mock chroot's %{_host_cpu} is 'powerpc64le' on AlmaLinux ppc chroots but 'ppc64le' on Rocky, so the single ppc64le compare left use_prebuilt unset on alma and pulled in the ppc-absent gnu-efi BuildRequires. Match both spellings. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- elilo/elilo-xcat.spec | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/elilo/elilo-xcat.spec b/elilo/elilo-xcat.spec index f9e6070..397a815 100644 --- a/elilo/elilo-xcat.spec +++ b/elilo/elilo-xcat.spec @@ -23,10 +23,16 @@ Source4: elilo-xcat-3.14-6.noarch.rpm # gnu-efi toolchain layout does not support elilo's build: ppc64le (no x86 EFI toolchain) and # EL8 (gnu-efi-devel places elf_x86_64_efi.lds under a path elilo's Makefile does not find). # elilo-x64.efi is a noarch artifact, so the prebuilt is byte-identical to the compiled one. -# Separate %ifarch / %if (NOT a single `... || ...` expression): the older rpm inside the EL8/EL9 -# mock chroot does not evaluate an `||` between a string compare and an arithmetic test the way -# EL10's rpm does, which silently left use_prebuilt unset and pulled in the gnu-efi BuildRequires. -%ifarch ppc64le +# Use the tracked prebuilt on ppc (no x86 EFI toolchain) and on EL8 (gnu-efi lds path gap). +# Three SEPARATE %if blocks, each a single simple compare -- NOT one `A || B` expression (the +# older rpm in the EL8/EL9 mock chroot mis-evaluates `||`), and NOT %ifarch (during `rpmbuild -bs` +# the srpm's BuildRequires are frozen against %{_host_cpu}, not the target). Crucially, alma ppc +# chroots report %{_host_cpu}=powerpc64le while rocky reports ppc64le, so BOTH must be matched -- +# that mismatch is exactly why elilo pulled in the (ppc-absent) gnu-efi BuildRequires on alma ppc. +%if "%{_host_cpu}" == "ppc64le" +%global use_prebuilt 1 +%endif +%if "%{_host_cpu}" == "powerpc64le" %global use_prebuilt 1 %endif %if 0%{?rhel} == 8 From 554a5583ab73db71b09a20b56ea4e26c8e960082 Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Thu, 16 Jul 2026 18:03:12 -0300 Subject: [PATCH 06/55] fix(build-apt-repo): add ubuntu20.04/focal + default to xCAT Signing Key Adds the missing focal (20.04) codename and switches the default apt signing identity from the legacy xcat@megware.com key to the xCAT Signing Key (64C82A868D818E69) so xcat-dep apt InRelease is signed by the same key as xcat-core apt. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- build-apt-repo.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/build-apt-repo.sh b/build-apt-repo.sh index 7a2b8ca..ce4400d 100755 --- a/build-apt-repo.sh +++ b/build-apt-repo.sh @@ -9,11 +9,14 @@ fi REPO_ROOT="$SCRIPT_DIR" APT_DIR="" -GPG_KEY_ID="xcat@megware.com" +# Default to the xCAT Signing Key (same key that signs xcat-core), NOT the legacy megware key, +# so xcat-dep apt InRelease is signed by the same key as xcat-core apt. Override with --gpg-key-id. +GPG_KEY_ID="64C82A868D818E69" SKIP_SIGN=0 DRY_RUN=0 declare -A CODENAME_MAP=( + [ubuntu20.04]=focal [ubuntu22.04]=jammy [ubuntu24.04]=noble [ubuntu26.04]=resolute From 6238eb641547ecf445b5ea4e717d8a4011398ff8 Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Thu, 16 Jul 2026 18:22:29 -0300 Subject: [PATCH 07/55] fix(mockbuild-all): resolve mock config by .cfg file existence, not by running mock mock --print-root-path can fail transiently (bootstrap chroot setup, a concurrent mock holding a lock), which made el10 flakily resolve to the long os_id form that has no .cfg and then die 'Could not find mock config for almalinux+epel-10-...'. Check /etc/mock/.cfg existence instead -- deterministic and fast. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- mockbuild-all.pl | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/mockbuild-all.pl b/mockbuild-all.pl index f3659a9..429d1f1 100755 --- a/mockbuild-all.pl +++ b/mockbuild-all.pl @@ -1047,21 +1047,19 @@ sub resolve_mock_cfg { 'centos-stream' => 'centos-stream', rocky => 'rocky', ); - my $candidate = "${os_id}+epel-${rel}-${arch}"; - my $rc = system("mock -r " . sh_quote($candidate) . " --print-root-path >/dev/null 2>&1"); - if ($rc == 0) { - return $candidate; - } - if (exists $short_forms{$os_id}) { - my $short = $short_forms{$os_id}; - $candidate = "${short}+epel-${rel}-${arch}"; - $rc = system("mock -r " . sh_quote($candidate) . " --print-root-path >/dev/null 2>&1"); - if ($rc == 0) { - print "Mock config resolved (short form): $candidate\n"; + # Resolve by CONFIG-FILE existence, not by running `mock --print-root-path`: the latter can fail + # transiently (bootstrap chroot setup, a concurrent mock holding a lock) and made el10 flakily + # "resolve" to the long form that has no .cfg. Checking /etc/mock/.cfg is deterministic. + for my $id ($os_id, (exists $short_forms{$os_id} ? ($short_forms{$os_id}) : ())) { + my $candidate = "${id}+epel-${rel}-${arch}"; + if (-f "/etc/mock/${candidate}.cfg") { + print "Mock config resolved: $candidate\n" if $id ne $os_id; return $candidate; } } - die "Could not find mock config for ${os_id}+epel-${rel}-${arch}\n"; + my $short = $short_forms{$os_id} // $os_id; + die "Could not find mock config for ${os_id}+epel-${rel}-${arch} " + . "(tried /etc/mock/${os_id}+epel-${rel}-${arch}.cfg and /etc/mock/${short}+epel-${rel}-${arch}.cfg)\n"; } sub build_mock_uniqueext { From 94daf29fc5acf5c863d2400422bae4c90aa11bb9 Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Thu, 16 Jul 2026 20:42:12 -0300 Subject: [PATCH 08/55] feat(mockbuild-all): support SUSE targets (opensuse-leap- -> sles/) Add target_osdir() so a build target maps to its deploy subdir + family: alma+epel-10-* -> (el, rh10), opensuse-leap-15.6-* -> (suse, sles15). Thread the osdir + family through deploy_target + write_dep_repo_metadata (sles/devel zypper baseurl for SUSE, yum for EL). Lets --target opensuse-leap-{15.6,16.0}- build the dep set into sles{15,16}/. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- mockbuild-all.pl | 37 ++++++++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/mockbuild-all.pl b/mockbuild-all.pl index 429d1f1..04b375a 100755 --- a/mockbuild-all.pl +++ b/mockbuild-all.pl @@ -260,8 +260,7 @@ sub build_one_target { # different targets (e.g. alma+epel-8 vs -9) share build-output/ and # cross-contaminate. Fold the target into run_id so each target gets its own tree. $run_id = "$target-$run_id" unless index($run_id, $target) >= 0; - my ($rel) = $target =~ /epel-(\d+)-/; - die "Could not parse EL release from target '$target'\n" unless defined $rel; + my ($tfam, $rel, $osdir) = target_osdir($target); my $run_root = "$output_root/$run_id"; my $build_root = "$run_root/build-results"; @@ -613,7 +612,18 @@ print "Summary: $summary_file\n" if !$dry_run; print "Tarball: $tarball\n" if !$skip_tarball; print "SRPM Tarball: $srpm_tarball\n" if !$skip_tarball; - return { repo_dir => $repo_dir, rel => $rel }; + return { repo_dir => $repo_dir, rel => $rel, osdir => $osdir, tfam => $tfam }; +} + +# Map a mock target to (family, release, deploy-subdir): +# alma+epel-10-x86_64 -> ('el', '10', 'rh10') +# opensuse-leap-15.6-x86_64 -> ('suse', '15', 'sles15') +# The deploy layout is // for BOTH families (rh yum, sles zypper). +sub target_osdir { + my ($t) = @_; + if (my ($r) = $t =~ /epel-(\d+)-/) { return ('el', $r, "rh$r"); } + if (my ($r) = $t =~ /opensuse-leap-(\d+)\./) { return ('suse', $r, "sles$r"); } + die "Cannot derive OS dir from target '$t' (expected *+epel-N-* or opensuse-leap-N.M-*)\n"; } # Assemble the built per-target repo into the deployable, signed per-EL layout @@ -621,9 +631,11 @@ print "SRPM Tarball: $srpm_tarball\n" if !$skip_tarball; # xcat-dep.repo / mklocalrepo.sh / buildinfo.txt (ready to push to xcat.org). sub deploy_target { my ($tgt, $info) = @_; - my $rel = $info->{rel}; + my $rel = $info->{rel}; + my $osdir = $info->{osdir}; + my $tfam = $info->{tfam}; my $src = $info->{repo_dir}; - my $dest = "$repo_dep/rh$rel/$arch"; + my $dest = "$repo_dep/$osdir/$arch"; print_step("Deploy $tgt -> $dest"); return if $dry_run; make_path($dest); @@ -634,9 +646,9 @@ sub deploy_target { } assert_required_deps($dest); sign_and_index_repo($dest); - write_dep_repo_metadata($dest, $rel); + write_dep_repo_metadata($dest, $osdir, $tfam); my $n = scalar(grep { !/\.src\.rpm$/ } glob("$dest/*.rpm")); - print "Deployed rh$rel/$arch: $n rpms\n"; + print "Deployed $osdir/$arch: $n rpms\n"; } # createrepo_c command with upstream-matching, deterministic metadata. The tool's @@ -668,14 +680,17 @@ sub sign_and_index_repo { } sub write_dep_repo_metadata { - my ($dir, $rel) = @_; - my $baseurl = "https://xcat.org/files/xcat/repos/yum/devel/xcat-dep/rh$rel/$arch"; + my ($dir, $osdir, $tfam) = @_; + # yum channel for EL (rh), zypper/sles channel for SUSE (sles). + my $baseurl = ($tfam // 'el') eq 'suse' + ? "https://xcat.org/files/xcat/repos/sles/devel/xcat-dep/$osdir/$arch" + : "https://xcat.org/files/xcat/repos/yum/devel/xcat-dep/$osdir/$arch"; my $gpgcheck = $gpg_sign ? 1 : 0; my $gpgkey_line = $gpg_sign ? "gpgkey=$baseurl/repodata/repomd.xml.key" : "# gpgkey="; open my $r, '>', "$dir/xcat-dep.repo" or die "Cannot write $dir/xcat-dep.repo: $!\n"; print {$r} <<"EOF"; [xcat-dep] -name=xCAT 2 dependencies (rh$rel $arch) +name=xCAT 2 dependencies ($osdir $arch) baseurl=$baseurl enabled=1 gpgcheck=$gpgcheck @@ -710,7 +725,7 @@ EOS my $release = strftime('snap%Y%m%d%H%M', gmtime($SOURCE_DATE_EPOCH)); open my $b, '>', "$dir/buildinfo.txt" or die "Cannot write $dir/buildinfo.txt: $!\n"; print {$b} <<"EOF"; -TARGET=rh$rel/$arch +TARGET=$osdir/$arch RELEASE=$release BUILD_TIME=$build_time BUILD_MACHINE=$build_machine From 90c80b0034ed81ff016f21888be2258957dc2764 Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Thu, 16 Jul 2026 21:18:41 -0300 Subject: [PATCH 09/55] fix(suse): elilo prebuilt on suse_version; perl BuildRequires compat repo - elilo: add %if 0%{?suse_version} to use_prebuilt (openSUSE lacks the gnu-efi linker inputs elilo compiles against, same as EL8/ppc). - mockbuild-perl-packages: on openSUSE mock, inject a tiny local repo whose one noarch rpm Provides perl-generators + perl-interpreter (Requires perl), so 'dnf builddep' on the Fedora perl srpms resolves those (SUSE-absent) names. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- elilo/elilo-xcat.spec | 3 +++ mockbuild-perl-packages.pl | 44 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/elilo/elilo-xcat.spec b/elilo/elilo-xcat.spec index 397a815..156244b 100644 --- a/elilo/elilo-xcat.spec +++ b/elilo/elilo-xcat.spec @@ -38,6 +38,9 @@ Source4: elilo-xcat-3.14-6.noarch.rpm %if 0%{?rhel} == 8 %global use_prebuilt 1 %endif +%if 0%{?suse_version} +%global use_prebuilt 1 +%endif BuildRequires: gcc BuildRequires: make BuildRequires: cpio diff --git a/mockbuild-perl-packages.pl b/mockbuild-perl-packages.pl index 9408b40..7076d1b 100755 --- a/mockbuild-perl-packages.pl +++ b/mockbuild-perl-packages.pl @@ -625,10 +625,54 @@ sub create_deterministic_mock_cfg { open my $fh, '>', $cfg_path or die "Cannot write $cfg_path: $!\n"; print $fh "include('/etc/mock/${base_cfg}.cfg')\n"; print $fh "config_opts['environment']['SOURCE_DATE_EPOCH'] = '$epoch'\n"; + # SUSE: the Fedora perl srpms BuildRequire perl-generators / perl-interpreter, which do not exist + # on openSUSE (there `perl` provides the interpreter and rpm generates perl deps itself). Add a + # tiny compat repo whose one package Provides those names (and pulls perl), so `dnf builddep` + # resolves them; the actual build still uses SUSE's perl. No-op on EL/Fedora. + if ($base_cfg =~ /opensuse|sles|suse/i) { + my $repo = suse_buildreq_compat_repo(); + print $fh "config_opts['dnf.conf'] += \"\"\"\n[xcat-buildreq-compat]\nname=xcat perl BuildRequires compat\nbaseurl=file://$repo\nenabled=1\ngpgcheck=0\npriority=1\n\"\"\"\n"; + } close $fh; return $cfg_path; } +# Build (once) a noarch rpm that Provides perl-generators + perl-interpreter and put it in a local +# createrepo'd dir; return that dir. Cached across packages/targets in one run. +my $SUSE_COMPAT_REPO; +sub suse_buildreq_compat_repo { + return $SUSE_COMPAT_REPO if $SUSE_COMPAT_REPO && -f "$SUSE_COMPAT_REPO/repodata/repomd.xml"; + my $base = "/tmp/xcat-suse-buildreq-compat"; + my $rpmroot = "$base/rpmbuild"; + File::Path::make_path("$rpmroot/SPECS", "$base/repo"); + my $spec = "$rpmroot/SPECS/xcat-perl-buildreq-compat.spec"; + open my $s, '>', $spec or die "Cannot write $spec: $!\n"; + print {$s} <<'SPEC'; +Name: xcat-perl-buildreq-compat +Version: 1 +Release: 1 +Summary: Build-only compat: provide Fedora perl BuildRequires names on SUSE +License: MIT +BuildArch: noarch +Provides: perl-generators +Provides: perl-interpreter +Requires: perl +%description +Satisfies the perl-generators / perl-interpreter BuildRequires of Fedora perl +source rpms when building them under openSUSE mock. Never shipped. +%files +%changelog +SPEC + close $s; + system("rpmbuild --define " . sh_quote("_topdir $rpmroot") . " -bb " . sh_quote($spec) + . " >/dev/null 2>&1") == 0 or die "FATAL: could not build SUSE buildreq compat rpm\n"; + system("cp " . sh_quote("$rpmroot/RPMS/noarch/") . "xcat-perl-buildreq-compat-*.noarch.rpm " + . sh_quote("$base/repo/") . " && createrepo_c " . sh_quote("$base/repo") . " >/dev/null 2>&1") == 0 + or die "FATAL: could not createrepo the SUSE buildreq compat repo\n"; + $SUSE_COMPAT_REPO = "$base/repo"; + return $SUSE_COMPAT_REPO; +} + sub resolve_source_urls { my ($spec_path) = @_; open my $fh, '<', $spec_path or return (); From be17cd716a73fcaf0546d9da88e8febf23e2e522 Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Thu, 16 Jul 2026 23:13:59 -0300 Subject: [PATCH 10/55] fix(suse): perl compat also Provide perl-devel perl-Sys-Virt / perl-Crypt-SSLeay (XS modules) BuildRequire perl-devel, absent on openSUSE (the dev headers live in the perl package). Add perl-devel to the compat provides so dnf builddep resolves it. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- mockbuild-perl-packages.pl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mockbuild-perl-packages.pl b/mockbuild-perl-packages.pl index 7076d1b..b2a337a 100755 --- a/mockbuild-perl-packages.pl +++ b/mockbuild-perl-packages.pl @@ -656,10 +656,12 @@ License: MIT BuildArch: noarch Provides: perl-generators Provides: perl-interpreter +Provides: perl-devel Requires: perl %description -Satisfies the perl-generators / perl-interpreter BuildRequires of Fedora perl -source rpms when building them under openSUSE mock. Never shipped. +Satisfies the perl-generators / perl-interpreter / perl-devel BuildRequires of +Fedora perl source rpms when building them under openSUSE mock (SUSE ships the +perl dev headers inside the perl package itself). Never shipped. %files %changelog SPEC From c7a5300f013c4a9e354d25469d10f5070c6e9332 Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Mon, 20 Jul 2026 16:45:42 -0300 Subject: [PATCH 11/55] fix(mockbuild-all): drop legacy xCAT-core build; fix CD Release bump coverage Addresses the PR #62 review (viniciusferrao): (2) Remove the monolithic xCAT-core build from mockbuild-all.pl. The full core is built by the xcat-core pipeline; mockbuild-all now builds ONLY xcat-dep (dep packages, perl packages, and the OS-dependent xCAT-genesis-base). This drops the core build that required perl-generators -- absent on openSUSE Leap -- so the normal SUSE build no longer fails, and it matches the Ubuntu build's split design. The --skip-xcat flag is gone (callers updated separately). (1) Fix packages that missed the CD Release bump so each run publishes a fresh, monotonic NVR instead of being skipped: - Sys-Virt: bump_dep_release_suffix now matches Release case-insensitively (Sys-Virt.spec uses a lowercase `release:`), preserving %{?dist}. - HTML-Form / IO-Stty / Net-Telnet build from committed .src.rpm files, which the in-tree spec bump cannot reach. mockbuild-all now passes the suffix down via --release-suffix; mockbuild-perl-packages re-stamps these by unpacking the srpm, appending the suffix to the spec Release (keeping %{?dist}), and rolling a fresh srpm before rebuild. No-op without a suffix (non-CD runs rebuild the committed srpm unchanged). (3) The SUSE perl BuildRequires compat repo is now built under the caller's per-package build dir instead of the shared /tmp/xcat-suse-buildreq-compat path, so concurrent perl builds (parallel packages and parallel arch/target invocations) never race on a shared location. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- mockbuild-all.pl | 72 +++++++++++++-------------------- mockbuild-perl-packages.pl | 83 ++++++++++++++++++++++++++++++++++---- 2 files changed, 102 insertions(+), 53 deletions(-) diff --git a/mockbuild-all.pl b/mockbuild-all.pl index 04b375a..c1d7a30 100755 --- a/mockbuild-all.pl +++ b/mockbuild-all.pl @@ -37,7 +37,6 @@ my $skip_install = 0; my $skip_build = 0; my $skip_xcat_dep = 0; my $skip_perl = 0; -my $skip_xcat = 0; my $skip_genesis = 0; my $skip_createrepo = 0; my $skip_tarball = 0; @@ -74,7 +73,6 @@ GetOptions( 'skip-build!' => \$skip_build, 'skip-xcat-dep!' => \$skip_xcat_dep, 'skip-perl!' => \$skip_perl, - 'skip-xcat!' => \$skip_xcat, 'skip-genesis!' => \$skip_genesis, 'skip-createrepo!' => \$skip_createrepo, 'skip-tarball!' => \$skip_tarball, @@ -225,10 +223,11 @@ sub bump_dep_release_suffix { close $in; my ($has_release, $changed) = (0, 0); for my $line (@lines) { - next unless $line =~ /^Release:\s*\S/; + # case-insensitive: some specs (e.g. Sys-Virt.spec) use a lowercase `release:` + next unless $line =~ /^Release:\s*\S/i; $has_release = 1; if ($line =~ /$qs\s*$/) { last } # already stamped (idempotent / concurrent arch) - $line =~ s/(^Release:\s*\S+)/$1$suffix/; + $line =~ s/(^Release:\s*\S+)/$1$suffix/i; $changed = 1; last; # only the first Release: line } @@ -286,8 +285,10 @@ my @dep_builders = ( my $perl_builder = "$repo_root/mockbuild-perl-packages.pl"; +# buildrpms.pl (in xcat-core) is only needed for the OS-dependent xCAT-genesis-base +# build below; the full xCAT core is built separately by the xcat-core pipeline. die "Missing xCAT build script: $xcat_src/buildrpms.pl\n" - if !$skip_xcat && !-f "$xcat_src/buildrpms.pl"; + if !$skip_genesis && !-f "$xcat_src/buildrpms.pl"; my @active_dep_builders; for my $b (@dep_builders) { @@ -319,7 +320,6 @@ print "parallel_builds: " . (defined($parallel_builds) ? $parallel_builds : 'au print "skip_build: $skip_build\n"; print "skip_xcat_dep: $skip_xcat_dep\n"; print "skip_perl: $skip_perl\n"; -print "skip_xcat: $skip_xcat\n"; print "skip_genesis: $skip_genesis\n"; print "skip_install: $skip_install\n"; print "skip_createrepo: $skip_createrepo\n"; @@ -389,6 +389,10 @@ if (!$skip_build) { '--work-dir', sh_quote("/tmp/mockbuild-all-$run_id/perl-list6"), (($max_build_workers && $max_build_workers >= 1) ? ('--jobs', $max_build_workers) : ()), '--build-timestamp', $SOURCE_DATE_EPOCH, + # CD bump: the in-tree spec Release bump above only reaches the spec-mode perl + # packages; the srpm-mode ones (HTML-Form, IO-Stty, Net-Telnet) build from a + # committed .src.rpm, so hand the suffix down for the builder to re-stamp them. + ($RELEASE_BUMP ne '' ? ('--release-suffix', sh_quote($RELEASE_BUMP)) : ()), ($skip_install ? '--skip-install' : ()), ); push @build_steps, { @@ -400,26 +404,11 @@ if (!$skip_build) { push @collect_roots, $perl_result; } - if (!$skip_xcat) { - # Own HOME per target (buildrpms.pl uses $HOME/rpmbuild) so parallel targets don't race. - my $xcat_home = "/tmp/mockbuild-all-$run_id/xcat-home"; - my $mktree = join(' ', map { sh_quote("$xcat_home/rpmbuild/$_") } qw(SOURCES SPECS BUILD BUILDROOT RPMS SRPMS)); - my $cmd = "mkdir -p $mktree && HOME=" . sh_quote($xcat_home) . ' ' . join(' ', - 'perl', sh_quote("$xcat_src/buildrpms.pl"), - '--target', sh_quote($target), - '--nproc', int($nproc), - '--force', - '--verbose', - '--xcat_dep_path', sh_quote($repo_root), - ); - push @build_steps, { - id => 'xcat', - step => 'Build xCAT packages', - cmd => $cmd, - cwd => $xcat_src, - log => "$log_root/xcat-build.log", - }; - } + # NOTE: this script builds ONLY xcat-dep (its dep packages, the perl packages, and + # the OS-dependent xCAT-genesis-base below). The full xCAT core is built separately + # by the xcat-core pipeline -- mockbuild-all no longer has a monolithic core-build + # path (it required perl-generators, which openSUSE Leap does not provide, and did + # not match the Ubuntu build's split design). # xCAT-genesis-base is OS-dependent (its initramfs bundles the build chroot's # kernel + glibc/busybox/perl), so it is built here, per target, and shipped @@ -467,18 +456,10 @@ if (!$skip_build) { } } +# The xCAT core is built by the xcat-core pipeline, NOT here -- so we deliberately do +# NOT collect the xCAT dist tree. Only the OS-dependent xCAT-genesis-base rpm (built by +# the genesis step above) is pulled out of it, individually, further below. my $xcat_rpms_dir = "$xcat_src/dist/$target/rpms"; -my $xcat_srpms_dir = "$xcat_src/dist/$target/srpms"; - -# In monolithic mode (no --skip-xcat) the whole xCAT core built here (incl. -# genesis-base) is collected into this repo. In the split pipeline (--skip-xcat, -# core built separately) the orchestrator (cluster-test.pl) routes -# xCAT-genesis-base from the xCAT dist tree into the per-EL dep repo itself -- -# robust to this script exiting non-zero on tolerated dep-builder failures -- so -# we deliberately do NOT collect the xCAT dist tree here. -if (!$skip_xcat) { - push @collect_roots, $xcat_rpms_dir; -} if ($skip_build) { push @collect_roots, @@ -495,9 +476,7 @@ if ($skip_build) { push @collect_roots, @extra_collect_dirs; @collect_roots = uniq(@collect_roots); -my @srpm_collect_roots = (!$skip_xcat) - ? uniq(@collect_roots, $xcat_srpms_dir) - : uniq(@collect_roots); +my @srpm_collect_roots = uniq(@collect_roots); print_step('Collect RPM artifacts'); print "collection roots:\n"; @@ -514,7 +493,8 @@ if (!$dry_run && $copied == 0) { } # Ensure the OS-dependent xCAT-genesis-base rpm (built by the genesis step above) -# lands in the dep repo even when the full xCAT core is built elsewhere (--skip-xcat). +# lands in the dep repo -- pull it individually out of the xcat-core dist tree (the +# rest of that tree, the full xCAT core, is built + published by the xcat-core pipeline). if (!$skip_genesis && !$dry_run) { for my $g (glob("$xcat_rpms_dir/xCAT-genesis-base-*.rpm")) { next if $g =~ /\.src\.rpm$/; @@ -740,7 +720,9 @@ sub usage { return <<"USAGE"; Usage: $0 [options] -Build xcat-dep and xCAT RPMs, consolidate binary/source artifacts, run createrepo, and create tarballs. +Build xcat-dep RPMs (dep packages, perl packages, and the OS-dependent xCAT-genesis-base), +consolidate binary/source artifacts, run createrepo, and create tarballs. The full xCAT core +is built separately by the xcat-core pipeline, not here. Options: --repo-root PATH xcat-dep repository root (default: script directory) @@ -770,7 +752,7 @@ Options: --skip-build Skip all build steps and only collect/create repo/tarballs --skip-xcat-dep Skip xcat-dep mockbuild.pl package steps --skip-perl Skip perl package build step - --skip-xcat Skip xCAT buildrpms.pl step + --skip-genesis Skip the xCAT-genesis-base build step --skip-createrepo Skip createrepo --skip-tarball Skip binary/SRPM tarball creation --scrub-all-chroots Run mock -r --scrub=all before build/collect @@ -780,8 +762,8 @@ Options: Notes: - Run this script as root on the build host. - ARCH is derived from: uname -m - - Top-level parallel queue includes xcat-dep mockbuild.pl steps, perl builder, - and ../xcat-core/buildrpms.pl. + - Top-level parallel queue includes xcat-dep mockbuild.pl steps, the perl builder, + and the xCAT-genesis-base build (../xcat-core/buildrpms.pl --package xCAT-genesis-base). - Child mockbuild scripts are invoked with per-step mock --uniqueext values to avoid lock collisions on the same mock config. - If --target is omitted, it is deduced from /etc/os-release: diff --git a/mockbuild-perl-packages.pl b/mockbuild-perl-packages.pl index b2a337a..fddb77e 100755 --- a/mockbuild-perl-packages.pl +++ b/mockbuild-perl-packages.pl @@ -20,6 +20,10 @@ my $jobs = 0; my $skip_install = 0; my $allow_erasing = 0; my $build_timestamp; +# CD version bump: appended to the Release of the srpm-mode packages (HTML-Form, IO-Stty, +# Net-Telnet), which build from a committed .src.rpm and so are NOT covered by mockbuild-all's +# in-tree spec bump. Spec-mode packages get bumped in-tree upstream, so we leave those alone. +my $release_suffix = ''; GetOptions( 'work-dir=s' => \$work_dir, @@ -32,6 +36,7 @@ GetOptions( 'skip-install!' => \$skip_install, 'allow-erasing!' => \$allow_erasing, 'build-timestamp=i' => \$build_timestamp, + 'release-suffix=s' => \$release_suffix, ) or die usage(); die "Run as root (current uid=$>)\n" if $> != 0; @@ -184,6 +189,7 @@ print "packages: " . join(', ', @packages) . "\n"; print "jobs: $jobs\n"; print "skip_install:$skip_install\n"; print "allow_erasing:$allow_erasing\n"; +print "release_suffix:" . ($release_suffix ne '' ? $release_suffix : '(none)') . "\n"; print_step("Mock config check"); run("mock -r " . sh_quote($mock_cfg) . $mock_uniqueext_opt . " --print-root-path >/dev/null"); @@ -222,6 +228,7 @@ for my $idx (0 .. $#packages) { arch => $arch, skip_install => $skip_install, allow_erasing => $allow_erasing, + release_suffix => $release_suffix, ); $pm->finish($ok ? 0 : 1); } @@ -282,6 +289,7 @@ sub build_package { my $arch = $args{arch}; my $skip_install = $args{skip_install}; my $allow_erasing = $args{allow_erasing}; + my $release_suffix = $args{release_suffix}; my $pkg_run_dir = "$work_dir/$pkg"; my $pkg_result = "$result_dir/$pkg"; @@ -317,6 +325,35 @@ sub build_package { if ($cfg->{mode} eq 'srpm') { $srpm_path = select_srpm($cfg->{srpm_globs}); die "Could not locate source RPM for $pkg\n" if !$srpm_path; + # CD version bump: these packages build from a committed .src.rpm, so the in-tree + # spec Release bump (mockbuild-all) never reaches them. Re-stamp here: unpack the + # srpm, append the suffix to its spec's Release (KEEPING %{?dist}, exactly like the + # spec-mode packages -> e.g. 19%{?dist} -> 19%{?dist}.snap...N), and roll a fresh + # srpm. With no suffix (non-CD run), rebuild the committed srpm unchanged. + if ($release_suffix ne '') { + my $ext = "$pkg_run_dir/restamp"; + for my $d (qw(BUILD BUILDROOT RPMS SOURCES SPECS SRPMS)) { make_path("$ext/$d"); } + run("rpm -i --define " . sh_quote("_topdir $ext") . ' ' . sh_quote($srpm_path) + . " > " . sh_quote("$pkg_log/srpm-unpack.log") . " 2>&1"); + my ($espec) = sort glob("$ext/SPECS/*.spec"); + die "No spec found after unpacking srpm for $pkg\n" if !$espec; + append_release_suffix($espec, $release_suffix); + my $restamp_result = "$pkg_run_dir/restamp-srpm"; + make_path($restamp_result); + run( + "mock -r " . sh_quote($det_mock_cfg) . $mock_uniqueext_opt . + " --buildsrpm --spec " . sh_quote($espec) . + " --sources " . sh_quote("$ext/SOURCES") . + " --define " . sh_quote("use_source_date_epoch_as_buildtime 1") . + " --define " . sh_quote("clamp_mtime_to_source_date_epoch 1") . + " --define " . sh_quote("_buildhost xcat-build") . + " --resultdir " . sh_quote($restamp_result) . + " > " . sh_quote("$pkg_log/mock-restamp-buildsrpm.log") . " 2>&1" + ); + my @restamped = sort glob("$restamp_result/*.src.rpm"); + die "No re-stamped SRPM produced for $pkg in $restamp_result\n" if !@restamped; + $srpm_path = $restamped[-1]; + } } else { my $spec = $cfg->{spec}; die "Missing spec for $pkg: $spec\n" if !-f $spec; @@ -478,11 +515,38 @@ Usage: $0 [options] --log-dir PATH Log directory (default: build-logs/list6/perl/) --packages LIST Comma-separated subset of packages to build --build-timestamp EPOCH Unix epoch for SOURCE_DATE_EPOCH (deterministic builds) + --release-suffix STR CD bump appended to the Release of the srpm-mode packages that build + from a committed .src.rpm (HTML-Form, IO-Stty, Net-Telnet) --skip-install Skip dnf install + perl module import checks --allow-erasing Allow dnf to erase conflicting packages during install smoke tests USAGE } +# Append $suffix (e.g. ".snap202607161200.57") to the first Release: line of $spec, in place. +# Mirrors mockbuild-all's bump_dep_release_suffix: case-insensitive (some specs use lowercase +# `release:`), preserves any %{?dist} macro on the line, and is idempotent (a line already +# carrying this exact suffix is left as-is). +sub append_release_suffix { + my ($spec, $suffix) = @_; + my $qs = quotemeta($suffix); + open my $in, '<', $spec or die "open $spec: $!\n"; + my @lines = <$in>; + close $in; + my $changed = 0; + for my $line (@lines) { + next unless $line =~ /^Release:\s*\S/i; + last if $line =~ /$qs\s*$/; # already stamped + $line =~ s/(^Release:\s*\S+)/$1$suffix/i; + $changed = 1; + last; # only the first Release: line + } + die "No Release: line to stamp in $spec\n" if !$changed && !grep { /^Release:\s*\S/i } @lines; + return if !$changed; + open my $out, '>', $spec or die "open> $spec: $!\n"; + print {$out} @lines; + close $out; +} + sub select_srpm { my ($globs_ref) = @_; for my $g (@{$globs_ref}) { @@ -630,19 +694,23 @@ sub create_deterministic_mock_cfg { # tiny compat repo whose one package Provides those names (and pulls perl), so `dnf builddep` # resolves them; the actual build still uses SUSE's perl. No-op on EL/Fedora. if ($base_cfg =~ /opensuse|sles|suse/i) { - my $repo = suse_buildreq_compat_repo(); + my $repo = suse_buildreq_compat_repo($dir); print $fh "config_opts['dnf.conf'] += \"\"\"\n[xcat-buildreq-compat]\nname=xcat perl BuildRequires compat\nbaseurl=file://$repo\nenabled=1\ngpgcheck=0\npriority=1\n\"\"\"\n"; } close $fh; return $cfg_path; } -# Build (once) a noarch rpm that Provides perl-generators + perl-interpreter and put it in a local -# createrepo'd dir; return that dir. Cached across packages/targets in one run. -my $SUSE_COMPAT_REPO; +# Build a noarch rpm that Provides perl-generators + perl-interpreter and put it in a local +# createrepo'd dir; return that dir. The repo is built UNDER the caller's per-package build dir +# (not a shared /tmp path): each build_package fork runs this once for its own package, and +# mockbuild-all gives every (run, target/arch) its own work-dir -- so concurrent perl builds, +# whether parallel packages in one invocation or parallel arch/target invocations, never race on +# a shared path. Idempotent within a dir (reuse if already built). sub suse_buildreq_compat_repo { - return $SUSE_COMPAT_REPO if $SUSE_COMPAT_REPO && -f "$SUSE_COMPAT_REPO/repodata/repomd.xml"; - my $base = "/tmp/xcat-suse-buildreq-compat"; + my ($base_dir) = @_; + my $base = "$base_dir/suse-buildreq-compat"; + return "$base/repo" if -f "$base/repo/repodata/repomd.xml"; my $rpmroot = "$base/rpmbuild"; File::Path::make_path("$rpmroot/SPECS", "$base/repo"); my $spec = "$rpmroot/SPECS/xcat-perl-buildreq-compat.spec"; @@ -671,8 +739,7 @@ SPEC system("cp " . sh_quote("$rpmroot/RPMS/noarch/") . "xcat-perl-buildreq-compat-*.noarch.rpm " . sh_quote("$base/repo/") . " && createrepo_c " . sh_quote("$base/repo") . " >/dev/null 2>&1") == 0 or die "FATAL: could not createrepo the SUSE buildreq compat repo\n"; - $SUSE_COMPAT_REPO = "$base/repo"; - return $SUSE_COMPAT_REPO; + return "$base/repo"; } sub resolve_source_urls { From d7311caf084db1c874f76f65ac313e49f6d09ba8 Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Mon, 20 Jul 2026 18:17:38 -0300 Subject: [PATCH 12/55] fix(xcat-dep): provide cross-arch xCAT-genesis-base in each dep repo In 2.17 every per-arch xcat-dep repo also shipped the OTHER arch's noarch xCAT-genesis-base (the x86_64 repo carried xCAT-genesis-base-ppc64, and the ppc64le repo carried xCAT-genesis-base-x86_64) so an MN could netboot nodes of the other architecture. The matrix build produces per-arch repos that each carry only their own genesis, and stale foreign-arch genesis rpms lingered in the repo unindexed -- which breaks `go-xcat install` on EL9 because the repodata does not resolve the whole install list (xcat2/xcat-core#7610). Add a build-free, lock-free `--finalize-xcat-dep --x86-repo --ppc-repo ` mode. For each matching /x86_64 and /ppc64le repo pair it copies the noarch xCAT-genesis-base-ppc64 into the x86_64 repo and xCAT-genesis-base-x86_64 into the ppc64le repo, drops any stale foreign-arch genesis first, re-signs the copied rpm + repomd under --gpg-sign, and re-indexes with createrepo_c. Idempotent: an already up-to-date pair copies nothing and is not re-indexed. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- mockbuild-all.pl | 128 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 127 insertions(+), 1 deletion(-) diff --git a/mockbuild-all.pl b/mockbuild-all.pl index c1d7a30..71952f3 100755 --- a/mockbuild-all.pl +++ b/mockbuild-all.pl @@ -48,6 +48,11 @@ my $gpg_sign = 0; my $gpg_key_name = 'xCAT Signing Key'; my $gpg_home = ''; my $force_unlock = 0; +# --finalize-xcat-dep: post-build cross-arch genesis provisioning (issue #7610). Takes the two +# per-arch repo roots and cross-populates the noarch xCAT-genesis-base between them. +my $finalize_xcat_dep = 0; +my $x86_repo = ''; +my $ppc_repo = ''; my $HELD_LOCK; # path of the output lock this process owns (for cleanup on exit) my $LOCK_OWNER_PID; # pid that created the lock; forked children must NOT remove it @@ -61,6 +66,9 @@ GetOptions( 'gpg-key-name=s' => \$gpg_key_name, 'gpg-home=s' => \$gpg_home, 'force-unlock!' => \$force_unlock, + 'finalize-xcat-dep!' => \$finalize_xcat_dep, + 'x86-repo=s' => \$x86_repo, + 'ppc-repo=s' => \$ppc_repo, 'target=s' => \$target, 'nproc=i' => \$nproc, 'parallel-builds=i' => \$parallel_builds, @@ -81,7 +89,7 @@ GetOptions( 'dry-run!' => \$dry_run, ) or die usage(); -die "Run as root (uid=$>)\n" if $> != 0; +die "Run as root (uid=$>)\n" if $> != 0 && !$finalize_xcat_dep; die "--parallel-builds must be >= 1\n" if defined($parallel_builds) && $parallel_builds < 1; @@ -103,6 +111,28 @@ if ($run_id eq '') { $run_id = strftime('%Y%m%d-%H%M%S', gmtime($SOURCE_DATE_EPOCH)); } +# --finalize-xcat-dep: a distinct, build-free mode. After BOTH arch build hosts have +# produced their per-EL repos (each carrying only its own xCAT-genesis-base), the x86_64 +# repo must ALSO ship the noarch xCAT-genesis-base-ppc64 (so an x86_64 MN can netboot ppc +# nodes) and the ppc64le repo must ship xCAT-genesis-base-x86_64 -- the 2.17 behaviour +# that issue #7610 regressed. This mode ONLY cross-copies the genesis-base rpm(s) between +# the two repos (dropping any stale foreign-arch genesis) and re-indexes + re-signs the +# affected repomd; it builds nothing and holds no output lock. +if ($finalize_xcat_dep) { + die "--finalize-xcat-dep requires --x86-repo and --ppc-repo\n" + if $x86_repo eq '' || $ppc_repo eq ''; + require_command('createrepo_c'); + require_command('rpm'); + require_command('rpmsign') if $gpg_sign; + require_command('gpg') if $gpg_sign; + my $x86 = abs_path($x86_repo) or die "--x86-repo '$x86_repo' not found\n"; + my $ppc = abs_path($ppc_repo) or die "--ppc-repo '$ppc_repo' not found\n"; + die "--x86-repo '$x86' is not a directory\n" if !-d $x86; + die "--ppc-repo '$ppc' is not a directory\n" if !-d $ppc; + finalize_xcat_dep($x86, $ppc); + exit 0; +} + # CD version bump. Rewrite every xcat-dep package spec's Release line in this # (freshly-checked-out, git-clean) tree so the built rpms carry a fresh, monotonic # NVR each run. Runs BEFORE any child builder is invoked. genesis-base lives under @@ -716,6 +746,93 @@ EOF close $b; } +# --finalize-xcat-dep: cross-populate the noarch xCAT-genesis-base between each matching +# /x86_64 and /ppc64le repo pair, then re-index + re-sign the repos that changed. +# / are the two per-arch repo roots (each holding rh8/rh9/rh10/, +# and possibly sles/). They may be the same path (both arches built into one tree) +# or two separate trees (one per build host); either way pairs are matched by subdir. +sub finalize_xcat_dep { + my ($x86_repo, $ppc_repo) = @_; + print_step('Finalize xcat-dep: cross-arch genesis-base provisioning (issue #7610)'); + print "x86-repo: $x86_repo\n"; + print "ppc-repo: $ppc_repo\n"; + # Every OS-release dir under the x86 repo that actually has an x86_64 sub-repo. + my @osdirs = grep { -d "$_/x86_64" } glob("$x86_repo/*"); + my $pairs = 0; + for my $p (sort @osdirs) { + my $osdir = basename($p); + my $x86dir = "$x86_repo/$osdir/x86_64"; + my $ppcdir = "$ppc_repo/$osdir/ppc64le"; + if (!-d $ppcdir) { + print "[finalize] $osdir: no ppc64le peer at $ppcdir -- skipping\n"; + next; + } + # xCAT collapses ppc/ppc64/ppc64le into tarch=ppc64, so the ppc genesis rpm is + # named xCAT-genesis-base-ppc64-*. Cross-copy both directions. + my $to_x86 = cross_copy_genesis($ppcdir, $x86dir, 'ppc64'); + my $to_ppc = cross_copy_genesis($x86dir, $ppcdir, 'x86_64'); + reindex_and_sign_repo($x86dir) if $to_x86; + reindex_and_sign_repo($ppcdir) if $to_ppc; + printf "[finalize] %s: %d ppc64 genesis -> x86_64, %d x86_64 genesis -> ppc64le\n", + $osdir, $to_x86, $to_ppc; + $pairs++; + } + die "FATAL: --finalize-xcat-dep found no /x86_64 + /ppc64le repo pair under\n" + . " --x86-repo '$x86_repo'\n --ppc-repo '$ppc_repo'\n" if $pairs == 0; + print_step('Finalize complete'); +} + +# Cross-copy the noarch xCAT-genesis-base--*.rpm from $from into $to. Drops any +# stale foreign-arch genesis already in $to (e.g. issue #7610's 2.16.3 ppc leftover) so +# the repo ends with exactly the fresh set. Returns the number of rpms newly copied +# (0 = already up to date, so the caller can skip re-indexing). Idempotent. +sub cross_copy_genesis { + my ($from, $to, $tarch) = @_; + my @src = grep { !/\.src\.rpm$/ } glob("$from/xCAT-genesis-base-$tarch-*.rpm"); + return 0 if !@src; + my %want = map { basename($_) => $_ } @src; + my @existing = grep { !/\.src\.rpm$/ } glob("$to/xCAT-genesis-base-$tarch-*.rpm"); + my %have = map { basename($_) => 1 } @existing; + # Already exactly the fresh set (same basenames)? idempotent no-op. + if (scalar(keys %want) == scalar(keys %have) && !grep { !$have{$_} } keys %want) { + return 0; + } + for my $old (@existing) { + unlink $old or die "Failed to remove stale genesis $old: $!\n"; + print "[finalize] - " . basename($old) . " (stale foreign-arch, removed from $to)\n"; + } + my $copied = 0; + for my $base (sort keys %want) { + copy($want{$base}, "$to/$base") + or die "Failed to cross-copy genesis $want{$base} -> $to: $!\n"; + print "[finalize] + $base ($from -> $to)\n"; + # The source rpm is already signed by the build, but re-assert it under --gpg-sign + # so the deploy signing gate never sees an unsigned cross-copied rpm. + if ($gpg_sign) { + local $ENV{GNUPGHOME} = $gpg_home if $gpg_home; + run_simple(qq(rpmsign --define "%_gpg_name $gpg_key_name" --addsign ) + . sh_quote("$to/$base")); + } + $copied++; + } + return $copied; +} + +# Re-run createrepo_c on a repo whose rpm set changed, and (under --gpg-sign) re-sign + +# re-export repomd. Does NOT re-sign the rpms (cross_copy_genesis already did the copied +# one; the rest keep their build-time signatures). +sub reindex_and_sign_repo { + my ($dir) = @_; + run_simple(createrepo_c_cmd($dir)); + if ($gpg_sign) { + local $ENV{GNUPGHOME} = $gpg_home if $gpg_home; + my $repomd = "$dir/repodata/repomd.xml"; + unlink "$repomd.asc" if -f "$repomd.asc"; + run_simple(qq(gpg -a --detach-sign --default-key "$gpg_key_name" ) . sh_quote($repomd)); + run_simple(qq(gpg -a --export "$gpg_key_name" > ) . sh_quote("$repomd.key")); + } +} + sub usage { return <<"USAGE"; Usage: $0 [options] @@ -734,6 +851,15 @@ Options: --repo-dep PATH Override the derived deployable per-EL output root; rh8/rh9/rh10/ are assembled + signed here (default: /xcat-dep) --force-unlock Remove a stale /.lock before acquiring it + --finalize-xcat-dep Post-build cross-arch genesis mode (builds nothing). Requires + --x86-repo and --ppc-repo. For each matching /x86_64 and + /ppc64le repo pair, copies the noarch xCAT-genesis-base-ppc64 + into the x86_64 repo and xCAT-genesis-base-x86_64 into the ppc64le + repo (dropping any stale foreign-arch genesis), then re-indexes + + re-signs. Restores the 2.17 cross-arch genesis (issue #7610). + Honors --gpg-sign/--gpg-key-name/--gpg-home. Use alone. + --x86-repo PATH (finalize) x86_64 repo root holding /x86_64 (e.g. rh9/x86_64) + --ppc-repo PATH (finalize) ppc64le repo root holding /ppc64le --gpg-sign Sign rpms + repomd.xml of each per-EL repo --gpg-key-name NAME GPG key name (default: "xCAT Signing Key") --gpg-home PATH GNUPGHOME for signing (default: system keyring) From a1ab3938d1521586354fc64ee7877f9a262b7571 Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Tue, 21 Jul 2026 18:09:09 -0300 Subject: [PATCH 13/55] fix(xcat-dep): scope PR to EL and address cross-arch review feedback Per review on xcat2/xcat-dep#62, narrow this PR to the EL matrix only (rh8/rh9/rh10 x x86_64/ppc64le). The SUSE and Ubuntu work is reverted out of the PR's net diff and will land in its own PR, so the reviewer's SUSE/Ubuntu points (SuSE breakage, ubuntu20.04/focal in the default set, the xcat@megware.com key default) are moot here -- those targets are no longer part of this change. Reverted (net-zero vs master): - SUSE target support in mockbuild-all.pl (opensuse-leap -> sles). - SUSE perl BuildRequires compat + elilo suse_version hunk. - build-apt-repo.sh Ubuntu changes (focal + key default). EL-relevant review fixes kept: - Rename the finalize options to precise arch names: --x86-repo/--ppc-repo -> --x86_64-repo/--ppc64le-repo (and matching vars/labels). "x86"/"ppc" was ambiguous, especially since the genesis package is named -ppc64 via tarch yet carries no big-endian code. - Clarify the "tolerated build" comment (reviewer #2): builder failures are tolerated only so one flaky builder cannot abort the others; correctness is enforced by RESULT via assert_required_deps (a missing REQUIRED rpm still fails the run), not by exit code. Toleration is load-bearing -- perl-Sys-Virt fails on el8 by design, and genesis "fails" cosmetically while still producing its rpm. - BUILD.md: drop all references to the removed --skip-xcat flag and the stale "unified xCAT repository" framing (the core is built by the xcat-core pipeline), and document --finalize-xcat-dep. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- BUILD.md | 99 +++++++++++++++++++-------------- build-apt-repo.sh | 5 +- elilo/elilo-xcat.spec | 3 - mockbuild-all.pl | 110 +++++++++++++++++-------------------- mockbuild-perl-packages.pl | 49 ----------------- 5 files changed, 109 insertions(+), 157 deletions(-) diff --git a/BUILD.md b/BUILD.md index 43fd88f..d386be7 100644 --- a/BUILD.md +++ b/BUILD.md @@ -1,12 +1,16 @@ -# Build Guide (`mockbuild-all.pl`) +# Build Guide (xcat-dep) -This guide explains how to use `mockbuild-all.pl` to build, validate, and package xCAT dependencies and optional xCAT packages into a unified EL10 repository layout. -It also documents the operational flags for controlling build, install-check, collection, and packaging behavior. +This guide explains how to build, validate, and package the **xcat-dep** dependency packages +with `mockbuild-all.pl` — the RPM build orchestrator for EL targets, driven by `mock`. + +The full xCAT **core** is NOT built here — it is built and published separately by the +xcat-core pipeline. `mockbuild-all.pl` builds only the dependency packages plus the +OS-dependent `xCAT-genesis-base` (pulled individually out of the xcat-core source tree). # Purpose -`mockbuild-all.pl` is the top-level build orchestrator for generating a **unified xCAT repository**. -It builds required dependency RPMs and, by default, xCAT RPMs, then assembles: +`mockbuild-all.pl` is the top-level build orchestrator for the **xcat-dep** RPM repository. +It builds the dependency RPMs and the OS-dependent `xCAT-genesis-base`, then assembles: - a binary RPM repo tree with repodata - an SRPM repo tree with repodata @@ -14,12 +18,14 @@ It builds required dependency RPMs and, by default, xCAT RPMs, then assembles: # Historical Context -Historically, the deployment flow used separate repositories: +The deployment flow uses two separate repositories: -- `xcat-core` for xCAT packages -- `xcat-dep` for dependency packages +- `xcat-core` for xCAT packages (built by the xcat-core pipeline) +- `xcat-dep` for dependency packages (built here) -The current flow produces a single **unified `xcat` repository** containing all required packages together. +An earlier iteration of this script also built the full xCAT core into one unified tree +(via a `--skip-xcat` toggle). That is gone: the core is always built by the xcat-core +pipeline now, and `mockbuild-all.pl` builds only xcat-dep plus `xCAT-genesis-base`. # Placeholder Conventions @@ -42,8 +48,10 @@ This guide uses the following placeholders consistently: - `/ipmitool/mockbuild.pl` - `/syslinux/mockbuild.pl` - `/goconserver/mockbuild.pl` +- `/conserver/mockbuild.pl` +- `/xnba/mockbuild.pl` - `/mockbuild-perl-packages.pl` -- `/buildrpms.pl` (unless `--skip-xcat` is set) +- `/buildrpms.pl` — only to build the OS-dependent `xCAT-genesis-base` package (unless `--skip-genesis` is set); the full xCAT core is built separately by the xcat-core pipeline, not here. Each build path uses `mock` for chroot isolation. Top-level steps are parallelized by `mockbuild-all.pl`, and perl dependency builds are also parallelized internally by `mockbuild-perl-packages.pl`. @@ -64,10 +72,10 @@ Use these flags to skip specific operations: - `--skip-install` - Skips install/smoke checks performed by child builder scripts after RPM build. -- `--skip-xcat` - - Skips `/buildrpms.pl` (xCAT package build step). +- `--skip-genesis` + - Skips the `xCAT-genesis-base` build (`/buildrpms.pl --package xCAT-genesis-base`). - `--skip-xcat-dep` - - Skips non-perl xcat-dep package builders (`elilo`, `grub2-xcat`, `ipmitool-xcat`, `syslinux-xcat`, `goconserver`). + - Skips non-perl xcat-dep package builders (`elilo`, `grub2-xcat`, `ipmitool-xcat`, `syslinux-xcat`, `goconserver`, `conserver-xcat`, `xnba-undi`). - `--skip-perl` - Skips `/mockbuild-perl-packages.pl`. - `--skip-build` @@ -95,7 +103,7 @@ Install baseline tooling: dnf -y install perl perl-Parallel-ForkManager mock createrepo tar rpm-build rpmdevtools dnf-plugins-core wget git ``` -If you will build xCAT packages (that is, you will **not** use `--skip-xcat`), install xCAT build dependencies: +If you will build the `xCAT-genesis-base` package (that is, you will **not** use `--skip-genesis`), install xCAT build dependencies: ```bash cd @@ -129,9 +137,14 @@ Equivalent derivation: mock -r ... ``` -# Build Full Unified Repository (xCAT + Dependencies) +By default (no `--target`), `mockbuild-all.pl` builds all three EL releases for the host +arch: `rh8`, `rh9`, and `rh10`. Pass `--target` (repeatable) to restrict the set. -Use this mode to build dependency packages and xCAT packages together. +# Build the Dependency Repository + +`mockbuild-all.pl` builds the xcat-dep packages (dep packages, perl packages, and the +OS-dependent `xCAT-genesis-base`). The full xCAT core is **not** built here — it is built and +published separately by the xcat-core pipeline. ```bash cd /root/xcat-dep @@ -145,31 +158,13 @@ Notes: - Install/smoke checks run by default inside child builders. - Add `--skip-install` to skip those checks. +- Add `--skip-genesis` to skip the `xCAT-genesis-base` build (the only step that invokes + `/buildrpms.pl`). - `` is optional; when omitted it is timestamp-based. -# Build Unified Repository Without xCAT (`--skip-xcat`) - -Use this mode to build dependency packages only and skip invoking `/root/xcat-dep/xcat-source-code/buildrpms.pl`. - -```bash -cd /root/xcat-dep -perl ./mockbuild-all.pl \ - --repo-root /root/xcat-dep \ - --xcat-source /root/xcat-dep/xcat-source-code \ - --scrub-all-chroots \ - --skip-xcat \ - --skip-install -``` - -Important behavior: - -- `--skip-xcat` skips the xCAT build step, but collection still scans: - - `/dist//rpms` -- If that path already has xCAT RPMs, they are included in the resulting unified repo. - # Common Build Modes -Full unified repo (xCAT + dependencies, with install/smoke checks): +xcat-dep repo (with install/smoke checks): ```bash cd @@ -179,7 +174,7 @@ perl ./mockbuild-all.pl \ --scrub-all-chroots ``` -Full unified repo (xCAT + dependencies, skip install/smoke checks): +xcat-dep repo (skip install/smoke checks): ```bash cd @@ -190,7 +185,7 @@ perl ./mockbuild-all.pl \ --skip-install ``` -Dependency-only repo (skip xCAT package build): +Dependency repo without the `xCAT-genesis-base` build: ```bash cd @@ -198,7 +193,7 @@ perl ./mockbuild-all.pl \ --repo-root \ --xcat-source \ --scrub-all-chroots \ - --skip-xcat \ + --skip-genesis \ --skip-install ``` @@ -212,6 +207,30 @@ perl ./mockbuild-all.pl \ --skip-build ``` +# Cross-arch genesis-base (`--finalize-xcat-dep`) + +`xCAT-genesis-base` is a noarch package whose *name* carries the target arch +(`xCAT-genesis-base-x86_64`, `xCAT-genesis-base-ppc64` — xCAT collapses `ppc64le` to `ppc64` +via `tarch`; there is no big-endian code in it). A management node must be able to netboot +nodes of the *other* arch, so — as in 2.17 — the `x86_64` dep repo must also ship the +`ppc64` genesis and the `ppc64le` dep repo must ship the `x86_64` genesis. + +Each arch is built on its own build host, so once both per-arch repos exist, run a final, +build-free pass that cross-copies the noarch genesis between them and re-indexes + re-signs +the affected repos: + +```bash +perl ./mockbuild-all.pl --finalize-xcat-dep \ + --x86_64-repo /x86_64> \ + --ppc64le-repo /ppc64le> \ + --gpg-sign --gpg-key-name "xCAT Signing Key" --gpg-home +``` + +- If both arches were built into one shared tree, pass the same path to both options. +- Idempotent: a repo pair already carrying the fresh foreign-arch genesis is left untouched; + any stale foreign-arch genesis is dropped before the fresh one is copied in. +- It builds nothing and holds no output lock — use it alone. + # Output Artifacts and Paths For each run: diff --git a/build-apt-repo.sh b/build-apt-repo.sh index ce4400d..7a2b8ca 100755 --- a/build-apt-repo.sh +++ b/build-apt-repo.sh @@ -9,14 +9,11 @@ fi REPO_ROOT="$SCRIPT_DIR" APT_DIR="" -# Default to the xCAT Signing Key (same key that signs xcat-core), NOT the legacy megware key, -# so xcat-dep apt InRelease is signed by the same key as xcat-core apt. Override with --gpg-key-id. -GPG_KEY_ID="64C82A868D818E69" +GPG_KEY_ID="xcat@megware.com" SKIP_SIGN=0 DRY_RUN=0 declare -A CODENAME_MAP=( - [ubuntu20.04]=focal [ubuntu22.04]=jammy [ubuntu24.04]=noble [ubuntu26.04]=resolute diff --git a/elilo/elilo-xcat.spec b/elilo/elilo-xcat.spec index 156244b..397a815 100644 --- a/elilo/elilo-xcat.spec +++ b/elilo/elilo-xcat.spec @@ -38,9 +38,6 @@ Source4: elilo-xcat-3.14-6.noarch.rpm %if 0%{?rhel} == 8 %global use_prebuilt 1 %endif -%if 0%{?suse_version} -%global use_prebuilt 1 -%endif BuildRequires: gcc BuildRequires: make BuildRequires: cpio diff --git a/mockbuild-all.pl b/mockbuild-all.pl index 71952f3..44c817f 100755 --- a/mockbuild-all.pl +++ b/mockbuild-all.pl @@ -51,8 +51,8 @@ my $force_unlock = 0; # --finalize-xcat-dep: post-build cross-arch genesis provisioning (issue #7610). Takes the two # per-arch repo roots and cross-populates the noarch xCAT-genesis-base between them. my $finalize_xcat_dep = 0; -my $x86_repo = ''; -my $ppc_repo = ''; +my $x86_64_repo = ''; +my $ppc64le_repo = ''; my $HELD_LOCK; # path of the output lock this process owns (for cleanup on exit) my $LOCK_OWNER_PID; # pid that created the lock; forked children must NOT remove it @@ -67,8 +67,8 @@ GetOptions( 'gpg-home=s' => \$gpg_home, 'force-unlock!' => \$force_unlock, 'finalize-xcat-dep!' => \$finalize_xcat_dep, - 'x86-repo=s' => \$x86_repo, - 'ppc-repo=s' => \$ppc_repo, + 'x86_64-repo=s' => \$x86_64_repo, + 'ppc64le-repo=s' => \$ppc64le_repo, 'target=s' => \$target, 'nproc=i' => \$nproc, 'parallel-builds=i' => \$parallel_builds, @@ -119,16 +119,16 @@ if ($run_id eq '') { # the two repos (dropping any stale foreign-arch genesis) and re-indexes + re-signs the # affected repomd; it builds nothing and holds no output lock. if ($finalize_xcat_dep) { - die "--finalize-xcat-dep requires --x86-repo and --ppc-repo\n" - if $x86_repo eq '' || $ppc_repo eq ''; + die "--finalize-xcat-dep requires --x86_64-repo and --ppc64le-repo\n" + if $x86_64_repo eq '' || $ppc64le_repo eq ''; require_command('createrepo_c'); require_command('rpm'); require_command('rpmsign') if $gpg_sign; require_command('gpg') if $gpg_sign; - my $x86 = abs_path($x86_repo) or die "--x86-repo '$x86_repo' not found\n"; - my $ppc = abs_path($ppc_repo) or die "--ppc-repo '$ppc_repo' not found\n"; - die "--x86-repo '$x86' is not a directory\n" if !-d $x86; - die "--ppc-repo '$ppc' is not a directory\n" if !-d $ppc; + my $x86 = abs_path($x86_64_repo) or die "--x86_64-repo '$x86_64_repo' not found\n"; + my $ppc = abs_path($ppc64le_repo) or die "--ppc64le-repo '$ppc64le_repo' not found\n"; + die "--x86_64-repo '$x86' is not a directory\n" if !-d $x86; + die "--ppc64le-repo '$ppc' is not a directory\n" if !-d $ppc; finalize_xcat_dep($x86, $ppc); exit 0; } @@ -289,7 +289,8 @@ sub build_one_target { # different targets (e.g. alma+epel-8 vs -9) share build-output/ and # cross-contaminate. Fold the target into run_id so each target gets its own tree. $run_id = "$target-$run_id" unless index($run_id, $target) >= 0; - my ($tfam, $rel, $osdir) = target_osdir($target); + my ($rel) = $target =~ /epel-(\d+)-/; + die "Could not parse EL release from target '$target'\n" unless defined $rel; my $run_root = "$output_root/$run_id"; my $build_root = "$run_root/build-results"; @@ -436,9 +437,7 @@ if (!$skip_build) { # NOTE: this script builds ONLY xcat-dep (its dep packages, the perl packages, and # the OS-dependent xCAT-genesis-base below). The full xCAT core is built separately - # by the xcat-core pipeline -- mockbuild-all no longer has a monolithic core-build - # path (it required perl-generators, which openSUSE Leap does not provide, and did - # not match the Ubuntu build's split design). + # by the xcat-core pipeline -- mockbuild-all no longer has a monolithic core-build path. # xCAT-genesis-base is OS-dependent (its initramfs bundles the build chroot's # kernel + glibc/busybox/perl), so it is built here, per target, and shipped @@ -622,18 +621,7 @@ print "Summary: $summary_file\n" if !$dry_run; print "Tarball: $tarball\n" if !$skip_tarball; print "SRPM Tarball: $srpm_tarball\n" if !$skip_tarball; - return { repo_dir => $repo_dir, rel => $rel, osdir => $osdir, tfam => $tfam }; -} - -# Map a mock target to (family, release, deploy-subdir): -# alma+epel-10-x86_64 -> ('el', '10', 'rh10') -# opensuse-leap-15.6-x86_64 -> ('suse', '15', 'sles15') -# The deploy layout is // for BOTH families (rh yum, sles zypper). -sub target_osdir { - my ($t) = @_; - if (my ($r) = $t =~ /epel-(\d+)-/) { return ('el', $r, "rh$r"); } - if (my ($r) = $t =~ /opensuse-leap-(\d+)\./) { return ('suse', $r, "sles$r"); } - die "Cannot derive OS dir from target '$t' (expected *+epel-N-* or opensuse-leap-N.M-*)\n"; + return { repo_dir => $repo_dir, rel => $rel }; } # Assemble the built per-target repo into the deployable, signed per-EL layout @@ -641,11 +629,9 @@ sub target_osdir { # xcat-dep.repo / mklocalrepo.sh / buildinfo.txt (ready to push to xcat.org). sub deploy_target { my ($tgt, $info) = @_; - my $rel = $info->{rel}; - my $osdir = $info->{osdir}; - my $tfam = $info->{tfam}; + my $rel = $info->{rel}; my $src = $info->{repo_dir}; - my $dest = "$repo_dep/$osdir/$arch"; + my $dest = "$repo_dep/rh$rel/$arch"; print_step("Deploy $tgt -> $dest"); return if $dry_run; make_path($dest); @@ -656,9 +642,9 @@ sub deploy_target { } assert_required_deps($dest); sign_and_index_repo($dest); - write_dep_repo_metadata($dest, $osdir, $tfam); + write_dep_repo_metadata($dest, $rel); my $n = scalar(grep { !/\.src\.rpm$/ } glob("$dest/*.rpm")); - print "Deployed $osdir/$arch: $n rpms\n"; + print "Deployed rh$rel/$arch: $n rpms\n"; } # createrepo_c command with upstream-matching, deterministic metadata. The tool's @@ -690,17 +676,14 @@ sub sign_and_index_repo { } sub write_dep_repo_metadata { - my ($dir, $osdir, $tfam) = @_; - # yum channel for EL (rh), zypper/sles channel for SUSE (sles). - my $baseurl = ($tfam // 'el') eq 'suse' - ? "https://xcat.org/files/xcat/repos/sles/devel/xcat-dep/$osdir/$arch" - : "https://xcat.org/files/xcat/repos/yum/devel/xcat-dep/$osdir/$arch"; + my ($dir, $rel) = @_; + my $baseurl = "https://xcat.org/files/xcat/repos/yum/devel/xcat-dep/rh$rel/$arch"; my $gpgcheck = $gpg_sign ? 1 : 0; my $gpgkey_line = $gpg_sign ? "gpgkey=$baseurl/repodata/repomd.xml.key" : "# gpgkey="; open my $r, '>', "$dir/xcat-dep.repo" or die "Cannot write $dir/xcat-dep.repo: $!\n"; print {$r} <<"EOF"; [xcat-dep] -name=xCAT 2 dependencies ($osdir $arch) +name=xCAT 2 dependencies (rh$rel $arch) baseurl=$baseurl enabled=1 gpgcheck=$gpgcheck @@ -735,7 +718,7 @@ EOS my $release = strftime('snap%Y%m%d%H%M', gmtime($SOURCE_DATE_EPOCH)); open my $b, '>', "$dir/buildinfo.txt" or die "Cannot write $dir/buildinfo.txt: $!\n"; print {$b} <<"EOF"; -TARGET=$osdir/$arch +TARGET=rh$rel/$arch RELEASE=$release BUILD_TIME=$build_time BUILD_MACHINE=$build_machine @@ -748,21 +731,21 @@ EOF # --finalize-xcat-dep: cross-populate the noarch xCAT-genesis-base between each matching # /x86_64 and /ppc64le repo pair, then re-index + re-sign the repos that changed. -# / are the two per-arch repo roots (each holding rh8/rh9/rh10/, -# and possibly sles/). They may be the same path (both arches built into one tree) -# or two separate trees (one per build host); either way pairs are matched by subdir. +# / are the two per-arch repo roots (each holding rh8/rh9/rh10/). +# They may be the same path (both arches built into one tree) or two separate trees (one per +# build host); either way pairs are matched by subdir. sub finalize_xcat_dep { - my ($x86_repo, $ppc_repo) = @_; + my ($x86_64_repo, $ppc64le_repo) = @_; print_step('Finalize xcat-dep: cross-arch genesis-base provisioning (issue #7610)'); - print "x86-repo: $x86_repo\n"; - print "ppc-repo: $ppc_repo\n"; - # Every OS-release dir under the x86 repo that actually has an x86_64 sub-repo. - my @osdirs = grep { -d "$_/x86_64" } glob("$x86_repo/*"); + print "x86_64-repo: $x86_64_repo\n"; + print "ppc64le-repo: $ppc64le_repo\n"; + # Every OS-release dir under the x86_64 repo that actually has an x86_64 sub-repo. + my @osdirs = grep { -d "$_/x86_64" } glob("$x86_64_repo/*"); my $pairs = 0; for my $p (sort @osdirs) { my $osdir = basename($p); - my $x86dir = "$x86_repo/$osdir/x86_64"; - my $ppcdir = "$ppc_repo/$osdir/ppc64le"; + my $x86dir = "$x86_64_repo/$osdir/x86_64"; + my $ppcdir = "$ppc64le_repo/$osdir/ppc64le"; if (!-d $ppcdir) { print "[finalize] $osdir: no ppc64le peer at $ppcdir -- skipping\n"; next; @@ -778,7 +761,7 @@ sub finalize_xcat_dep { $pairs++; } die "FATAL: --finalize-xcat-dep found no /x86_64 + /ppc64le repo pair under\n" - . " --x86-repo '$x86_repo'\n --ppc-repo '$ppc_repo'\n" if $pairs == 0; + . " --x86_64-repo '$x86_64_repo'\n --ppc64le-repo '$ppc64le_repo'\n" if $pairs == 0; print_step('Finalize complete'); } @@ -852,14 +835,15 @@ Options: are assembled + signed here (default: /xcat-dep) --force-unlock Remove a stale /.lock before acquiring it --finalize-xcat-dep Post-build cross-arch genesis mode (builds nothing). Requires - --x86-repo and --ppc-repo. For each matching /x86_64 and + --x86_64-repo and --ppc64le-repo. For each matching /x86_64 and /ppc64le repo pair, copies the noarch xCAT-genesis-base-ppc64 - into the x86_64 repo and xCAT-genesis-base-x86_64 into the ppc64le - repo (dropping any stale foreign-arch genesis), then re-indexes + - re-signs. Restores the 2.17 cross-arch genesis (issue #7610). - Honors --gpg-sign/--gpg-key-name/--gpg-home. Use alone. - --x86-repo PATH (finalize) x86_64 repo root holding /x86_64 (e.g. rh9/x86_64) - --ppc-repo PATH (finalize) ppc64le repo root holding /ppc64le + (the ppc64le genesis; xCAT names it -ppc64 via tarch, no big-endian + code) into the x86_64 repo and xCAT-genesis-base-x86_64 into the + ppc64le repo (dropping any stale foreign-arch genesis), then + re-indexes + re-signs. Restores the 2.17 cross-arch genesis + (issue #7610). Honors --gpg-sign/--gpg-key-name/--gpg-home. Use alone. + --x86_64-repo PATH (finalize) x86_64 repo root holding /x86_64 (e.g. rh9/x86_64) + --ppc64le-repo PATH (finalize) ppc64le repo root holding /ppc64le --gpg-sign Sign rpms + repomd.xml of each per-EL repo --gpg-key-name NAME GPG key name (default: "xCAT Signing Key") --gpg-home PATH GNUPGHOME for signing (default: system keyring) @@ -969,10 +953,14 @@ sub run_build_steps_parallel { my $max_processes = $args{max_processes} // 1; return if !@{$steps}; - # Individual dep-builder failures are TOLERATED (some packages are el-/arch-pinned or - # have dead upstream source URLs, e.g. elilo on el10, perl-Sys-Virt on el8, a moved grub2 - # src.rpm). We collect whatever built and assert the REQUIRED set later (assert_required_deps), - # matching the historical build behaviour. + # Individual dep-builder failures here are TOLERATED only so one flaky builder does not abort + # the others. This is load-bearing, NOT laziness: some builders are expected to fail on a given + # arch/el (e.g. perl-Sys-Virt on el8 -- not a required dep), and some REQUIRED builders "fail" + # cosmetically while still producing their rpm (xCAT-genesis-base: xcat-core buildrpms.pl exits + # non-zero on an unrelated post-build xCAT-release-latest cp, yet the genesis rpm is built). So + # correctness is enforced by RESULT, not exit code: assert_required_deps runs after collection + # and fails the whole run if any REQUIRED rpm is missing -- caught at assert time, not swept + # under the rug. (A blanket "die on any builder failure" reddens the build on these non-issues.) if ($dry_run || $max_processes <= 1 || @{$steps} == 1) { for my $step (@{$steps}) { my $ok = eval { run_step(%{$step}); 1 }; diff --git a/mockbuild-perl-packages.pl b/mockbuild-perl-packages.pl index fddb77e..fba8c62 100755 --- a/mockbuild-perl-packages.pl +++ b/mockbuild-perl-packages.pl @@ -689,59 +689,10 @@ sub create_deterministic_mock_cfg { open my $fh, '>', $cfg_path or die "Cannot write $cfg_path: $!\n"; print $fh "include('/etc/mock/${base_cfg}.cfg')\n"; print $fh "config_opts['environment']['SOURCE_DATE_EPOCH'] = '$epoch'\n"; - # SUSE: the Fedora perl srpms BuildRequire perl-generators / perl-interpreter, which do not exist - # on openSUSE (there `perl` provides the interpreter and rpm generates perl deps itself). Add a - # tiny compat repo whose one package Provides those names (and pulls perl), so `dnf builddep` - # resolves them; the actual build still uses SUSE's perl. No-op on EL/Fedora. - if ($base_cfg =~ /opensuse|sles|suse/i) { - my $repo = suse_buildreq_compat_repo($dir); - print $fh "config_opts['dnf.conf'] += \"\"\"\n[xcat-buildreq-compat]\nname=xcat perl BuildRequires compat\nbaseurl=file://$repo\nenabled=1\ngpgcheck=0\npriority=1\n\"\"\"\n"; - } close $fh; return $cfg_path; } -# Build a noarch rpm that Provides perl-generators + perl-interpreter and put it in a local -# createrepo'd dir; return that dir. The repo is built UNDER the caller's per-package build dir -# (not a shared /tmp path): each build_package fork runs this once for its own package, and -# mockbuild-all gives every (run, target/arch) its own work-dir -- so concurrent perl builds, -# whether parallel packages in one invocation or parallel arch/target invocations, never race on -# a shared path. Idempotent within a dir (reuse if already built). -sub suse_buildreq_compat_repo { - my ($base_dir) = @_; - my $base = "$base_dir/suse-buildreq-compat"; - return "$base/repo" if -f "$base/repo/repodata/repomd.xml"; - my $rpmroot = "$base/rpmbuild"; - File::Path::make_path("$rpmroot/SPECS", "$base/repo"); - my $spec = "$rpmroot/SPECS/xcat-perl-buildreq-compat.spec"; - open my $s, '>', $spec or die "Cannot write $spec: $!\n"; - print {$s} <<'SPEC'; -Name: xcat-perl-buildreq-compat -Version: 1 -Release: 1 -Summary: Build-only compat: provide Fedora perl BuildRequires names on SUSE -License: MIT -BuildArch: noarch -Provides: perl-generators -Provides: perl-interpreter -Provides: perl-devel -Requires: perl -%description -Satisfies the perl-generators / perl-interpreter / perl-devel BuildRequires of -Fedora perl source rpms when building them under openSUSE mock (SUSE ships the -perl dev headers inside the perl package itself). Never shipped. -%files -%changelog -SPEC - close $s; - system("rpmbuild --define " . sh_quote("_topdir $rpmroot") . " -bb " . sh_quote($spec) - . " >/dev/null 2>&1") == 0 or die "FATAL: could not build SUSE buildreq compat rpm\n"; - system("cp " . sh_quote("$rpmroot/RPMS/noarch/") . "xcat-perl-buildreq-compat-*.noarch.rpm " - . sh_quote("$base/repo/") . " && createrepo_c " . sh_quote("$base/repo") . " >/dev/null 2>&1") == 0 - or die "FATAL: could not createrepo the SUSE buildreq compat repo\n"; - return "$base/repo"; -} - sub resolve_source_urls { my ($spec_path) = @_; open my $fh, '<', $spec_path or return (); From 6742e5f8621ad8409862c672428b995a65b5dd5b Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Tue, 21 Jul 2026 22:53:30 -0300 Subject: [PATCH 14/55] fix(elilo): use tracked normalized source read-only to avoid a fetch race elilo/mockbuild.pl wget'd the upstream tarball over $source_path and then normalized it in place, rewriting the tracked source file. Parallel builds share that same source file, so builds running concurrently raced to fetch and rewrite it -- one build could read the file mid-rewrite and get a truncated archive, an intermittent "Normalized source archive still missing elilo top-level tree" failure. Because elilo is a required dep, one such flake failed the whole run. The tracked tarball is already normalized (elilo/ top-level), so use it read-only when normalized and only fetch upstream when it is absent or not yet normalized. This removes the in-place rewrite (hence the race) and the flaky sourceforge dependency, and is more reproducible. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- elilo/mockbuild.pl | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/elilo/mockbuild.pl b/elilo/mockbuild.pl index aee1619..f8d03f4 100755 --- a/elilo/mockbuild.pl +++ b/elilo/mockbuild.pl @@ -90,10 +90,28 @@ make_path($log_dir); print_step("Mock config check"); run("mock -r " . sh_quote($mock_cfg) . $mock_uniqueext_opt . " --print-root-path >/dev/null"); -print_step("Download upstream source"); -run("wget --spider " . sh_quote($source_url)); -run("wget -O " . sh_quote($source_path) . " " . sh_quote($source_url)); -normalize_source_archive($source_path, $version, $work_dir); +print_step("Prepare source archive"); +# The elilo source tarball is tracked in the repo, already normalized to an elilo/ top-level +# tree. Re-downloading + normalizing rewrites $source_path IN PLACE -- and it lives in the +# shared (NFS) checkout that BOTH arch build hosts (x86 + ppc) build against at the same time, +# so the other host's concurrent elilo build can read it mid-rewrite and get a truncated +# archive (intermittent "missing elilo top-level tree" failures). Use the tracked copy +# read-only when it is already normalized; only fetch upstream if it is absent/unnormalized. +my $have_normalized = 0; +if (-f $source_path) { + my $top = capture( + "tar -tzf " . sh_quote($source_path) . + " 2>/dev/null | grep -E '^(\\./)?elilo/' | head -n1 || true" + ); + $have_normalized = 1 if $top ne ''; +} +if ($have_normalized) { + print "Using tracked normalized source archive (no upstream fetch, no shared write): $source_path\n"; +} else { + run("wget --spider " . sh_quote($source_url)); + run("wget -O " . sh_quote($source_path) . " " . sh_quote($source_url)); + normalize_source_archive($source_path, $version, $work_dir); +} print_step("Verify spec assets"); for my $asset (@spec_assets) { From 090aadf86e368165baa0e6f237bc94c86ba00b80 Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Thu, 23 Jul 2026 08:14:01 -0300 Subject: [PATCH 15/55] fix(goconserver): ship server.conf as YAML, not INI The goconserver binary parses /etc/goconserver/server.conf as YAML, but the package shipped it in the old INI ("[server]\nhost = ...") format. The YAML parser reads the leading [server] as a sequence, so the daemon panics at startup ("yaml: cannot unmarshal !!seq into common.ServerConfig"); systemd then rate-limits the service to 'failed' before xCAT (Goconserver.pm) rewrites the config as YAML. On a management node this leaves goconserver down, so the provisioning test cases' makegocons cannot register a console and the case fails. Ship a valid minimal YAML default matching the schema xCAT itself writes (global/api/console; api port 12429, console port 12430, datadir); xCAT still overwrites it with the cert-enabled config on the MN, and this default now only has to parse + start. Bumps release el -> 4. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- goconserver/mockbuild.pl | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/goconserver/mockbuild.pl b/goconserver/mockbuild.pl index 3b5eb15..572743e 100755 --- a/goconserver/mockbuild.pl +++ b/goconserver/mockbuild.pl @@ -171,14 +171,22 @@ StateDirectory=goconserver WantedBy=multi-user.target SERVICE +# The goconserver binary parses server.conf as YAML. Ship a VALID YAML default: the old INI-style +# ([server]\nhost = ...) is read by the YAML parser as a sequence -> `panic: cannot unmarshal !!seq into +# common.ServerConfig` at startup -> systemd rate-limits the service to `failed`. On an xCAT MN, +# xCAT::Goconserver.pm overwrites this with a cert-enabled config, so this default only has to PARSE and +# start (no SSL here -- the xcat certs don't exist until xCAT is configured). Keys/ports mirror the schema +# xCAT itself writes (api 12429, console 12430). write_file("$payload_dir/etc/goconserver/server.conf", <<'CONF'); -[server] -host = 0.0.0.0 -port = 12430 -console_port = 12431 -log_file = /var/log/goconserver/server.log -log_timestamp = true -log_level = info +global: + host: 0.0.0.0 + logfile: /var/log/goconserver/server.log +api: + port: 12429 +console: + datadir: /var/lib/goconserver/ + port: 12430 + log_timestamp: true CONF my $tarball = "$rpmbuild_top/SOURCES/goconserver-$version.tar.gz"; @@ -190,7 +198,7 @@ print_step("Create spec and build RPM"); my $spec_content = <<"SPEC"; Name: goconserver Version: $version -Release: 3.el$rel +Release: 4.el$rel Summary: Console server written in Go for xCAT License: EPL-1.0 URL: https://github.com/xcat2/goconserver @@ -226,6 +234,11 @@ install -m 644 etc/goconserver/server.conf %{buildroot}/etc/goconserver/server.c %dir /var/lib/goconserver %changelog +* Thu Jul 23 2026 xCAT build - 0.3.3-4.el10 +- Ship /etc/goconserver/server.conf in YAML (the format the goconserver binary parses) instead of the + old INI [server] style, which the YAML parser reads as a sequence -> panic (cannot unmarshal !!seq) at + startup -> systemd rate-limits the service to failed before xCAT can convert the config. Fixes console + provisioning (makegocons) on the management node. * Mon Jun 08 2026 xCAT EL10 build - 0.3.3-2.el10 - Replace archived github.com/kr/pty with github.com/creack/pty to fix "Setctty set but Ctty not valid in child" console fork failure on modern Go. From c70e09ceeb6739df898dc58bbc629fba724c688d Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Sun, 26 Jul 2026 21:15:39 -0300 Subject: [PATCH 16/55] fix(build): stop concurrent per-arch builds racing on shared package sources The elilo, ipmitool and syslinux builders each fetched their upstream source and rewrote the tracked source tarball in place, inside the package source directory that both arch builds share. When the two per-arch builds run in parallel they were racing to fetch the source: one build truncated and rewrote the tarball while the other read it, so the reader got a truncated archive and failed intermittently with "missing top-level tree" errors. The correct, normalized source is already tracked in the repository and is what mock consumes, so the fetch is redundant as well as unsafe. Drop the download/normalize entirely and verify the tracked source read-only (it exists and has the expected top-level tree). With no writer, the shared source is only ever read, so parallel per-arch builds can no longer race on it. Also removes the now-dead --source-url / --skip-upstream-download options, the normalize helper, and the wget dependency check. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- elilo/mockbuild.pl | 88 +++++++++---------------------------------- ipmitool/mockbuild.pl | 60 +++++++++-------------------- syslinux/mockbuild.pl | 41 +++++++++----------- 3 files changed, 53 insertions(+), 136 deletions(-) diff --git a/elilo/mockbuild.pl b/elilo/mockbuild.pl index f8d03f4..6e4aa32 100755 --- a/elilo/mockbuild.pl +++ b/elilo/mockbuild.pl @@ -13,7 +13,6 @@ my $repo_root = abs_path("$script_dir/.."); my $pkg_dir = "$repo_root/elilo"; my $spec_file = "$pkg_dir/elilo-xcat.spec"; -my $source_url = 'https://downloads.sourceforge.net/project/elilo/elilo/elilo-3.14/elilo-3.14-all.tar.gz'; my $source_file = ''; my $work_dir = '/tmp/elilo-xcat-mockbuild'; my $mock_cfg = ''; @@ -24,7 +23,6 @@ my $skip_install = 0; my $build_timestamp; GetOptions( - 'source-url=s' => \$source_url, 'source-file=s' => \$source_file, 'work-dir=s' => \$work_dir, 'mock-cfg=s' => \$mock_cfg, @@ -38,7 +36,7 @@ GetOptions( die "Run as root (current uid=$>)\n" if $> != 0; die "Missing spec file: $spec_file\n" if !-f $spec_file; -for my $bin (qw(wget mock rpmbuild rpm dnf file bash grep)) { +for my $bin (qw(mock rpmbuild rpm dnf file bash grep)) { run("command -v " . sh_quote($bin) . " >/dev/null 2>&1"); } @@ -80,7 +78,6 @@ print "result_dir: $result_dir\n"; print "log_dir: $log_dir\n"; print "mock_cfg: $mock_cfg\n"; print "mock_uniqueext: " . ($mock_uniqueext ne '' ? $mock_uniqueext : '(none)') . "\n"; -print "source_url: $source_url\n"; print "source_file:$source_file\n"; print "skip_install: $skip_install\n"; @@ -90,28 +87,23 @@ make_path($log_dir); print_step("Mock config check"); run("mock -r " . sh_quote($mock_cfg) . $mock_uniqueext_opt . " --print-root-path >/dev/null"); -print_step("Prepare source archive"); -# The elilo source tarball is tracked in the repo, already normalized to an elilo/ top-level -# tree. Re-downloading + normalizing rewrites $source_path IN PLACE -- and it lives in the -# shared (NFS) checkout that BOTH arch build hosts (x86 + ppc) build against at the same time, -# so the other host's concurrent elilo build can read it mid-rewrite and get a truncated -# archive (intermittent "missing elilo top-level tree" failures). Use the tracked copy -# read-only when it is already normalized; only fetch upstream if it is absent/unnormalized. -my $have_normalized = 0; -if (-f $source_path) { - my $top = capture( - "tar -tzf " . sh_quote($source_path) . - " 2>/dev/null | grep -E '^(\\./)?elilo/' | head -n1 || true" - ); - $have_normalized = 1 if $top ne ''; -} -if ($have_normalized) { - print "Using tracked normalized source archive (no upstream fetch, no shared write): $source_path\n"; -} else { - run("wget --spider " . sh_quote($source_url)); - run("wget -O " . sh_quote($source_path) . " " . sh_quote($source_url)); - normalize_source_archive($source_path, $version, $work_dir); -} +print_step("Verify tracked source archive"); +# Source0 (elilo--source.tar.gz) is tracked in the repo, already normalized to an elilo/ +# top-level tree, and consumed directly by mock (--sources $pkg_dir below). There is nothing to +# download: the old fetch re-derived this SAME tracked file and rewrote it IN PLACE. Because the +# checkout is on a shared (NFS) mount that BOTH arch build hosts (x86 + ppc) build against at the +# same time, that in-place rewrite raced the other host's concurrent elilo build -- it could read +# the file mid-write and get a truncated archive ("missing elilo top-level tree" failures). We now +# only READ the tracked file, so concurrent builds can never race on it. Fail loudly (do NOT +# silently re-fetch) if the checkout is missing/broken -- that is repo corruption, not a fetch miss. +die "Tracked elilo source missing: $source_path (incomplete checkout?)\n" if !-f $source_path; +my $top = capture( + "tar -tzf " . sh_quote($source_path) . + " 2>/dev/null | grep -E '^(\\./)?elilo/' | head -n1 || true" +); +die "Tracked elilo source is not normalized (no elilo/ top-level tree): $source_path\n" + if $top eq ''; +print "Using tracked normalized source archive (read-only, no fetch, no shared write): $source_path\n"; print_step("Verify spec assets"); for my $asset (@spec_assets) { @@ -253,7 +245,6 @@ exit 0; sub usage { return <<"USAGE"; Usage: $0 [options] - --source-url URL Upstream tarball URL (default: $source_url) --source-file FILE Source filename stored in elilo/ (default: inferred from spec version) --work-dir PATH Temporary work dir (default: $work_dir) --mock-cfg NAME Mock config (default: +epel-10-) @@ -302,49 +293,6 @@ sub parse_spec { return ($version, @assets); } -sub normalize_source_archive { - my ($archive, $version, $work_base) = @_; - - my $has_elilo = capture( - "tar -tzf " . sh_quote($archive) . - " | grep -E '^(\\./)?elilo/' | head -n1 || true" - ); - return if $has_elilo ne ''; - - my $nested = capture( - "tar -tzf " . sh_quote($archive) . - " | grep -E '^(\\./)?elilo-$version-source\\.tar\\.gz\$' | head -n1 || true" - ); - die "Downloaded archive does not contain elilo source payload: $archive\n" - if $nested eq ''; - - my $normalize_dir = "$work_base/source-normalize"; - remove_tree($normalize_dir) if -d $normalize_dir; - make_path($normalize_dir); - - run( - "tar -xzf " . sh_quote($archive) . - " -C " . sh_quote($normalize_dir) . - " " . sh_quote($nested) - ); - - my $nested_rel = $nested; - $nested_rel =~ s{^\./}{}; - my $nested_path = "$normalize_dir/$nested_rel"; - die "Failed to extract nested source archive: $nested_path\n" - if !-f $nested_path; - - copy($nested_path, $archive) - or die "Failed to normalize source archive $archive: $!\n"; - - my $recheck = capture( - "tar -tzf " . sh_quote($archive) . - " | grep -E '^(\\./)?elilo/' | head -n1 || true" - ); - die "Normalized source archive still missing elilo top-level tree: $archive\n" - if $recheck eq ''; -} - sub print_step { my ($msg) = @_; print "\n== $msg ==\n"; diff --git a/ipmitool/mockbuild.pl b/ipmitool/mockbuild.pl index 3afcb26..dbd9241 100755 --- a/ipmitool/mockbuild.pl +++ b/ipmitool/mockbuild.pl @@ -3,7 +3,7 @@ use strict; use warnings; use Cwd qw(abs_path); -use File::Basename qw(dirname basename); +use File::Basename qw(dirname); use File::Copy qw(copy); use File::Path qw(make_path remove_tree); use Getopt::Long qw(GetOptions); @@ -13,7 +13,6 @@ my $repo_root = abs_path("$script_dir/.."); my $pkg_dir = "$repo_root/ipmitool"; my $spec_file = "$pkg_dir/ipmitool.spec"; -my $source_url = 'https://github.com/ipmitool/ipmitool/archive/refs/tags/IPMITOOL_1_8_18.tar.gz'; my $source_file = ''; my $work_dir = '/tmp/ipmitool-xcat-mockbuild'; my $mock_cfg = ''; @@ -24,7 +23,6 @@ my $skip_install = 0; my $build_timestamp; GetOptions( - 'source-url=s' => \$source_url, 'source-file=s' => \$source_file, 'work-dir=s' => \$work_dir, 'mock-cfg=s' => \$mock_cfg, @@ -38,7 +36,7 @@ GetOptions( die "Run as root (current uid=$>)\n" if $> != 0; die "Missing spec file: $spec_file\n" if !-f $spec_file; -for my $bin (qw(wget mock rpmbuild rpm dnf ldd bash)) { +for my $bin (qw(mock rpmbuild rpm dnf ldd bash)) { run("command -v " . sh_quote($bin) . " >/dev/null 2>&1"); } @@ -80,7 +78,6 @@ print "result_dir: $result_dir\n"; print "log_dir: $log_dir\n"; print "mock_cfg: $mock_cfg\n"; print "mock_uniqueext: " . ($mock_uniqueext ne '' ? $mock_uniqueext : '(none)') . "\n"; -print "source_url: $source_url\n"; print "source_file:$source_file\n"; print "skip_install: $skip_install\n"; print "SOURCE_DATE_EPOCH: $SOURCE_DATE_EPOCH\n"; @@ -91,10 +88,22 @@ make_path($log_dir); print_step("Mock config check"); run("mock -r " . sh_quote($mock_cfg) . $mock_uniqueext_opt . " --print-root-path >/dev/null"); -print_step("Download upstream source"); -run("wget --spider " . sh_quote($source_url)); -run("wget -O " . sh_quote($source_path) . " " . sh_quote($source_url)); -normalize_source_archive($source_path, $version, $work_dir); +print_step("Verify tracked source archive"); +# The ipmitool source (ipmitool-.tar.gz) is tracked in the repo, already normalized to the +# ipmitool-/ top-level that %setup -n expects, and consumed directly by mock (--sources +# $pkg_dir below). There is nothing to download: the old fetch re-derived this SAME tracked file +# and rewrote it IN PLACE, and the checkout is shared between the two arch build hosts building at +# once -- so the in-place rewrite raced the other host's concurrent ipmitool build, which could +# read the file mid-write and get a truncated archive. We only READ it now, so concurrent builds +# can never race on it. Fail loudly (do NOT silently re-fetch) if the checkout is missing/broken. +die "Tracked ipmitool source missing: $source_path (incomplete checkout?)\n" if !-f $source_path; +my $top = capture( + "tar -tzf " . sh_quote($source_path) . + " 2>/dev/null | grep -E '^(\\./)?ipmitool-$version/' | head -n1 || true" +); +die "Tracked ipmitool source is not the expected ipmitool-$version/ tree: $source_path\n" + if $top eq ''; +print "Using tracked source archive (read-only, no fetch, no shared write): $source_path\n"; print_step("Verify spec assets"); for my $asset (@spec_assets) { @@ -258,7 +267,6 @@ exit 0; sub usage { return <<"USAGE"; Usage: $0 [options] - --source-url URL Upstream tarball URL (default: $source_url) --source-file FILE Source filename stored in ipmitool/ (default: inferred from spec version) --work-dir PATH Temporary work dir (default: $work_dir) --mock-cfg NAME Mock config (default: +epel-10-) @@ -308,38 +316,6 @@ sub parse_spec { return ($version, @assets); } -sub normalize_source_archive { - my ($archive, $version, $work_base) = @_; - - my $normalize_dir = "$work_base/source-normalize"; - remove_tree($normalize_dir) if -d $normalize_dir; - make_path($normalize_dir); - - run("tar -xzf " . sh_quote($archive) . " -C " . sh_quote($normalize_dir)); - - my @entries = grep { $_ !~ m{/\.\.?$} } glob("$normalize_dir/*"); - die "Unexpected archive layout in $archive\n" if @entries != 1; - my $top_path = $entries[0]; - die "Unexpected non-directory top-level entry in $archive: $top_path\n" - if !-d $top_path; - - my $expected_top = "ipmitool-$version"; - my $actual_top = basename($top_path); - if ($actual_top ne $expected_top) { - my $new_path = "$normalize_dir/$expected_top"; - run("rm -rf " . sh_quote($new_path)); - run("mv " . sh_quote($top_path) . " " . sh_quote($new_path)); - } - - # Repack using the expected top-level directory required by the spec. - run( - "tar --sort=name --owner=0 --group=0 --mtime=\@$SOURCE_DATE_EPOCH" . - " -C " . sh_quote($normalize_dir) . - " -czf " . sh_quote($archive) . - " " . sh_quote($expected_top) - ); -} - sub print_step { my ($msg) = @_; print "\n== $msg ==\n"; diff --git a/syslinux/mockbuild.pl b/syslinux/mockbuild.pl index cc320c1..d7bd499 100755 --- a/syslinux/mockbuild.pl +++ b/syslinux/mockbuild.pl @@ -13,7 +13,6 @@ my $repo_root = abs_path("$script_dir/.."); my $pkg_dir = "$repo_root/syslinux"; my $spec_file = "$pkg_dir/syslinux-xcat.spec"; -my $source_url = 'https://www.kernel.org/pub/linux/utils/boot/syslinux/syslinux-6.03.tar.xz'; my $source_file = ''; my $work_dir = '/tmp/syslinux-xcat-mockbuild'; my $mock_cfg = ''; @@ -21,11 +20,9 @@ my $mock_uniqueext = ''; my $result_dir = "$repo_root/build-output/list3/syslinux-xcat"; my $log_dir = "$repo_root/build-logs/list3/syslinux-xcat"; my $skip_install = 0; -my $skip_upstream_download = 0; my $build_timestamp; GetOptions( - 'source-url=s' => \$source_url, 'source-file=s' => \$source_file, 'work-dir=s' => \$work_dir, 'mock-cfg=s' => \$mock_cfg, @@ -33,14 +30,13 @@ GetOptions( 'result-dir=s' => \$result_dir, 'log-dir=s' => \$log_dir, 'skip-install!' => \$skip_install, - 'skip-upstream-download!' => \$skip_upstream_download, 'build-timestamp=i' => \$build_timestamp, ) or die usage(); die "Run as root (current uid=$>)\n" if $> != 0; die "Missing spec file: $spec_file\n" if !-f $spec_file; -for my $bin (qw(wget mock rpmbuild rpm dnf file bash grep cut)) { +for my $bin (qw(mock rpmbuild rpm dnf file bash grep)) { run("command -v " . sh_quote($bin) . " >/dev/null 2>&1"); } @@ -90,10 +86,8 @@ print "pkg_name: $pkg_name\n"; print "version: $version\n"; print "mock_cfg: $mock_cfg\n"; print "mock_uniqueext: " . ($mock_uniqueext ne '' ? $mock_uniqueext : '(none)') . "\n"; -print "source_url: $source_url\n"; print "source_file:$source_file\n"; print "skip_install: $skip_install\n"; -print "skip_upstream_download: $skip_upstream_download\n"; make_path($result_dir); make_path($log_dir); @@ -101,21 +95,22 @@ make_path($log_dir); print_step("Mock config check"); run("mock -r " . sh_quote($mock_cfg) . $mock_uniqueext_opt . " --print-root-path >/dev/null"); -if (!$skip_upstream_download) { - print_step("Download upstream source"); - run("wget --spider " . sh_quote($source_url)); - run("wget -O " . sh_quote($source_path) . " " . sh_quote($source_url)); - - my $sha = capture("sha256sum " . sh_quote($source_path) . " | cut -d ' ' -f1"); - my $meta_file = "$log_dir/upstream-source.txt"; - open my $mfh, '>', $meta_file or die "Cannot write $meta_file: $!\n"; - print {$mfh} "url=$source_url\n"; - print {$mfh} "file=$source_path\n"; - print {$mfh} "sha256=$sha\n"; - close $mfh; - print "Downloaded source: $source_path\n"; - print "SHA256: $sha\n"; -} +print_step("Verify tracked source archive"); +# The syslinux source (syslinux-.tar.xz, Source0) is tracked in the repo, has the +# syslinux-/ top-level that %setup -n expects, and is consumed directly by mock (--sources +# $pkg_dir below). There is nothing to download: the old fetch re-downloaded this SAME tracked file +# and rewrote it IN PLACE, and the checkout is shared between the two arch build hosts building at +# once -- so the in-place rewrite raced the other host's concurrent syslinux build, which could +# read the file mid-write and get a truncated archive. We only READ it now, so concurrent builds +# can never race on it. Fail loudly (do NOT silently re-fetch) if the checkout is missing/broken. +die "Tracked syslinux source missing: $source_path (incomplete checkout?)\n" if !-f $source_path; +my $top = capture( + "tar -tf " . sh_quote($source_path) . + " 2>/dev/null | grep -E '^(\\./)?syslinux-' | head -n1 || true" +); +die "Tracked syslinux source is not a syslinux-*/ source tree: $source_path\n" + if $top eq ''; +print "Using tracked source archive (read-only, no fetch, no shared write): $source_path\n"; print_step("Verify spec assets"); for my $asset (@all_assets) { @@ -280,7 +275,6 @@ exit 0; sub usage { return <<"USAGE"; Usage: $0 [options] - --source-url URL Upstream tarball URL (default: $source_url) --source-file FILE Source filename stored in syslinux/ (default: inferred from spec) --work-dir PATH Temporary work dir (default: $work_dir) --mock-cfg NAME Mock config (default: +epel-10-) @@ -288,7 +282,6 @@ Usage: $0 [options] --result-dir PATH Output RPM/SRPM directory (default: $result_dir) --log-dir PATH Log directory (default: $log_dir) --build-timestamp EPOCH SOURCE_DATE_EPOCH for deterministic builds - --skip-upstream-download Skip wget download step --skip-install Skip dnf install + smoke tests USAGE } From 81c5a27eb8d82a8e6a5eec8a806500d2fcb67d51 Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Tue, 28 Jul 2026 07:22:59 -0300 Subject: [PATCH 17/55] fix(xcat-dep): scrub mock chroots after each build to stop /var/lib/mock leak The EL build orchestrator never reclaimed the per-step mock buildroots it created. Each build step makes a build chroot AND a per-uniqueext bootstrap chroot under /var/lib/mock (dep packages, per-package perl chroots, and xCAT-genesis-base); the child builders copy their RPMs/logs to their --result-dir and exit without scrubbing, and mock's own cleanup leaves them behind (and keeps them entirely on failure). So /var/lib/mock grew ~15-17G per run, unbounded, until the build host filled to 99% and every mock dnf transaction failed for lack of space ("Error: needs N MB more space on the / filesystem", rc=30), cascading to rc=2/rc=255 across packages and killing otherwise-healthy builds. After the parallel build phase, scrub each step's buildroot with "mock -r --uniqueext --scrub=chroot --scrub=bootstrap" -- mock's own lock-safe scrub: a chroot still held by a concurrent build is refused and skipped (never rm, which would race a live build). Both the build chroot and its per-uniqueext bootstrap are removed (each is per-uniqueext, so both leak); the shared root cache under /var/cache/mock is kept so rebuilds stay fast. The orchestrator scrubs the dep-package and genesis chroots (whose uniqueext it assigns); mockbuild-perl-packages.pl scrubs each of its per-package chroots (it derives its own per-package uniqueext). Scrubs are non-fatal -- a cleanup hiccup never fails the build. --keep-buildroots preserves the buildroots for debugging. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- BUILD.md | 23 ++++++++++++++----- mockbuild-all.pl | 47 ++++++++++++++++++++++++++++++++++++++ mockbuild-perl-packages.pl | 13 +++++++++++ 3 files changed, 77 insertions(+), 6 deletions(-) diff --git a/BUILD.md b/BUILD.md index d386be7..062c308 100644 --- a/BUILD.md +++ b/BUILD.md @@ -59,12 +59,13 @@ Each build path uses `mock` for chroot isolation. Top-level steps are paralleliz 1. Optional chroot cleanup (`--scrub-all-chroots`) 2. Parallel build execution -3. Optional install/smoke checks inside child builders (disabled with `--skip-install`) -4. Binary RPM collection into `repo//` -5. Source RPM collection into `repo-src/` -6. `createrepo --update` on both repo trees -7. Tarball creation for both repo trees -8. Summary generation (`summary.txt`) +3. Post-build chroot scrub — reclaims each build step's mock chroot (unless `--keep-buildroots`) +4. Optional install/smoke checks inside child builders (disabled with `--skip-install`) +5. Binary RPM collection into `repo//` +6. Source RPM collection into `repo-src/` +7. `createrepo --update` on both repo trees +8. Tarball creation for both repo trees +9. Summary generation (`summary.txt`) # Skip and Control Flags @@ -86,6 +87,16 @@ Use these flags to skip specific operations: - Skips tarball creation for both binary and SRPM repos. - `--scrub-all-chroots` - Runs `mock -r --scrub=all` before build and collection. +- `--keep-buildroots` + - Keeps each build step's mock chroot after the build instead of scrubbing it. By default, + after the parallel build phase every step's buildroot (dep packages, the per-package perl + chroots, and `xCAT-genesis-base`) is reclaimed with + `mock -r --uniqueext --scrub=chroot --scrub=bootstrap` — a lock-safe scrub (a + chroot still held by a concurrent build is refused and skipped). Both the build chroot and its + per-uniqueext bootstrap chroot are removed (each build step gets its own bootstrap, so both + must go); the shared root cache under `/var/cache/mock` is kept so rebuilds stay fast. This + stops `/var/lib/mock` from growing unbounded across runs. Pass `--keep-buildroots` to preserve + a buildroot for debugging a failed build. - `--collect-dir ` - Adds extra artifact roots to the collection phase (repeatable). - `--dry-run` diff --git a/mockbuild-all.pl b/mockbuild-all.pl index 44c817f..4754dbc 100755 --- a/mockbuild-all.pl +++ b/mockbuild-all.pl @@ -41,6 +41,7 @@ my $skip_genesis = 0; my $skip_createrepo = 0; my $skip_tarball = 0; my $scrub_all_chroots = 0; +my $keep_buildroots = 0; # keep per-step mock chroots after build (default: --scrub=chroot each) my $dry_run = 0; my @extra_collect_dirs; my $repo_dep = ''; @@ -85,6 +86,7 @@ GetOptions( 'skip-createrepo!' => \$skip_createrepo, 'skip-tarball!' => \$skip_tarball, 'scrub-all-chroots!' => \$scrub_all_chroots, + 'keep-buildroots!' => \$keep_buildroots, 'collect-dir=s@' => \@extra_collect_dirs, 'dry-run!' => \$dry_run, ) or die usage(); @@ -356,6 +358,7 @@ print "skip_install: $skip_install\n"; print "skip_createrepo: $skip_createrepo\n"; print "skip_tarball: $skip_tarball\n"; print "scrub_all_chroots:$scrub_all_chroots\n"; +print "keep_buildroots: $keep_buildroots\n"; print "dry_run: $dry_run\n"; print "perl_builder: $perl_builder\n"; print "tarball: $tarball\n"; @@ -399,6 +402,8 @@ if (!$skip_build) { step => "Build xcat-dep: $name", cmd => $cmd, log => "$log_root/$name/run.log", + scrub_cfg => $target, + scrub_uniqueext => $step_uniqueext, }; push @collect_roots, $step_result; } @@ -425,6 +430,7 @@ if (!$skip_build) { # committed .src.rpm, so hand the suffix down for the builder to re-stamp them. ($RELEASE_BUMP ne '' ? ('--release-suffix', sh_quote($RELEASE_BUMP)) : ()), ($skip_install ? '--skip-install' : ()), + ($keep_buildroots ? '--keep-buildroots' : ()), ); push @build_steps, { id => 'perl', @@ -468,6 +474,7 @@ if (!$skip_build) { cmd => $cmd, cwd => $xcat_src, log => "$log_root/genesis-build.log", + scrub_cfg => "xCAT-genesis-base-$target", }; } @@ -482,6 +489,23 @@ if (!$skip_build) { steps => \@build_steps, max_processes => $effective_parallel_builds, ); + + # Reclaim each build step's mock chroot now that the step copied its RPMs/logs out to + # its --result-dir (collect_rpms reads those, never /var/lib/mock). mock's own cleanup + # leaves these chroots behind -- and keeps them entirely on failure -- so /var/lib/mock + # grows ~15-17G per run until the host fills and every dnf transaction fails for lack of + # space. Scrub each via `mock --scrub=chroot --scrub=bootstrap` (never rm): it takes the + # chroot lock, so a chroot still used by a concurrent build is refused and safely skipped. + # Both the build chroot and its per-uniqueext bootstrap are removed; the root cache stays + # for fast rebuilds. Perl packages are scrubbed inside mockbuild-perl-packages.pl (it + # derives its own per-package uniqueexts). + unless ($keep_buildroots) { + for my $s (@build_steps) { + next unless defined $s->{scrub_cfg}; + (my $slug = $s->{id}) =~ s/[^\w.-]+/-/g; + scrub_buildroot($s->{scrub_cfg}, $s->{scrub_uniqueext}, "$log_root/scrub-$slug.log"); + } + } } } @@ -947,6 +971,29 @@ sub run_step { } } +# Scrub a single mock buildroot via mock's own lock-safe --scrub. Never rm: if a concurrent build +# still holds the chroot lock, mock refuses and we skip it. Failures (already scrubbed, locked, or +# config missing) are tolerated -- a cleanup hiccup must never fail the build. Scrubs both the +# build chroot and its per-uniqueext bootstrap chroot (each build step gets its own bootstrap, so +# both must go or /var/lib/mock still leaks). The shared root cache under /var/cache/mock is kept, +# so rebuilds stay fast. $uniqueext is optional (genesis has none). +sub scrub_buildroot { + my ($cfg, $uniqueext, $log) = @_; + return if !defined $cfg || $cfg eq ''; + my $ext = (defined $uniqueext && $uniqueext ne '') + ? ' --uniqueext ' . sh_quote($uniqueext) : ''; + eval { + run_step( + step => "Scrub chroot $cfg$ext", + cmd => "mock -r " . sh_quote($cfg) . $ext . " --scrub=chroot --scrub=bootstrap", + log => $log, + ); + 1; + } or do { + warn "WARN: chroot scrub failed (tolerated) for $cfg$ext: $@"; + }; +} + sub run_build_steps_parallel { my (%args) = @_; my $steps = $args{steps} // []; diff --git a/mockbuild-perl-packages.pl b/mockbuild-perl-packages.pl index fba8c62..c31aac9 100755 --- a/mockbuild-perl-packages.pl +++ b/mockbuild-perl-packages.pl @@ -13,6 +13,7 @@ my $repo_root = abs_path(dirname(__FILE__)); my $work_dir = '/tmp/perl-list6-mockbuild'; my $mock_cfg = ''; my $mock_uniqueext = ''; +my $keep_buildroots = 0; my $result_dir = ''; my $log_dir = ''; my $packages_csv = ''; @@ -29,6 +30,7 @@ GetOptions( 'work-dir=s' => \$work_dir, 'mock-cfg=s' => \$mock_cfg, 'mock-uniqueext=s' => \$mock_uniqueext, + 'keep-buildroots!' => \$keep_buildroots, 'result-dir=s' => \$result_dir, 'log-dir=s' => \$log_dir, 'packages=s' => \$packages_csv, @@ -230,6 +232,17 @@ for my $idx (0 .. $#packages) { allow_erasing => $allow_erasing, release_suffix => $release_suffix, ); + unless ($keep_buildroots) { + # Reclaim this perl package's chroot AND its per-uniqueext bootstrap via mock's lock-safe + # --scrub (never rm). Each package has its own uniqueext (hence its own chroot+bootstrap), + # so scrubbing one never touches a sibling's concurrent build. Runs regardless of build + # result so failed chroots are reclaimed too; the root cache is kept for fast rebuilds. + # The orchestrator can't name these chroots (package_uniqueext derives them), so we scrub + # here. + (my $ps = $pkg) =~ s/[^\w.-]+/-/g; + system("mock -r " . sh_quote($mock_cfg) . " --uniqueext " . sh_quote($pkg_uniqueext) + . " --scrub=chroot --scrub=bootstrap > " . sh_quote("$log_dir/scrub-$ps.log") . " 2>&1"); + } $pm->finish($ok ? 0 : 1); } $pm->wait_all_children; From 657da58934ae21f68f40175b9dbaf2ab395687f2 Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Tue, 28 Jul 2026 11:19:37 -0300 Subject: [PATCH 18/55] feat(xcat-dep): build a per-target required-package manifest, fail on any failure Until now mockbuild-all.pl built every dep package + every perl package + genesis on every target, and TOLERATED build-step failures: a builder "expected to fail on a given arch/el" (e.g. perl-Sys-Virt on el8) was warned and swept under the rug, with correctness only re-checked after the fact by a hardcoded assert_required_deps set. That hid real failures until the post-collection gate and shipped packages a target does not need. Replace that with an explicit, empirically-derived manifest. package-manifest.conf has one [] section per (EL, arch) listing =; each target builds ONLY the packages listed for it. The sets were derived authoritatively -- on a clean MN of each of the six targets, xcat.org LATEST xcat-core + xcat-dep were configured, `dnf install xCAT` was run, and the packages whose from_repo=xcat-dep were captured. That is exactly what xCAT pulls from xcat-dep on that target. Results: conserver-xcat is required by no target (goconserver supersedes it), and the variable perl modules differ per EL because the OS/EPEL already provides the rest there. Build failures are no longer tolerated: run_build_steps_parallel now returns the failed step ids and build_one_target fails the whole run if any required package failed. The one exception is xCAT-genesis-base -- xcat-core's buildrpms.pl exits non-zero on an unrelated post-build xCAT-release-latest cp even when the genesis rpm IS produced, so genesis is judged by rpm-produced, not exit code. mockbuild-perl-packages.pl already honors --packages; the orchestrator now passes each target's required perl subset so only those are built. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- BUILD.md | 16 ++++++- mockbuild-all.pl | 83 ++++++++++++++++++++++++++++------- packages-manifest.conf | 98 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 180 insertions(+), 17 deletions(-) create mode 100644 packages-manifest.conf diff --git a/BUILD.md b/BUILD.md index 062c308..4e882b9 100644 --- a/BUILD.md +++ b/BUILD.md @@ -55,10 +55,24 @@ This guide uses the following placeholders consistently: Each build path uses `mock` for chroot isolation. Top-level steps are parallelized by `mockbuild-all.pl`, and perl dependency builds are also parallelized internally by `mockbuild-perl-packages.pl`. +## Per-target package manifest + +`packages-manifest.conf` (repo root) declares, per target, exactly which packages are required — +one `[]` section (matching `--target`, e.g. `[alma+epel-10-x86_64]`) of +`=` lines. For each target, `mockbuild-all.pl` builds **only** the packages +listed for it; a package absent from a target's section is not built for that target (e.g. +`conserver-xcat` is not required by any target, and the per-EL perl set differs because the OS/ +EPEL already provides some modules). The lists were derived empirically — on a clean MN of each +(EL, arch), `dnf install xCAT` from xcat.org latest, and the packages whose `from_repo=xcat-dep` +are exactly the required set. See the file header for details. + +Build failures are **not tolerated**: any required (manifest) package that fails to build fails +the whole run. + `mockbuild-all.pl` does more than building RPMs. In a default run it performs these stages: 1. Optional chroot cleanup (`--scrub-all-chroots`) -2. Parallel build execution +2. Parallel build execution — only the target's manifest packages; any failure fails the run 3. Post-build chroot scrub — reclaims each build step's mock chroot (unless `--keep-buildroots`) 4. Optional install/smoke checks inside child builders (disabled with `--skip-install`) 5. Binary RPM collection into `repo//` diff --git a/mockbuild-all.pl b/mockbuild-all.pl index 4754dbc..f1651ae 100755 --- a/mockbuild-all.pl +++ b/mockbuild-all.pl @@ -294,6 +294,13 @@ sub build_one_target { my ($rel) = $target =~ /epel-(\d+)-/; die "Could not parse EL release from target '$target'\n" unless defined $rel; + # Per-target required set from packages-manifest.conf: build ONLY these packages, and fail the + # run if any of them fails. A package absent from this target's section is not built for it. + my %MANIFEST = read_manifest("$repo_root/packages-manifest.conf"); + my %req = %{ $MANIFEST{$target} // {} }; + die "FATAL: no manifest section for target '$target' in packages-manifest.conf\n" + if !$skip_build && !%req; + my $run_root = "$output_root/$run_id"; my $build_root = "$run_root/build-results"; my $log_root = "$run_root/build-logs"; @@ -381,6 +388,7 @@ if (!$skip_build) { if (!$skip_xcat_dep) { for my $builder (@active_dep_builders) { + next unless $req{ $builder->{name} }; # manifest: build only required dep packages my $name = $builder->{name}; my $script = $builder->{script}; my $step_result = "$build_root/$name"; @@ -409,7 +417,8 @@ if (!$skip_build) { } } - if (!$skip_perl) { + my @perl_pkgs = sort grep { /^perl-/ } keys %req; # manifest: perl packages required here + if (!$skip_perl && @perl_pkgs) { my $perl_result = "$build_root/perl/$arch"; my $perl_log = "$log_root/perl/$arch"; my $perl_uniqueext = build_mock_uniqueext($run_id, ++$build_step_seq, 'perl-list6'); @@ -423,6 +432,7 @@ if (!$skip_build) { '--result-dir', sh_quote($perl_result), '--log-dir', sh_quote($perl_log), '--work-dir', sh_quote("/tmp/mockbuild-all-$run_id/perl-list6"), + '--packages', sh_quote(join(',', @perl_pkgs)), # manifest: only required perl pkgs (($max_build_workers && $max_build_workers >= 1) ? ('--jobs', $max_build_workers) : ()), '--build-timestamp', $SOURCE_DATE_EPOCH, # CD bump: the in-tree spec Release bump above only reaches the spec-mode perl @@ -451,7 +461,7 @@ if (!$skip_build) { # (run in the xcat-core dir) derives the same snapYYYYMMDDHHMM Release from # xcat-core's Gitepoch, so it matches xCAT-genesis-scripts (built in core) and # the exact-version dependency genesis-scripts -> genesis-base resolves. - if (!$skip_genesis) { + if (!$skip_genesis && $req{'xCAT-genesis-base'}) { # buildrpms.pl stages sources in $HOME/rpmbuild (via rpmdev-setuptree). Give each # per-target genesis build its own HOME so parallel EL targets don't race on the shared # /root/rpmbuild tree (that race is what made concurrent genesis builds fail). @@ -485,7 +495,7 @@ if (!$skip_build) { ($max_build_workers && $max_build_workers >= 1) ? $max_build_workers : defined($parallel_builds) ? $parallel_builds : scalar(@build_steps); - run_build_steps_parallel( + my @failed = run_build_steps_parallel( steps => \@build_steps, max_processes => $effective_parallel_builds, ); @@ -506,6 +516,23 @@ if (!$skip_build) { scrub_buildroot($s->{scrub_cfg}, $s->{scrub_uniqueext}, "$log_root/scrub-$slug.log"); } } + + # Zero-tolerance: any required (manifest) package that failed to build fails the run. + # genesis is the one exception -- xcat-core's buildrpms.pl exits non-zero on an unrelated + # post-build xCAT-release-latest cp even when the genesis rpm IS produced, so genesis + # counts as failed only if its rpm is absent, not on exit code. + my @hard; + for my $id (@failed) { + if ($id eq 'genesis') { + my @g = grep { !/\.src\.rpm$/ } + glob("$xcat_src/dist/$target/rpms/xCAT-genesis-base-*.rpm"); + push @hard, $id unless @g; + } + else { + push @hard, $id; + } + } + die "FATAL: required build step(s) failed for $target: @hard\n" if @hard; } } @@ -1000,20 +1027,21 @@ sub run_build_steps_parallel { my $max_processes = $args{max_processes} // 1; return if !@{$steps}; - # Individual dep-builder failures here are TOLERATED only so one flaky builder does not abort - # the others. This is load-bearing, NOT laziness: some builders are expected to fail on a given - # arch/el (e.g. perl-Sys-Virt on el8 -- not a required dep), and some REQUIRED builders "fail" - # cosmetically while still producing their rpm (xCAT-genesis-base: xcat-core buildrpms.pl exits - # non-zero on an unrelated post-build xCAT-release-latest cp, yet the genesis rpm is built). So - # correctness is enforced by RESULT, not exit code: assert_required_deps runs after collection - # and fails the whole run if any REQUIRED rpm is missing -- caught at assert time, not swept - # under the rug. (A blanket "die on any builder failure" reddens the build on these non-issues.) + # Returns the ids of any steps that failed; the caller (build_one_target) enforces + # zero-tolerance -- any failed manifest package fails the whole run. We build only packages + # required for the target (per packages-manifest.conf), so there is no "expected to fail on this + # arch/el" case left to tolerate. genesis is the sole exception the CALLER handles: xcat-core's + # buildrpms.pl exits non-zero on an unrelated post-build xCAT-release-latest cp even when the + # genesis rpm IS built, so the caller treats genesis as failed only if its rpm is absent. if ($dry_run || $max_processes <= 1 || @{$steps} == 1) { + my @failed; for my $step (@{$steps}) { my $ok = eval { run_step(%{$step}); 1 }; - warn "WARN: build step failed (tolerated): $step->{step}\n" . ($@ // '') unless $ok; + next if $ok; + warn "ERROR: build step failed: $step->{step}\n" . ($@ // ''); + push @failed, (defined($step->{id}) && $step->{id} ne '' ? $step->{id} : $step->{step}); } - return; + return @failed; } my $workers = $max_processes; @@ -1068,10 +1096,9 @@ sub run_build_steps_parallel { push @lines, "$id (exit=$f->{exit}, signal=$f->{signal}, core_dump=$f->{core_dump})"; } - # Tolerated: warn, don't die. The REQUIRED set is asserted after collection/deploy. - warn "WARN: some build steps failed (tolerated; required deps asserted after deploy):\n " - . join("\n ", @lines) . "\n"; + warn "ERROR: build step(s) failed:\n " . join("\n ", @lines) . "\n"; } + return sort keys %failed; } # have_rpm: is there a non-src rpm named -... under $dir? @@ -1081,6 +1108,30 @@ sub have_rpm { return scalar(@m) > 0; } +# read_manifest: parse packages-manifest.conf into %{ target => { package => version|'*' } }. +# INI format: [target] sections; "package=version|*" entries; blank / "#" / ";" lines ignored. +# Returns an empty hash if the file is absent (callers that build require a section per target). +sub read_manifest { + my ($path) = @_; + my %m; + return %m unless -f $path; + open my $fh, '<', $path or die "Cannot read manifest $path: $!\n"; + my $sec; + while (my $line = <$fh>) { + $line =~ s/\r?\n\z//; + $line =~ s/^\s+|\s+$//g; + next if $line eq '' || $line =~ /^[#;]/; + if ($line =~ /^\[(.+?)\]$/) { $sec = $1; $m{$sec} ||= {}; next; } + next unless defined $sec; + my ($k, $v) = split /=/, $line, 2; + $k =~ s/\s+\z//; + $v = defined($v) ? ($v =~ s/^\s+//r) : ''; + $m{$sec}{$k} = ($v ne '') ? $v : '*'; + } + close $fh; + return %m; +} + # assert_required_deps: the per-EL dep repo is unusable without these, so a MISSING one is # fatal even though individual builder failures are tolerated above. genesis-base is required # unless --skip-genesis. diff --git a/packages-manifest.conf b/packages-manifest.conf new file mode 100644 index 0000000..924c088 --- /dev/null +++ b/packages-manifest.conf @@ -0,0 +1,98 @@ +# Per-target required xcat-dep package manifest. +# +# One [section] per mockbuild-all target (matches --target). Each entry is +# = +# where is the builder/package name (the dep builder name, the perl +# package name, or xCAT-genesis-base) and the value is a required version or `*` +# for "any version the source produces". +# +# mockbuild-all.pl reads this file and, per target, builds ONLY the listed +# packages -- a package not listed for a target is not built for it. Any listed +# package that fails to build fails the whole run (no tolerated failures). +# +# +# Notable results: conserver-xcat is never required (goconserver supersedes it); +# the variable perl modules differ per EL because the OS/EPEL already provides +# the others there (so xcat-dep is not required for them on that EL). + +[alma+epel-8-x86_64] +elilo-xcat=* +goconserver=* +grub2-xcat=* +ipmitool-xcat=* +syslinux-xcat=* +xnba-undi=* +perl-HTML-Form=* +perl-HTTP-Async=* +perl-IO-Stty=* +perl-Net-HTTPS-NB=* +xCAT-genesis-base=* + +[alma+epel-8-ppc64le] +elilo-xcat=* +goconserver=* +grub2-xcat=* +ipmitool-xcat=* +syslinux-xcat=* +xnba-undi=* +perl-HTML-Form=* +perl-HTTP-Async=* +perl-IO-Stty=* +perl-Net-HTTPS-NB=* +xCAT-genesis-base=* + +[alma+epel-9-x86_64] +elilo-xcat=* +goconserver=* +grub2-xcat=* +ipmitool-xcat=* +syslinux-xcat=* +xnba-undi=* +perl-HTTP-Async=* +perl-IO-Stty=* +perl-Net-HTTPS-NB=* +perl-Sys-Virt=* +xCAT-genesis-base=* + +[alma+epel-9-ppc64le] +elilo-xcat=* +goconserver=* +grub2-xcat=* +ipmitool-xcat=* +syslinux-xcat=* +xnba-undi=* +perl-HTTP-Async=* +perl-IO-Stty=* +perl-Net-HTTPS-NB=* +perl-Sys-Virt=* +xCAT-genesis-base=* + +[alma+epel-10-x86_64] +elilo-xcat=* +goconserver=* +grub2-xcat=* +ipmitool-xcat=* +syslinux-xcat=* +xnba-undi=* +perl-Crypt-SSLeay=* +perl-HTTP-Async=* +perl-IO-Stty=* +perl-Net-HTTPS-NB=* +perl-Net-Telnet=* +perl-Sys-Virt=* +xCAT-genesis-base=* + +[alma+epel-10-ppc64le] +elilo-xcat=* +goconserver=* +grub2-xcat=* +ipmitool-xcat=* +syslinux-xcat=* +xnba-undi=* +perl-Crypt-SSLeay=* +perl-HTTP-Async=* +perl-IO-Stty=* +perl-Net-HTTPS-NB=* +perl-Net-Telnet=* +perl-Sys-Virt=* +xCAT-genesis-base=* From 3d92fa97371d78c49aa36d04cc92277a3640f815 Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Tue, 28 Jul 2026 11:34:24 -0300 Subject: [PATCH 19/55] fix(xcat-dep): manifest -- build conserver-xcat everywhere; document per-EL perl set conserver-xcat is not pulled by `dnf install xCAT` (goconserver superseded it), but some users still deploy conserver, so build+ship it on every target. Document why the variable perl modules are scoped per EL: each is required from xcat-dep only where neither the base OS nor EPEL provides it. In particular perl-Sys-Virt is omitted on el8 because EPEL provides it on AlmaLinux 8 (not a build issue); likewise perl-HTML-Form (el9/el10), perl-Crypt-SSLeay and perl-Net-Telnet (el8/el9) come from the OS/EPEL on the releases where they are omitted. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- packages-manifest.conf | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/packages-manifest.conf b/packages-manifest.conf index 924c088..9fe17fc 100644 --- a/packages-manifest.conf +++ b/packages-manifest.conf @@ -11,11 +11,19 @@ # package that fails to build fails the whole run (no tolerated failures). # # -# Notable results: conserver-xcat is never required (goconserver supersedes it); -# the variable perl modules differ per EL because the OS/EPEL already provides -# the others there (so xcat-dep is not required for them on that EL). +# Per-EL perl set: a perl module is required from xcat-dep only on the releases +# where neither the base OS nor EPEL provides it; on the other releases EPEL/OS +# supplies it (so `dnf install xCAT` pulls the OS/EPEL copy, not xcat-dep's). +# perl-Sys-Virt omitted on el8 -- provided by EPEL on AlmaLinux 8. +# perl-HTML-Form only on el8 -- provided by the OS/EPEL on el9/el10. +# perl-Crypt-SSLeay only on el10 -- provided by the OS/EPEL on el8/el9. +# perl-Net-Telnet only on el10 -- provided by the OS/EPEL on el8/el9. +# +# conserver-xcat is NOT pulled by `dnf install xCAT` (goconserver superseded it), +# but it is built for every target anyway because some users still deploy it. [alma+epel-8-x86_64] +conserver-xcat=* elilo-xcat=* goconserver=* grub2-xcat=* @@ -29,6 +37,7 @@ perl-Net-HTTPS-NB=* xCAT-genesis-base=* [alma+epel-8-ppc64le] +conserver-xcat=* elilo-xcat=* goconserver=* grub2-xcat=* @@ -42,6 +51,7 @@ perl-Net-HTTPS-NB=* xCAT-genesis-base=* [alma+epel-9-x86_64] +conserver-xcat=* elilo-xcat=* goconserver=* grub2-xcat=* @@ -55,6 +65,7 @@ perl-Sys-Virt=* xCAT-genesis-base=* [alma+epel-9-ppc64le] +conserver-xcat=* elilo-xcat=* goconserver=* grub2-xcat=* @@ -68,6 +79,7 @@ perl-Sys-Virt=* xCAT-genesis-base=* [alma+epel-10-x86_64] +conserver-xcat=* elilo-xcat=* goconserver=* grub2-xcat=* @@ -83,6 +95,7 @@ perl-Sys-Virt=* xCAT-genesis-base=* [alma+epel-10-ppc64le] +conserver-xcat=* elilo-xcat=* goconserver=* grub2-xcat=* From 55c49c4ab287214da5fe1988a78f1db930f704e8 Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Tue, 28 Jul 2026 11:39:49 -0300 Subject: [PATCH 20/55] fix(xcat-dep): pin manifest package Versions and enforce them at build time Replace the `*` placeholders in package-manifest.conf with the concrete package Versions each source builds (e.g. ipmitool-xcat=1.8.18, perl-Sys-Virt=11.10.0, xCAT-genesis-base=2.19.0). Only the Version is pinned, not the Release (per-EL dist tag / genesis snap), and the Version is identical across all targets so the same pin applies everywhere. Enforce the pins: after collection, rpm_version() reads each required package's built %{version} from the repo and build_one_target fails the run if it differs from the pin (or the package is absent). `*` still accepts any version. This turns an unnoticed source Version bump into an explicit, actionable failure instead of a silently-shipped surprise. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- mockbuild-all.pl | 39 ++++++++++ packages-manifest.conf | 163 +++++++++++++++++++++-------------------- 2 files changed, 123 insertions(+), 79 deletions(-) diff --git a/mockbuild-all.pl b/mockbuild-all.pl index f1651ae..b074877 100755 --- a/mockbuild-all.pl +++ b/mockbuild-all.pl @@ -584,6 +584,24 @@ if (!$skip_genesis && !$dry_run) { } } +# Manifest version pins: every required package must be present at its pinned version. A build +# that produces a different version (a source version bump not reflected here) fails the run; +# a manifest value of '*' accepts any version. Only the Version is pinned, not the Release +# (which carries the per-EL dist tag and the genesis snap timestamp). +if (!$dry_run && !$skip_build) { + my @vmiss; + for my $pkg (sort keys %req) { + my $want = $req{$pkg}; + next if !defined($want) || $want eq '*'; + my $got = rpm_version($repo_dir, $pkg); + if (!defined $got) { push @vmiss, "$pkg: not built"; } + elsif ($got ne $want) { push @vmiss, "$pkg: built $got, manifest pins $want"; } + } + die "FATAL: manifest version mismatch for $target:\n " . join("\n ", @vmiss) . "\n" + if @vmiss; + print "[manifest] version pins satisfied for $target\n"; +} + print_step('Collect source RPM artifacts'); print "source collection roots:\n"; print " $_\n" for @srpm_collect_roots; @@ -1108,6 +1126,27 @@ sub have_rpm { return scalar(@m) > 0; } +# rpm_version: %{version} of the built binary rpm named under $dir (undef if absent). +# Skips src/debug rpms and confirms the rpm's real %{name} matches (glob can over-match). +# 'xCAT-genesis-base' matches the arch-suffixed rpm name (xCAT-genesis-base-x86_64 / -ppc64). +sub rpm_version { + my ($dir, $name) = @_; + my $glob = ($name eq 'xCAT-genesis-base') + ? "$dir/xCAT-genesis-base-*.rpm" + : "$dir/${name}-*.rpm"; + for my $f (sort glob($glob)) { + next if $f =~ /\.src\.rpm$/ || $f =~ /-debug(?:info|source)-/; + my $n = `rpm -qp --qf '%{name}' ${\ sh_quote($f)} 2>/dev/null`; + my $match = ($name eq 'xCAT-genesis-base') + ? ($n =~ /^xCAT-genesis-base-/) : ($n eq $name); + next unless $match; + my $v = `rpm -qp --qf '%{version}' ${\ sh_quote($f)} 2>/dev/null`; + chomp $v; + return $v; + } + return undef; +} + # read_manifest: parse packages-manifest.conf into %{ target => { package => version|'*' } }. # INI format: [target] sections; "package=version|*" entries; blank / "#" / ";" lines ignored. # Returns an empty hash if the file is absent (callers that build require a section per target). diff --git a/packages-manifest.conf b/packages-manifest.conf index 9fe17fc..56fbee8 100644 --- a/packages-manifest.conf +++ b/packages-manifest.conf @@ -3,12 +3,17 @@ # One [section] per mockbuild-all target (matches --target). Each entry is # = # where is the builder/package name (the dep builder name, the perl -# package name, or xCAT-genesis-base) and the value is a required version or `*` -# for "any version the source produces". +# package name, or xCAT-genesis-base) and the value is the required package +# Version. The build must produce exactly that Version or the run fails (a value +# of `*` accepts any version). Only the Version is pinned -- not the Release, +# which carries the per-EL dist tag (elN) and the genesis snap. Bump a +# pin here when the corresponding source Version is bumped; xCAT-genesis-base's +# Version tracks xcat-core's marketing Version. # # mockbuild-all.pl reads this file and, per target, builds ONLY the listed # packages -- a package not listed for a target is not built for it. Any listed -# package that fails to build fails the whole run (no tolerated failures). +# package that fails to build (or builds a mismatched version) fails the whole +# run (no tolerated failures). # # # Per-EL perl set: a perl module is required from xcat-dep only on the releases @@ -23,89 +28,89 @@ # but it is built for every target anyway because some users still deploy it. [alma+epel-8-x86_64] -conserver-xcat=* -elilo-xcat=* -goconserver=* -grub2-xcat=* -ipmitool-xcat=* -syslinux-xcat=* -xnba-undi=* -perl-HTML-Form=* -perl-HTTP-Async=* -perl-IO-Stty=* -perl-Net-HTTPS-NB=* -xCAT-genesis-base=* +conserver-xcat=8.2.1 +elilo-xcat=3.14 +goconserver=0.3.3 +grub2-xcat=1.0 +ipmitool-xcat=1.8.18 +syslinux-xcat=6.03 +xnba-undi=1.21.1 +perl-HTML-Form=6.07 +perl-HTTP-Async=0.30 +perl-IO-Stty=0.04 +perl-Net-HTTPS-NB=0.14 +xCAT-genesis-base=2.19.0 [alma+epel-8-ppc64le] -conserver-xcat=* -elilo-xcat=* -goconserver=* -grub2-xcat=* -ipmitool-xcat=* -syslinux-xcat=* -xnba-undi=* -perl-HTML-Form=* -perl-HTTP-Async=* -perl-IO-Stty=* -perl-Net-HTTPS-NB=* -xCAT-genesis-base=* +conserver-xcat=8.2.1 +elilo-xcat=3.14 +goconserver=0.3.3 +grub2-xcat=1.0 +ipmitool-xcat=1.8.18 +syslinux-xcat=6.03 +xnba-undi=1.21.1 +perl-HTML-Form=6.07 +perl-HTTP-Async=0.30 +perl-IO-Stty=0.04 +perl-Net-HTTPS-NB=0.14 +xCAT-genesis-base=2.19.0 [alma+epel-9-x86_64] -conserver-xcat=* -elilo-xcat=* -goconserver=* -grub2-xcat=* -ipmitool-xcat=* -syslinux-xcat=* -xnba-undi=* -perl-HTTP-Async=* -perl-IO-Stty=* -perl-Net-HTTPS-NB=* -perl-Sys-Virt=* -xCAT-genesis-base=* +conserver-xcat=8.2.1 +elilo-xcat=3.14 +goconserver=0.3.3 +grub2-xcat=1.0 +ipmitool-xcat=1.8.18 +syslinux-xcat=6.03 +xnba-undi=1.21.1 +perl-HTTP-Async=0.30 +perl-IO-Stty=0.04 +perl-Net-HTTPS-NB=0.14 +perl-Sys-Virt=11.10.0 +xCAT-genesis-base=2.19.0 [alma+epel-9-ppc64le] -conserver-xcat=* -elilo-xcat=* -goconserver=* -grub2-xcat=* -ipmitool-xcat=* -syslinux-xcat=* -xnba-undi=* -perl-HTTP-Async=* -perl-IO-Stty=* -perl-Net-HTTPS-NB=* -perl-Sys-Virt=* -xCAT-genesis-base=* +conserver-xcat=8.2.1 +elilo-xcat=3.14 +goconserver=0.3.3 +grub2-xcat=1.0 +ipmitool-xcat=1.8.18 +syslinux-xcat=6.03 +xnba-undi=1.21.1 +perl-HTTP-Async=0.30 +perl-IO-Stty=0.04 +perl-Net-HTTPS-NB=0.14 +perl-Sys-Virt=11.10.0 +xCAT-genesis-base=2.19.0 [alma+epel-10-x86_64] -conserver-xcat=* -elilo-xcat=* -goconserver=* -grub2-xcat=* -ipmitool-xcat=* -syslinux-xcat=* -xnba-undi=* -perl-Crypt-SSLeay=* -perl-HTTP-Async=* -perl-IO-Stty=* -perl-Net-HTTPS-NB=* -perl-Net-Telnet=* -perl-Sys-Virt=* -xCAT-genesis-base=* +conserver-xcat=8.2.1 +elilo-xcat=3.14 +goconserver=0.3.3 +grub2-xcat=1.0 +ipmitool-xcat=1.8.18 +syslinux-xcat=6.03 +xnba-undi=1.21.1 +perl-Crypt-SSLeay=0.72 +perl-HTTP-Async=0.30 +perl-IO-Stty=0.04 +perl-Net-HTTPS-NB=0.14 +perl-Net-Telnet=3.04 +perl-Sys-Virt=11.10.0 +xCAT-genesis-base=2.19.0 [alma+epel-10-ppc64le] -conserver-xcat=* -elilo-xcat=* -goconserver=* -grub2-xcat=* -ipmitool-xcat=* -syslinux-xcat=* -xnba-undi=* -perl-Crypt-SSLeay=* -perl-HTTP-Async=* -perl-IO-Stty=* -perl-Net-HTTPS-NB=* -perl-Net-Telnet=* -perl-Sys-Virt=* -xCAT-genesis-base=* +conserver-xcat=8.2.1 +elilo-xcat=3.14 +goconserver=0.3.3 +grub2-xcat=1.0 +ipmitool-xcat=1.8.18 +syslinux-xcat=6.03 +xnba-undi=1.21.1 +perl-Crypt-SSLeay=0.72 +perl-HTTP-Async=0.30 +perl-IO-Stty=0.04 +perl-Net-HTTPS-NB=0.14 +perl-Net-Telnet=3.04 +perl-Sys-Virt=11.10.0 +xCAT-genesis-base=2.19.0 From 74fc191cfb5a37ab56e57992f7a6a6cbd1393b2f Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Tue, 28 Jul 2026 11:58:43 -0300 Subject: [PATCH 21/55] feat(xcat-dep): support glob version pins; pin xCAT-genesis-base=2.* Manifest version pins may now be an exact Version, a shell-style glob (* and ?), or `*`. version_matches() anchors the glob (quotemeta then *->.* , ?->.), so 2.* matches 2.18.x / 2.19.x but not 3.x or 20.x. Use it to pin xCAT-genesis-base=2.* on every target. genesis-base's Version is not owned by xcat-dep -- it is whatever xcat-core (XCAT_CORE_REF) the genesis build compiles against, so it walks with the paired core (2.18.x today, 2.19.x on master). An exact pin would fail the run whenever the dep is built against a different core; 2.* asserts "a 2.x genesis" without coupling the manifest to a single core release. xCAT-genesis-scripts Requires xCAT-genesis-base >= 2:2.18.0 (a minimum, Epoch 2), so any 2.x genesis-base installs against a 2.18+ core. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- mockbuild-all.pl | 18 ++++++++++++++++-- packages-manifest.conf | 34 +++++++++++++++++++++------------- 2 files changed, 37 insertions(+), 15 deletions(-) diff --git a/mockbuild-all.pl b/mockbuild-all.pl index b074877..fd6a09c 100755 --- a/mockbuild-all.pl +++ b/mockbuild-all.pl @@ -594,8 +594,8 @@ if (!$dry_run && !$skip_build) { my $want = $req{$pkg}; next if !defined($want) || $want eq '*'; my $got = rpm_version($repo_dir, $pkg); - if (!defined $got) { push @vmiss, "$pkg: not built"; } - elsif ($got ne $want) { push @vmiss, "$pkg: built $got, manifest pins $want"; } + if (!defined $got) { push @vmiss, "$pkg: not built"; } + elsif (!version_matches($got, $want)) { push @vmiss, "$pkg: built $got, manifest pins $want"; } } die "FATAL: manifest version mismatch for $target:\n " . join("\n ", @vmiss) . "\n" if @vmiss; @@ -1129,6 +1129,20 @@ sub have_rpm { # rpm_version: %{version} of the built binary rpm named under $dir (undef if absent). # Skips src/debug rpms and confirms the rpm's real %{name} matches (glob can over-match). # 'xCAT-genesis-base' matches the arch-suffixed rpm name (xCAT-genesis-base-x86_64 / -ppc64). +# version_matches: does the built version $got satisfy the manifest pin $want? $want may be an +# exact version (2.19.0), a shell-style glob (2.* or 2.19.*), or '*' (any). Globs support * and +# ? and are anchored. Used so xCAT-genesis-base can pin 2.* (its Version walks with xcat-core) +# while the real xcat-dep packages stay exactly pinned. +sub version_matches { + my ($got, $want) = @_; + return 1 if !defined($want) || $want eq '*'; + return ($got eq $want) unless $want =~ /[*?]/; + my $re = quotemeta($want); + $re =~ s/\\\*/.*/g; + $re =~ s/\\\?/./g; + return $got =~ /\A$re\z/ ? 1 : 0; +} + sub rpm_version { my ($dir, $name) = @_; my $glob = ($name eq 'xCAT-genesis-base') diff --git a/packages-manifest.conf b/packages-manifest.conf index 56fbee8..1669a87 100644 --- a/packages-manifest.conf +++ b/packages-manifest.conf @@ -1,14 +1,22 @@ # Per-target required xcat-dep package manifest. # # One [section] per mockbuild-all target (matches --target). Each entry is -# = +# = # where is the builder/package name (the dep builder name, the perl -# package name, or xCAT-genesis-base) and the value is the required package -# Version. The build must produce exactly that Version or the run fails (a value -# of `*` accepts any version). Only the Version is pinned -- not the Release, -# which carries the per-EL dist tag (elN) and the genesis snap. Bump a -# pin here when the corresponding source Version is bumped; xCAT-genesis-base's -# Version tracks xcat-core's marketing Version. +# package name, or xCAT-genesis-base) and is one of: +# - an exact Version (e.g. 1.8.18) -- the build must produce exactly it; +# - a shell-style glob (e.g. 2.*) -- the built Version must match it (* and ?); +# - '*' -- any version is accepted. +# Only the Version is matched, never the Release (which carries the per-EL dist +# tag elN and the genesis snap). Bump an exact pin here when the +# corresponding in-tree source Version is bumped. +# +# xCAT-genesis-base is pinned as 2.* (not an exact version) on purpose: its +# Version is NOT owned by xcat-dep -- it is whatever xcat-core the genesis build +# compiles against (XCAT_CORE_REF), so it walks with the paired core (2.18.x, +# 2.19.x, ...). 2.* asserts "a 2.x genesis" without coupling the manifest to one +# core release. (xCAT-genesis-scripts Requires xCAT-genesis-base >= 2:2.18.0 -- a +# minimum with Epoch 2 -- so any 2.x genesis-base installs against a 2.18+ core.) # # mockbuild-all.pl reads this file and, per target, builds ONLY the listed # packages -- a package not listed for a target is not built for it. Any listed @@ -39,7 +47,7 @@ perl-HTML-Form=6.07 perl-HTTP-Async=0.30 perl-IO-Stty=0.04 perl-Net-HTTPS-NB=0.14 -xCAT-genesis-base=2.19.0 +xCAT-genesis-base=2.* [alma+epel-8-ppc64le] conserver-xcat=8.2.1 @@ -53,7 +61,7 @@ perl-HTML-Form=6.07 perl-HTTP-Async=0.30 perl-IO-Stty=0.04 perl-Net-HTTPS-NB=0.14 -xCAT-genesis-base=2.19.0 +xCAT-genesis-base=2.* [alma+epel-9-x86_64] conserver-xcat=8.2.1 @@ -67,7 +75,7 @@ perl-HTTP-Async=0.30 perl-IO-Stty=0.04 perl-Net-HTTPS-NB=0.14 perl-Sys-Virt=11.10.0 -xCAT-genesis-base=2.19.0 +xCAT-genesis-base=2.* [alma+epel-9-ppc64le] conserver-xcat=8.2.1 @@ -81,7 +89,7 @@ perl-HTTP-Async=0.30 perl-IO-Stty=0.04 perl-Net-HTTPS-NB=0.14 perl-Sys-Virt=11.10.0 -xCAT-genesis-base=2.19.0 +xCAT-genesis-base=2.* [alma+epel-10-x86_64] conserver-xcat=8.2.1 @@ -97,7 +105,7 @@ perl-IO-Stty=0.04 perl-Net-HTTPS-NB=0.14 perl-Net-Telnet=3.04 perl-Sys-Virt=11.10.0 -xCAT-genesis-base=2.19.0 +xCAT-genesis-base=2.* [alma+epel-10-ppc64le] conserver-xcat=8.2.1 @@ -113,4 +121,4 @@ perl-IO-Stty=0.04 perl-Net-HTTPS-NB=0.14 perl-Net-Telnet=3.04 perl-Sys-Virt=11.10.0 -xCAT-genesis-base=2.19.0 +xCAT-genesis-base=2.* From 12b7f10b60c27d8bed8959da6575a825b054ceca Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:01:31 -0300 Subject: [PATCH 22/55] docs(xcat-dep): record upstream source URLs in mockbuild.pl scripts elilo, ipmitool and syslinux build from a tracked, in-repo source tarball and no longer fetch at build time, so the upstream download URL was undocumented. Add it back as a provenance comment next to the tracked-source block so it is clear where the tarball came from and where to re-download when bumping the version. (xnba already records https://ipxe.org/ and goconserver keeps its git repo URL in $go_repo; grub2-xcat is repackaged from the distribution grub2 and conserver uses a dummy spec, so neither has a single upstream download URL.) Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- elilo/mockbuild.pl | 2 ++ ipmitool/mockbuild.pl | 2 ++ syslinux/mockbuild.pl | 2 ++ 3 files changed, 6 insertions(+) diff --git a/elilo/mockbuild.pl b/elilo/mockbuild.pl index 6e4aa32..d6d7b5e 100755 --- a/elilo/mockbuild.pl +++ b/elilo/mockbuild.pl @@ -88,6 +88,8 @@ print_step("Mock config check"); run("mock -r " . sh_quote($mock_cfg) . $mock_uniqueext_opt . " --print-root-path >/dev/null"); print_step("Verify tracked source archive"); +# Upstream source (documented for provenance; NOT fetched at build time -- see below): +# https://downloads.sourceforge.net/project/elilo/elilo/elilo-3.14/elilo-3.14-all.tar.gz # Source0 (elilo--source.tar.gz) is tracked in the repo, already normalized to an elilo/ # top-level tree, and consumed directly by mock (--sources $pkg_dir below). There is nothing to # download: the old fetch re-derived this SAME tracked file and rewrote it IN PLACE. Because the diff --git a/ipmitool/mockbuild.pl b/ipmitool/mockbuild.pl index dbd9241..f02f2f2 100755 --- a/ipmitool/mockbuild.pl +++ b/ipmitool/mockbuild.pl @@ -89,6 +89,8 @@ print_step("Mock config check"); run("mock -r " . sh_quote($mock_cfg) . $mock_uniqueext_opt . " --print-root-path >/dev/null"); print_step("Verify tracked source archive"); +# Upstream source (documented for provenance; NOT fetched at build time -- see below): +# https://github.com/ipmitool/ipmitool/archive/refs/tags/IPMITOOL_1_8_18.tar.gz # The ipmitool source (ipmitool-.tar.gz) is tracked in the repo, already normalized to the # ipmitool-/ top-level that %setup -n expects, and consumed directly by mock (--sources # $pkg_dir below). There is nothing to download: the old fetch re-derived this SAME tracked file diff --git a/syslinux/mockbuild.pl b/syslinux/mockbuild.pl index d7bd499..ca82bd8 100755 --- a/syslinux/mockbuild.pl +++ b/syslinux/mockbuild.pl @@ -96,6 +96,8 @@ print_step("Mock config check"); run("mock -r " . sh_quote($mock_cfg) . $mock_uniqueext_opt . " --print-root-path >/dev/null"); print_step("Verify tracked source archive"); +# Upstream source (documented for provenance; NOT fetched at build time -- see below): +# https://www.kernel.org/pub/linux/utils/boot/syslinux/syslinux-6.03.tar.xz # The syslinux source (syslinux-.tar.xz, Source0) is tracked in the repo, has the # syslinux-/ top-level that %setup -n expects, and is consumed directly by mock (--sources # $pkg_dir below). There is nothing to download: the old fetch re-downloaded this SAME tracked file From ee6524ade55ebac633617020be1dea932d1aa5b0 Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Tue, 28 Jul 2026 16:57:35 -0300 Subject: [PATCH 23/55] fix(xcat-dep): address PR #62 review -- finalize/genesis/skip correctness + tests Review feedback (viniciusferrao): 1. --finalize-xcat-dep no longer succeeds with no genesis rpms and no longer treats a shared filename as up to date when the content differs. - finalize_xcat_dep now REQUIRES each arch's own genesis rpm for every repo pair it processes (a pair with none is a hard FATAL, not a silent exit-0 no-op). - cross_copy_genesis compares RPM identity by SIGMD5 (header+payload digest, independent of the GPG signature), so a stale rpm that merely shares a basename is refreshed instead of being mistaken for up to date. 2. Remove the genesis workaround. xcat-core #7696 is merged, so buildrpms.pl now exits 0 iff it produced the genesis rpm; the zero-tolerance check no longer ignores a genesis failure when a matching (possibly stale) rpm already exists -- any failed build step, genesis included, fails the run. 3. Skip modes work. required_pkgs() drops the packages whose builder was skipped, and both the version-pin check and assert_required_deps use it, so a clean --skip-genesis / --skip-xcat-dep / --skip-perl run no longer fails validating packages it deliberately did not build. 4. Tests. The reusable, side-effect-free helpers are factored into MockBuildUtils.pm (cross_copy_genesis and finalize_xcat_dep take injected sign/reindex callbacks so they carry no gpg/createrepo state) and t/mockbuild-all.t adds focused fixture tests for all of the above: skip-mode selection, version-pin globs, SIGMD5-based RPM-identity comparison + cross_copy refresh/idempotency, and the finalize require-inputs guard. Run with `prove t/`. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- BUILD.md | 13 +++ MockBuildUtils.pm | 204 ++++++++++++++++++++++++++++++++++++++++++++++ mockbuild-all.pl | 192 +++++++------------------------------------ t/mockbuild-all.t | 142 ++++++++++++++++++++++++++++++++ 4 files changed, 388 insertions(+), 163 deletions(-) create mode 100644 MockBuildUtils.pm create mode 100644 t/mockbuild-all.t diff --git a/BUILD.md b/BUILD.md index 4e882b9..e74a6a5 100644 --- a/BUILD.md +++ b/BUILD.md @@ -304,6 +304,19 @@ find /build-output/mockbuild-all//build-logs -type f | sort - `mock target not found` - Validate with `mock -r --print-root-path` and install the required mock config packages. +# Tests + +The reusable, side-effect-free helpers live in `MockBuildUtils.pm` (package selection under the +`--skip-*` flags, version-pin matching incl. globs, RPM-identity comparison, and the cross-arch +genesis `finalize` logic). Focused fixture tests cover them: + +```bash +prove t/ # or: perl t/mockbuild-all.t +``` + +The RPM-identity / `cross_copy_genesis` cases build tiny fixture rpms and are skipped +automatically if `rpmbuild` is unavailable. + # References - [mock project repository](https://github.com/rpm-software-management/mock) diff --git a/MockBuildUtils.pm b/MockBuildUtils.pm new file mode 100644 index 0000000..7924f6a --- /dev/null +++ b/MockBuildUtils.pm @@ -0,0 +1,204 @@ +package MockBuildUtils; +# Reusable, unit-testable helpers factored out of mockbuild-all.pl. Kept free of that script's +# globals so t/mockbuild-all.t can exercise them directly. The two orchestration helpers that +# need signing / re-indexing (cross_copy_genesis, finalize_xcat_dep) take those as injected +# callbacks instead of reaching for gpg/createrepo state, so they stay pure and testable. +use strict; +use warnings; +use Exporter 'import'; +use File::Basename qw(basename); +use File::Copy qw(copy); + +our @EXPORT_OK = qw( + sh_quote print_step + version_matches required_pkgs have_rpm read_manifest + rpm_version rpm_sigmd5 + cross_copy_genesis finalize_xcat_dep +); + +# sh_quote: single-quote a string for safe use in a shell command. +sub sh_quote { + my ($s) = @_; + $s = '' if !defined $s; + $s =~ s/'/'"'"'/g; + return "'$s'"; +} + +# print_step: print a step banner. +sub print_step { + my ($msg) = @_; + print "\n== $msg ==\n"; +} + +# version_matches: does the built version $got satisfy the manifest pin $want? $want may be an +# exact version (2.19.0), a shell-style glob (2.* or 2.19.*), or '*' (any). Globs support * and +# ? and are anchored. Used so xCAT-genesis-base can pin 2.* (its Version walks with xcat-core) +# while the real xcat-dep packages stay exactly pinned. +sub version_matches { + my ($got, $want) = @_; + return 1 if !defined($want) || $want eq '*'; + return ($got eq $want) unless $want =~ /[*?]/; + my $re = quotemeta($want); + $re =~ s/\\\*/.*/g; + $re =~ s/\\\?/./g; + return $got =~ /\A$re\z/ ? 1 : 0; +} + +# required_pkgs: given a list of manifest package names and the skip flags, return the subset +# that must actually be built and validated. A package whose builder was skipped is NOT required: +# --skip-genesis drops xCAT-genesis-base, --skip-perl drops perl-*, --skip-xcat-dep drops the dep +# builders (everything that is neither genesis nor perl). Pure function (flags passed in) so both +# the version-pin check and assert_required_deps use it and it is unit-testable. +sub required_pkgs { + my ($pkgs, $skip_genesis, $skip_perl, $skip_dep) = @_; + return grep { + !($skip_genesis && $_ eq 'xCAT-genesis-base') + && !($skip_perl && /^perl-/) + && !($skip_dep && $_ ne 'xCAT-genesis-base' && $_ !~ /^perl-/) + } @$pkgs; +} + +# have_rpm: is there a non-src rpm named -... under $dir? +sub have_rpm { + my ($dir, $name) = @_; + my @m = grep { !/\.src\.rpm$/ } glob("$dir/${name}-*.rpm"); + return scalar(@m) > 0; +} + +# rpm_sigmd5: the SIGMD5 of an rpm -- the digest of its header+payload, independent of the GPG +# signature. Used to compare RPM identity/content: two rpms that share a basename but differ in +# content have different SIGMD5 (a bare filename match is not enough to call them identical). +sub rpm_sigmd5 { + my ($f) = @_; + return '' unless defined $f && -f $f; + my $v = `rpm -qp --qf '%{SIGMD5}' ${\ sh_quote($f)} 2>/dev/null`; + chomp $v; + return $v; +} + +# rpm_version: %{version} of the built binary rpm named under $dir (undef if absent). +# Skips src/debug rpms and confirms the rpm's real %{name} matches (glob can over-match). +# 'xCAT-genesis-base' matches the arch-suffixed rpm name (xCAT-genesis-base-x86_64 / -ppc64). +sub rpm_version { + my ($dir, $name) = @_; + my $glob = ($name eq 'xCAT-genesis-base') + ? "$dir/xCAT-genesis-base-*.rpm" + : "$dir/${name}-*.rpm"; + for my $f (sort glob($glob)) { + next if $f =~ /\.src\.rpm$/ || $f =~ /-debug(?:info|source)-/; + my $n = `rpm -qp --qf '%{name}' ${\ sh_quote($f)} 2>/dev/null`; + my $match = ($name eq 'xCAT-genesis-base') + ? ($n =~ /^xCAT-genesis-base-/) : ($n eq $name); + next unless $match; + my $v = `rpm -qp --qf '%{version}' ${\ sh_quote($f)} 2>/dev/null`; + chomp $v; + return $v; + } + return undef; +} + +# read_manifest: parse packages-manifest.conf into %{ target => { package => version|'*' } }. +# INI format: [target] sections; "package=version|*" entries; blank / "#" / ";" lines ignored. +# Returns an empty hash if the file is absent (callers that build require a section per target). +sub read_manifest { + my ($path) = @_; + my %m; + return %m unless -f $path; + open my $fh, '<', $path or die "Cannot read manifest $path: $!\n"; + my $sec; + while (my $line = <$fh>) { + $line =~ s/\r?\n\z//; + $line =~ s/^\s+|\s+$//g; + next if $line eq '' || $line =~ /^[#;]/; + if ($line =~ /^\[(.+?)\]$/) { $sec = $1; $m{$sec} ||= {}; next; } + next unless defined $sec; + my ($k, $v) = split /=/, $line, 2; + $k =~ s/\s+\z//; + $v = defined($v) ? ($v =~ s/^\s+//r) : ''; + $m{$sec}{$k} = ($v ne '') ? $v : '*'; + } + close $fh; + return %m; +} + +# cross_copy_genesis: copy the noarch xCAT-genesis-base--*.rpm from $from into $to, dropping +# any stale foreign-arch genesis already in $to so the repo ends with exactly the fresh set. +# Returns the count of rpms newly copied (0 = already up to date, so the caller can skip +# re-indexing). Idempotent. $sign is an optional coderef ($rpm_path) invoked on each copied rpm +# (e.g. to re-sign it); pass undef to skip signing. Content is compared by SIGMD5, so a stale +# same-name rpm is refreshed rather than mistaken for up to date. +sub cross_copy_genesis { + my ($from, $to, $tarch, $sign) = @_; + my @src = grep { !/\.src\.rpm$/ } glob("$from/xCAT-genesis-base-$tarch-*.rpm"); + return 0 if !@src; + my %want = map { basename($_) => $_ } @src; + my @existing = grep { !/\.src\.rpm$/ } glob("$to/xCAT-genesis-base-$tarch-*.rpm"); + if (scalar(@existing) == scalar(keys %want)) { + my $up_to_date = 1; + for my $base (keys %want) { + my $dst = "$to/$base"; + if (!-f $dst || rpm_sigmd5($want{$base}) ne rpm_sigmd5($dst)) { $up_to_date = 0; last; } + } + return 0 if $up_to_date; + } + for my $old (@existing) { + unlink $old or die "Failed to remove stale genesis $old: $!\n"; + print "[finalize] - " . basename($old) . " (stale foreign-arch, removed from $to)\n"; + } + my $copied = 0; + for my $base (sort keys %want) { + copy($want{$base}, "$to/$base") + or die "Failed to cross-copy genesis $want{$base} -> $to: $!\n"; + print "[finalize] + $base ($from -> $to)\n"; + $sign->("$to/$base") if $sign; # e.g. re-sign so the deploy gate never sees an unsigned rpm + $copied++; + } + return $copied; +} + +# finalize_xcat_dep: cross-populate the noarch xCAT-genesis-base between each matching +# /x86_64 and /ppc64le repo pair (issue #7610), then re-index the repos that changed. +# %opt: sign => coderef($rpm) applied to copied rpms (or undef); reindex => coderef($dir) run on +# a repo whose rpm set changed (or undef). Both injected so this stays free of gpg/createrepo +# state and is unit-testable. Requires each arch's own genesis rpm to be present (a pair with no +# genesis is a hard error, never a silent no-op) and fails if no repo pair is found at all. +sub finalize_xcat_dep { + my ($x86_64_repo, $ppc64le_repo, %opt) = @_; + my $sign = $opt{sign}; + my $reindex = $opt{reindex}; + print_step('Finalize xcat-dep: cross-arch genesis-base provisioning (issue #7610)'); + print "x86_64-repo: $x86_64_repo\n"; + print "ppc64le-repo: $ppc64le_repo\n"; + my @osdirs = grep { -d "$_/x86_64" } glob("$x86_64_repo/*"); + my $pairs = 0; + for my $p (sort @osdirs) { + my $osdir = basename($p); + my $x86dir = "$x86_64_repo/$osdir/x86_64"; + my $ppcdir = "$ppc64le_repo/$osdir/ppc64le"; + if (!-d $ppcdir) { + print "[finalize] $osdir: no ppc64le peer at $ppcdir -- skipping\n"; + next; + } + # Require the expected inputs: each arch's build must have produced its OWN genesis rpm + # before finalize cross-populates them. Without this, a pair whose builds produced no + # genesis rpms would make finalize a silent no-op that still exits 0 (the bug this guards). + die "FATAL: [finalize] $osdir: no x86_64 xCAT-genesis-base rpm in $x86dir\n" + if !grep { !/\.src\.rpm$/ } glob("$x86dir/xCAT-genesis-base-x86_64-*.rpm"); + die "FATAL: [finalize] $osdir: no ppc64 xCAT-genesis-base rpm in $ppcdir\n" + if !grep { !/\.src\.rpm$/ } glob("$ppcdir/xCAT-genesis-base-ppc64-*.rpm"); + # xCAT collapses ppc/ppc64/ppc64le into tarch=ppc64, so the ppc genesis rpm is + # named xCAT-genesis-base-ppc64-*. Cross-copy both directions. + my $to_x86 = cross_copy_genesis($ppcdir, $x86dir, 'ppc64', $sign); + my $to_ppc = cross_copy_genesis($x86dir, $ppcdir, 'x86_64', $sign); + $reindex->($x86dir) if $to_x86 && $reindex; + $reindex->($ppcdir) if $to_ppc && $reindex; + printf "[finalize] %s: %d ppc64 genesis -> x86_64, %d x86_64 genesis -> ppc64le\n", + $osdir, $to_x86, $to_ppc; + $pairs++; + } + die "FATAL: --finalize-xcat-dep found no /x86_64 + /ppc64le repo pair under\n" + . " --x86_64-repo '$x86_64_repo'\n --ppc64le-repo '$ppc64le_repo'\n" if $pairs == 0; + print_step('Finalize complete'); +} + +1; diff --git a/mockbuild-all.pl b/mockbuild-all.pl index fd6a09c..0ae4ff2 100755 --- a/mockbuild-all.pl +++ b/mockbuild-all.pl @@ -11,6 +11,10 @@ use File::Path qw(make_path); use Getopt::Long qw(GetOptions); use Parallel::ForkManager; use POSIX qw(strftime); +use FindBin qw($RealBin); +use lib $RealBin; +use MockBuildUtils qw(sh_quote print_step version_matches required_pkgs have_rpm + read_manifest rpm_version rpm_sigmd5 cross_copy_genesis finalize_xcat_dep); my $script_dir = abs_path(dirname(__FILE__)); my $repo_root = abs_path($script_dir); @@ -131,7 +135,16 @@ if ($finalize_xcat_dep) { my $ppc = abs_path($ppc64le_repo) or die "--ppc64le-repo '$ppc64le_repo' not found\n"; die "--x86_64-repo '$x86' is not a directory\n" if !-d $x86; die "--ppc64le-repo '$ppc' is not a directory\n" if !-d $ppc; - finalize_xcat_dep($x86, $ppc); + # Inject the per-rpm gpg re-sign and the repo re-index as callbacks so the finalize logic in + # MockBuildUtils stays free of this script's gpg/createrepo state. + finalize_xcat_dep($x86, $ppc, + sign => ($gpg_sign ? sub { + my ($rpm) = @_; + local $ENV{GNUPGHOME} = $gpg_home if $gpg_home; + run_simple(qq(rpmsign --define "%_gpg_name $gpg_key_name" --addsign ) . sh_quote($rpm)); + } : undef), + reindex => \&reindex_and_sign_repo, + ); exit 0; } @@ -517,22 +530,12 @@ if (!$skip_build) { } } - # Zero-tolerance: any required (manifest) package that failed to build fails the run. - # genesis is the one exception -- xcat-core's buildrpms.pl exits non-zero on an unrelated - # post-build xCAT-release-latest cp even when the genesis rpm IS produced, so genesis - # counts as failed only if its rpm is absent, not on exit code. - my @hard; - for my $id (@failed) { - if ($id eq 'genesis') { - my @g = grep { !/\.src\.rpm$/ } - glob("$xcat_src/dist/$target/rpms/xCAT-genesis-base-*.rpm"); - push @hard, $id unless @g; - } - else { - push @hard, $id; - } - } - die "FATAL: required build step(s) failed for $target: @hard\n" if @hard; + # Zero-tolerance: any build step that failed fails the whole run -- genesis included. + # (xcat-core #7696 is merged: buildrpms.pl now exits 0 iff it actually produced the + # genesis rpm, so there is no cosmetic non-zero exit left to tolerate. The old workaround + # -- ignore a genesis failure when a matching rpm already exists in dist/ -- is gone; a + # stale artifact from a previous build must never mask a failed genesis build.) + die "FATAL: required build step(s) failed for $target: @failed\n" if @failed; } } @@ -590,7 +593,9 @@ if (!$skip_genesis && !$dry_run) { # (which carries the per-EL dist tag and the genesis snap timestamp). if (!$dry_run && !$skip_build) { my @vmiss; - for my $pkg (sort keys %req) { + # Only validate packages whose builder was NOT skipped -- so a clean --skip-* run does not + # fail on packages it deliberately did not build. + for my $pkg (required_pkgs([sort keys %req], $skip_genesis, $skip_perl, $skip_xcat_dep)) { my $want = $req{$pkg}; next if !defined($want) || $want eq '*'; my $got = rpm_version($repo_dir, $pkg); @@ -798,77 +803,7 @@ EOF close $b; } -# --finalize-xcat-dep: cross-populate the noarch xCAT-genesis-base between each matching -# /x86_64 and /ppc64le repo pair, then re-index + re-sign the repos that changed. -# / are the two per-arch repo roots (each holding rh8/rh9/rh10/). -# They may be the same path (both arches built into one tree) or two separate trees (one per -# build host); either way pairs are matched by subdir. -sub finalize_xcat_dep { - my ($x86_64_repo, $ppc64le_repo) = @_; - print_step('Finalize xcat-dep: cross-arch genesis-base provisioning (issue #7610)'); - print "x86_64-repo: $x86_64_repo\n"; - print "ppc64le-repo: $ppc64le_repo\n"; - # Every OS-release dir under the x86_64 repo that actually has an x86_64 sub-repo. - my @osdirs = grep { -d "$_/x86_64" } glob("$x86_64_repo/*"); - my $pairs = 0; - for my $p (sort @osdirs) { - my $osdir = basename($p); - my $x86dir = "$x86_64_repo/$osdir/x86_64"; - my $ppcdir = "$ppc64le_repo/$osdir/ppc64le"; - if (!-d $ppcdir) { - print "[finalize] $osdir: no ppc64le peer at $ppcdir -- skipping\n"; - next; - } - # xCAT collapses ppc/ppc64/ppc64le into tarch=ppc64, so the ppc genesis rpm is - # named xCAT-genesis-base-ppc64-*. Cross-copy both directions. - my $to_x86 = cross_copy_genesis($ppcdir, $x86dir, 'ppc64'); - my $to_ppc = cross_copy_genesis($x86dir, $ppcdir, 'x86_64'); - reindex_and_sign_repo($x86dir) if $to_x86; - reindex_and_sign_repo($ppcdir) if $to_ppc; - printf "[finalize] %s: %d ppc64 genesis -> x86_64, %d x86_64 genesis -> ppc64le\n", - $osdir, $to_x86, $to_ppc; - $pairs++; - } - die "FATAL: --finalize-xcat-dep found no /x86_64 + /ppc64le repo pair under\n" - . " --x86_64-repo '$x86_64_repo'\n --ppc64le-repo '$ppc64le_repo'\n" if $pairs == 0; - print_step('Finalize complete'); -} -# Cross-copy the noarch xCAT-genesis-base--*.rpm from $from into $to. Drops any -# stale foreign-arch genesis already in $to (e.g. issue #7610's 2.16.3 ppc leftover) so -# the repo ends with exactly the fresh set. Returns the number of rpms newly copied -# (0 = already up to date, so the caller can skip re-indexing). Idempotent. -sub cross_copy_genesis { - my ($from, $to, $tarch) = @_; - my @src = grep { !/\.src\.rpm$/ } glob("$from/xCAT-genesis-base-$tarch-*.rpm"); - return 0 if !@src; - my %want = map { basename($_) => $_ } @src; - my @existing = grep { !/\.src\.rpm$/ } glob("$to/xCAT-genesis-base-$tarch-*.rpm"); - my %have = map { basename($_) => 1 } @existing; - # Already exactly the fresh set (same basenames)? idempotent no-op. - if (scalar(keys %want) == scalar(keys %have) && !grep { !$have{$_} } keys %want) { - return 0; - } - for my $old (@existing) { - unlink $old or die "Failed to remove stale genesis $old: $!\n"; - print "[finalize] - " . basename($old) . " (stale foreign-arch, removed from $to)\n"; - } - my $copied = 0; - for my $base (sort keys %want) { - copy($want{$base}, "$to/$base") - or die "Failed to cross-copy genesis $want{$base} -> $to: $!\n"; - print "[finalize] + $base ($from -> $to)\n"; - # The source rpm is already signed by the build, but re-assert it under --gpg-sign - # so the deploy signing gate never sees an unsigned cross-copied rpm. - if ($gpg_sign) { - local $ENV{GNUPGHOME} = $gpg_home if $gpg_home; - run_simple(qq(rpmsign --define "%_gpg_name $gpg_key_name" --addsign ) - . sh_quote("$to/$base")); - } - $copied++; - } - return $copied; -} # Re-run createrepo_c on a repo whose rpm set changed, and (under --gpg-sign) re-sign + # re-export repomd. Does NOT re-sign the rpms (cross_copy_genesis already did the copied @@ -950,10 +885,6 @@ Notes: USAGE } -sub print_step { - my ($msg) = @_; - print "\n== $msg ==\n"; -} sub require_command { my ($cmd) = @_; @@ -1119,71 +1050,11 @@ sub run_build_steps_parallel { return sort keys %failed; } -# have_rpm: is there a non-src rpm named -... under $dir? -sub have_rpm { - my ($dir, $name) = @_; - my @m = grep { !/\.src\.rpm$/ } glob("$dir/${name}-*.rpm"); - return scalar(@m) > 0; -} -# rpm_version: %{version} of the built binary rpm named under $dir (undef if absent). -# Skips src/debug rpms and confirms the rpm's real %{name} matches (glob can over-match). -# 'xCAT-genesis-base' matches the arch-suffixed rpm name (xCAT-genesis-base-x86_64 / -ppc64). -# version_matches: does the built version $got satisfy the manifest pin $want? $want may be an -# exact version (2.19.0), a shell-style glob (2.* or 2.19.*), or '*' (any). Globs support * and -# ? and are anchored. Used so xCAT-genesis-base can pin 2.* (its Version walks with xcat-core) -# while the real xcat-dep packages stay exactly pinned. -sub version_matches { - my ($got, $want) = @_; - return 1 if !defined($want) || $want eq '*'; - return ($got eq $want) unless $want =~ /[*?]/; - my $re = quotemeta($want); - $re =~ s/\\\*/.*/g; - $re =~ s/\\\?/./g; - return $got =~ /\A$re\z/ ? 1 : 0; -} -sub rpm_version { - my ($dir, $name) = @_; - my $glob = ($name eq 'xCAT-genesis-base') - ? "$dir/xCAT-genesis-base-*.rpm" - : "$dir/${name}-*.rpm"; - for my $f (sort glob($glob)) { - next if $f =~ /\.src\.rpm$/ || $f =~ /-debug(?:info|source)-/; - my $n = `rpm -qp --qf '%{name}' ${\ sh_quote($f)} 2>/dev/null`; - my $match = ($name eq 'xCAT-genesis-base') - ? ($n =~ /^xCAT-genesis-base-/) : ($n eq $name); - next unless $match; - my $v = `rpm -qp --qf '%{version}' ${\ sh_quote($f)} 2>/dev/null`; - chomp $v; - return $v; - } - return undef; -} -# read_manifest: parse packages-manifest.conf into %{ target => { package => version|'*' } }. -# INI format: [target] sections; "package=version|*" entries; blank / "#" / ";" lines ignored. -# Returns an empty hash if the file is absent (callers that build require a section per target). -sub read_manifest { - my ($path) = @_; - my %m; - return %m unless -f $path; - open my $fh, '<', $path or die "Cannot read manifest $path: $!\n"; - my $sec; - while (my $line = <$fh>) { - $line =~ s/\r?\n\z//; - $line =~ s/^\s+|\s+$//g; - next if $line eq '' || $line =~ /^[#;]/; - if ($line =~ /^\[(.+?)\]$/) { $sec = $1; $m{$sec} ||= {}; next; } - next unless defined $sec; - my ($k, $v) = split /=/, $line, 2; - $k =~ s/\s+\z//; - $v = defined($v) ? ($v =~ s/^\s+//r) : ''; - $m{$sec}{$k} = ($v ne '') ? $v : '*'; - } - close $fh; - return %m; -} + + # assert_required_deps: the per-EL dep repo is unusable without these, so a MISSING one is # fatal even though individual builder failures are tolerated above. genesis-base is required @@ -1196,9 +1067,10 @@ sub assert_required_deps { # elilo-xcat is noarch but xCAT hard-requires it (Requires: elilo-xcat >= 3.14-6) on EVERY arch, # so a missing elilo makes the whole dep repo uninstallable -- it MUST be required here, not # silently tolerated (it builds from a tracked prebuilt on ppc64le/EL8, compiled elsewhere). - my @req = qw(elilo-xcat ipmitool-xcat syslinux-xcat grub2-xcat xnba-undi - perl-IO-Stty perl-HTTP-Async perl-Net-HTTPS-NB); - push @req, 'xCAT-genesis-base' unless $skip_genesis; + # A package whose builder was skipped is not required (else a clean --skip-* run fails). + my @all = qw(elilo-xcat ipmitool-xcat syslinux-xcat grub2-xcat xnba-undi + perl-IO-Stty perl-HTTP-Async perl-Net-HTTPS-NB xCAT-genesis-base); + my @req = required_pkgs(\@all, $skip_genesis, $skip_perl, $skip_xcat_dep); my @missing = grep { !have_rpm($dir, $_) } @req; die "FATAL: required deps missing from $dir: @missing\n" if @missing; print "[deps] required set present in $dir: @req\n"; @@ -1443,9 +1315,3 @@ sub slurp_chomp { return $line // ''; } -sub sh_quote { - my ($s) = @_; - $s = '' if !defined $s; - $s =~ s/'/'"'"'/g; - return "'$s'"; -} diff --git a/t/mockbuild-all.t b/t/mockbuild-all.t new file mode 100644 index 0000000..9f0b690 --- /dev/null +++ b/t/mockbuild-all.t @@ -0,0 +1,142 @@ +#!/usr/bin/perl +# Focused fixture tests for the xcat-dep build helpers (MockBuildUtils.pm), covering the review +# feedback on PR #62: skip-mode package selection, version pins, RPM-identity comparison in the +# cross-arch genesis finalize, and the "require the genesis input" guard. +use strict; +use warnings; +use Test::More; +use FindBin qw($RealBin); +use lib "$RealBin/.."; +use File::Temp qw(tempdir); +use File::Path qw(make_path); +use File::Basename qw(basename); +use MockBuildUtils qw(required_pkgs version_matches rpm_sigmd5 + cross_copy_genesis finalize_xcat_dep read_manifest); + +# Run a printing sub with STDOUT muted so its progress lines do not pollute TAP. +sub quiet(&) { + my ($code) = @_; + open(my $save, '>&', \*STDOUT) or die "dup STDOUT: $!"; + open(STDOUT, '>', '/dev/null') or die "mute STDOUT: $!"; + my @r = eval { $code->() }; + my $err = $@; + open(STDOUT, '>&', $save) or die "restore STDOUT: $!"; + die $err if $err; + return wantarray ? @r : $r[0]; +} + +# ---- required_pkgs: a skipped builder's packages are not required (clean --skip-* runs) ------- +my @all = qw(elilo-xcat ipmitool-xcat perl-IO-Stty perl-Sys-Virt xCAT-genesis-base); +is_deeply([required_pkgs(\@all, 0, 0, 0)], \@all, + 'no skips -> every package required'); +is_deeply([required_pkgs(\@all, 1, 0, 0)], [qw(elilo-xcat ipmitool-xcat perl-IO-Stty perl-Sys-Virt)], + '--skip-genesis drops xCAT-genesis-base'); +is_deeply([required_pkgs(\@all, 0, 1, 0)], [qw(elilo-xcat ipmitool-xcat xCAT-genesis-base)], + '--skip-perl drops perl-*'); +is_deeply([required_pkgs(\@all, 0, 0, 1)], [qw(perl-IO-Stty perl-Sys-Virt xCAT-genesis-base)], + '--skip-xcat-dep drops the dep builders'); +is_deeply([required_pkgs(\@all, 1, 1, 1)], [], + 'all skips -> nothing required (a clean skip run validates nothing)'); + +# ---- version_matches: exact + shell-glob pins ------------------------------------------------ +ok( version_matches('2.19.0', '2.*'), '2.* matches 2.19.0'); +ok( version_matches('2.18.2', '2.*'), '2.* matches 2.18.2 (walks with xcat-core)'); +ok(!version_matches('3.0.0', '2.*'), '2.* rejects 3.0.0'); +ok(!version_matches('20.0', '2.*'), '2.* rejects 20.0 (anchored, literal dot)'); +ok( version_matches('2.19.0', '2.19.*'), '2.19.* matches 2.19.0'); +ok(!version_matches('2.20.0', '2.19.*'), '2.19.* rejects 2.20.0'); +ok( version_matches('1.8.18', '1.8.18'), 'exact pin matches'); +ok(!version_matches('1.8.19', '1.8.18'), 'exact pin rejects a different version'); +ok( version_matches('anything', '*'), "'*' matches any version"); + +# ---- read_manifest: sections + entries ------------------------------------------------------- +{ + my $dir = tempdir(CLEANUP => 1); + my $f = "$dir/m.conf"; + open my $fh, '>', $f or die; + print $fh "# comment\n[alma+epel-8-x86_64]\nelilo-xcat=3.14\nxCAT-genesis-base=2.*\n\n" + . "[alma+epel-9-x86_64]\nperl-Sys-Virt=11.10.0\n"; + close $fh; + my %m = read_manifest($f); + is($m{'alma+epel-8-x86_64'}{'elilo-xcat'}, '3.14', 'read_manifest: exact pin'); + is($m{'alma+epel-8-x86_64'}{'xCAT-genesis-base'},'2.*', 'read_manifest: glob pin'); + is($m{'alma+epel-9-x86_64'}{'perl-Sys-Virt'}, '11.10.0', 'read_manifest: second section'); + is_deeply({read_manifest("$dir/nope.conf")}, {}, 'read_manifest: missing file -> empty'); +} + +# ---- RPM-identity comparison + cross_copy_genesis (needs rpmbuild for real rpms) -------------- +SKIP: { + skip 'rpmbuild not available', 5 if system('command -v rpmbuild >/dev/null 2>&1') != 0; + my $tmp = tempdir(CLEANUP => 1); + my $seq = 0; + my $mk = sub { # build a genesis-named rpm with a given marker payload + my ($tarch, $content) = @_; + my $out = "$tmp/out" . (++$seq); # unique dir: same NVR would overwrite in a shared one + my $spec = "$tmp/$tarch-$seq.spec"; + open my $fh, '>', $spec or die; + print $fh <<"SPEC"; +Name: xCAT-genesis-base-$tarch +Version: 2.19.0 +Release: snapTEST +Summary: test fixture +License: EPL +BuildArch: noarch +%description +test fixture +%install +mkdir -p %{buildroot}/opt/xcat/t +echo '$content' > %{buildroot}/opt/xcat/t/marker +%files +/opt/xcat/t/marker +SPEC + close $fh; + system("rpmbuild -bb --quiet --define '_topdir $tmp/rpmb$seq' --define '_rpmdir $out' " + . "'$spec' >/dev/null 2>&1") == 0 or die "rpmbuild failed for $tarch/$content"; + my ($rpm) = glob("$out/noarch/xCAT-genesis-base-$tarch-*.rpm"); + return $rpm; + }; + my $rpmA = $mk->('ppc64', 'CONTENT_A'); + my $rpmB = $mk->('ppc64', 'CONTENT_B_is_different'); # same NVR/basename, different payload + + isnt(rpm_sigmd5($rpmA), rpm_sigmd5($rpmB), + 'rpm_sigmd5 differs for same-name rpms with different content'); + + my $base = basename($rpmA); + my ($from, $to) = ("$tmp/from", "$tmp/to"); + make_path($from, $to); + system("cp '$rpmA' '$from/$base'"); # the fresh source + system("cp '$rpmB' '$to/$base'"); # a STALE dest rpm sharing the filename + + my $n = quiet { cross_copy_genesis($from, $to, 'ppc64', undef) }; + ok($n >= 1, "cross_copy refreshes a stale same-name rpm by content (copied=$n)"); + is(rpm_sigmd5("$to/$base"), rpm_sigmd5($rpmA), + 'after cross_copy the dest matches the source content'); + + my $n2 = quiet { cross_copy_genesis($from, $to, 'ppc64', undef) }; + is($n2, 0, 'cross_copy is a no-op when content is already identical (idempotent)'); + + # A signer callback is invoked for each copied rpm. + my ($from2, $to2) = ("$tmp/from2", "$tmp/to2"); + make_path($from2, $to2); + system("cp '$rpmA' '$from2/$base'"); + my @signed; + quiet { cross_copy_genesis($from2, $to2, 'ppc64', sub { push @signed, $_[0] }) }; + is_deeply(\@signed, ["$to2/$base"], 'the sign callback runs on each copied rpm'); +} + +# ---- finalize_xcat_dep: require the genesis inputs (no silent no-op) -------------------------- +{ + my $tmp = tempdir(CLEANUP => 1); + make_path("$tmp/x/rh9/x86_64", "$tmp/p/rh9/ppc64le"); # a pair exists, but NO genesis rpms + my $ok = eval { quiet { finalize_xcat_dep("$tmp/x", "$tmp/p") }; 1 }; + ok(!$ok, 'finalize dies when a repo pair has no genesis rpms (was a silent success)'); + like($@, qr/no (x86_64|ppc64) xCAT-genesis-base/, + 'finalize error names the missing genesis input'); + + my $tmp2 = tempdir(CLEANUP => 1); # no /x86_64 pair at all + make_path("$tmp2/x", "$tmp2/p"); + my $ok2 = eval { quiet { finalize_xcat_dep("$tmp2/x", "$tmp2/p") }; 1 }; + ok(!$ok2, 'finalize dies when no /x86_64 + /ppc64le pair is found'); +} + +done_testing; From e75b56a405d8bf8af8363b7fe0a6eeb1d3c33f72 Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Tue, 28 Jul 2026 17:23:39 -0300 Subject: [PATCH 24/55] fix(xcat-dep): harden finalize peer requirement + rpm version/identity edges Follow-up self-review hardening on top of the PR #62 review response: - finalize_xcat_dep now treats a missing ppc64le PEER repo (not just missing genesis rpms) as fatal instead of silently skipping the OS -- in the CD both arches build every EL, so a missing peer is an incomplete input that would otherwise leave the x86_64 repo without the ppc64 genesis and still exit 0. - cross_copy_genesis treats an empty SIGMD5 (unreadable rpm) as "cannot confirm identical" and refreshes, rather than risking a false up-to-date match when two unreadable rpms both return an empty digest. - rpm_version fails when a directory holds more than one distinct version of a package (a stale artifact not cleaned before the build) instead of silently returning the first sorted match, which a version pin could pass against while the stale rpm still ships. Both arches share a Version for genesis, so a normal x86_64+ppc64 pair is a single entry. t/mockbuild-all.t: +4 cases (30 total) -- missing-peer fatal, rpm_sigmd5 on a missing rpm, and rpm_version multi-version failure. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- MockBuildUtils.pm | 26 +++++++++++++++++++------- t/mockbuild-all.t | 27 +++++++++++++++++++++++---- 2 files changed, 42 insertions(+), 11 deletions(-) diff --git a/MockBuildUtils.pm b/MockBuildUtils.pm index 7924f6a..30e0cf3 100644 --- a/MockBuildUtils.pm +++ b/MockBuildUtils.pm @@ -84,6 +84,7 @@ sub rpm_version { my $glob = ($name eq 'xCAT-genesis-base') ? "$dir/xCAT-genesis-base-*.rpm" : "$dir/${name}-*.rpm"; + my %vers; # distinct %{version}s of the matching binary rpms for my $f (sort glob($glob)) { next if $f =~ /\.src\.rpm$/ || $f =~ /-debug(?:info|source)-/; my $n = `rpm -qp --qf '%{name}' ${\ sh_quote($f)} 2>/dev/null`; @@ -92,9 +93,16 @@ sub rpm_version { next unless $match; my $v = `rpm -qp --qf '%{version}' ${\ sh_quote($f)} 2>/dev/null`; chomp $v; - return $v; + $vers{$v} = 1 if $v ne ''; } - return undef; + return undef unless %vers; + # More than one distinct version present means a stale artifact was not cleaned before the + # build -- a version pin could then pass against the wrong rpm and both could be shipped. + # (For genesis both arches share the same Version, so a normal x86_64+ppc64 pair is one entry.) + die "Multiple versions of $name present in $dir: " . join(', ', sort keys %vers) + . " (stale artifact not cleaned before the build)\n" if keys(%vers) > 1; + my ($v) = keys %vers; + return $v; } # read_manifest: parse packages-manifest.conf into %{ target => { package => version|'*' } }. @@ -137,7 +145,10 @@ sub cross_copy_genesis { my $up_to_date = 1; for my $base (keys %want) { my $dst = "$to/$base"; - if (!-f $dst || rpm_sigmd5($want{$base}) ne rpm_sigmd5($dst)) { $up_to_date = 0; last; } + my $src_sig = rpm_sigmd5($want{$base}); + # An empty SIGMD5 (unreadable rpm) means "cannot confirm identical" -> refresh rather + # than risk skipping on a false match (two '' would otherwise compare equal). + if (!-f $dst || $src_sig eq '' || $src_sig ne rpm_sigmd5($dst)) { $up_to_date = 0; last; } } return 0 if $up_to_date; } @@ -175,10 +186,11 @@ sub finalize_xcat_dep { my $osdir = basename($p); my $x86dir = "$x86_64_repo/$osdir/x86_64"; my $ppcdir = "$ppc64le_repo/$osdir/ppc64le"; - if (!-d $ppcdir) { - print "[finalize] $osdir: no ppc64le peer at $ppcdir -- skipping\n"; - next; - } + # Require the peer repo itself: in the CD both arches build every EL, so a missing + # ppc64le peer for an x86_64 OS means an incomplete input, not something to skip past + # (skipping would leave that OS's x86_64 repo without the ppc64 genesis and still exit 0). + die "FATAL: [finalize] $osdir: no ppc64le peer repo at $ppcdir\n" + . " (both arches must build every EL before finalize)\n" if !-d $ppcdir; # Require the expected inputs: each arch's build must have produced its OWN genesis rpm # before finalize cross-populates them. Without this, a pair whose builds produced no # genesis rpms would make finalize a silent no-op that still exits 0 (the bug this guards). diff --git a/t/mockbuild-all.t b/t/mockbuild-all.t index 9f0b690..f43d032 100644 --- a/t/mockbuild-all.t +++ b/t/mockbuild-all.t @@ -10,7 +10,7 @@ use lib "$RealBin/.."; use File::Temp qw(tempdir); use File::Path qw(make_path); use File::Basename qw(basename); -use MockBuildUtils qw(required_pkgs version_matches rpm_sigmd5 +use MockBuildUtils qw(required_pkgs version_matches rpm_sigmd5 rpm_version cross_copy_genesis finalize_xcat_dep read_manifest); # Run a printing sub with STDOUT muted so its progress lines do not pollute TAP. @@ -64,19 +64,24 @@ ok( version_matches('anything', '*'), "'*' matches any version"); is_deeply({read_manifest("$dir/nope.conf")}, {}, 'read_manifest: missing file -> empty'); } +# rpm_sigmd5 on a missing/unreadable rpm returns '' (so cross_copy treats it as "not identical"). +is(rpm_sigmd5('/nonexistent/xCAT-genesis-base-ppc64-9.9.9.noarch.rpm'), '', + 'rpm_sigmd5 returns empty for a missing rpm'); + # ---- RPM-identity comparison + cross_copy_genesis (needs rpmbuild for real rpms) -------------- SKIP: { - skip 'rpmbuild not available', 5 if system('command -v rpmbuild >/dev/null 2>&1') != 0; + skip 'rpmbuild not available', 6 if system('command -v rpmbuild >/dev/null 2>&1') != 0; my $tmp = tempdir(CLEANUP => 1); my $seq = 0; my $mk = sub { # build a genesis-named rpm with a given marker payload - my ($tarch, $content) = @_; + my ($tarch, $content, $version) = @_; + $version ||= '2.19.0'; my $out = "$tmp/out" . (++$seq); # unique dir: same NVR would overwrite in a shared one my $spec = "$tmp/$tarch-$seq.spec"; open my $fh, '>', $spec or die; print $fh <<"SPEC"; Name: xCAT-genesis-base-$tarch -Version: 2.19.0 +Version: $version Release: snapTEST Summary: test fixture License: EPL @@ -122,6 +127,13 @@ SPEC my @signed; quiet { cross_copy_genesis($from2, $to2, 'ppc64', sub { push @signed, $_[0] }) }; is_deeply(\@signed, ["$to2/$base"], 'the sign callback runs on each copied rpm'); + + # rpm_version dies when a dir holds two DIFFERENT versions of the same package (stale artifact). + my $vdir = "$tmp/vers"; make_path($vdir); + system("cp '" . $mk->('ppc64', 'x', '2.19.0') . "' '$vdir/'"); + system("cp '" . $mk->('ppc64', 'x', '2.18.0') . "' '$vdir/'"); + my $vdied = !eval { rpm_version($vdir, 'xCAT-genesis-base'); 1 }; + ok($vdied, 'rpm_version dies when a dir holds multiple distinct versions of a package'); } # ---- finalize_xcat_dep: require the genesis inputs (no silent no-op) -------------------------- @@ -137,6 +149,13 @@ SPEC make_path("$tmp2/x", "$tmp2/p"); my $ok2 = eval { quiet { finalize_xcat_dep("$tmp2/x", "$tmp2/p") }; 1 }; ok(!$ok2, 'finalize dies when no /x86_64 + /ppc64le pair is found'); + + # A missing ppc64le PEER repo (not just missing rpms) is fatal, not a silent skip. + my $tmp3 = tempdir(CLEANUP => 1); + make_path("$tmp3/x/rh9/x86_64"); # x86_64 OS present, but NO ppc64le peer dir at all + my $ok3 = eval { quiet { finalize_xcat_dep("$tmp3/x", "$tmp3/p") }; 1 }; + ok(!$ok3, 'finalize dies when an x86_64 OS has no ppc64le peer repo (no silent skip)'); + like($@, qr/no ppc64le peer repo/, 'finalize error names the missing peer'); } done_testing; From ca1f2f3a366af9cbe443b7e2bb20aaa0d5cfb5e4 Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Thu, 30 Jul 2026 14:48:50 -0300 Subject: [PATCH 25/55] fix(xcat-dep): stop mid-batch bootstrap scrub racing concurrent perl builds The parallel perl-package builder scrubbed each package's chroot AND bootstrap (`mock --uniqueext --scrub=chroot --scrub=bootstrap`) as soon as that package finished, mid-batch. The comment assumed the --uniqueext made this sibling-safe, but mock's bootstrap scrub ignores --uniqueext and removes the CONFIG-LEVEL shared bootstrap cache (/var/cache/mock/-bootstrap/, keyed by config name, not uniqueext). Under concurrency a faster sibling's post-build scrub deleted that shared cache while a slower sibling was still setting up its buildsrpm chroot and about to bind-mount it, so the bind failed with mount rc=32 and the whole target failed a build that was otherwise fine. Observed: on alma+epel-10-ppc64le, perl-Sys-Virt (the slowest, a libvirt C binding) died at --buildsrpm binding /var/cache/mock/alma+epel-10-ppc64le-bootstrap/yum_cache ~1s after two faster siblings had just scrubbed that shared bootstrap; the other five perl packages passed. Non-deterministic and load-triggered, so it surfaced under the 3-way concurrent CD load. Reclaim only the (uniqueext-local) build chroot per package during the batch, and defer the shared bootstrap reclamation to a single serialized pass after all workers join, when nothing can be binding it. Disk reclamation is preserved (the ~GB build chroots are still freed immediately; the bootstrap roots + shared cache are freed at batch end). Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- mockbuild-perl-packages.pl | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/mockbuild-perl-packages.pl b/mockbuild-perl-packages.pl index c31aac9..ecd0260 100755 --- a/mockbuild-perl-packages.pl +++ b/mockbuild-perl-packages.pl @@ -233,20 +233,35 @@ for my $idx (0 .. $#packages) { release_suffix => $release_suffix, ); unless ($keep_buildroots) { - # Reclaim this perl package's chroot AND its per-uniqueext bootstrap via mock's lock-safe - # --scrub (never rm). Each package has its own uniqueext (hence its own chroot+bootstrap), - # so scrubbing one never touches a sibling's concurrent build. Runs regardless of build - # result so failed chroots are reclaimed too; the root cache is kept for fast rebuilds. - # The orchestrator can't name these chroots (package_uniqueext derives them), so we scrub - # here. + # Reclaim ONLY this package's build chroot here (it's the ~GB disk hog). --scrub=chroot is + # uniqueext-local, so it never touches a concurrent sibling. Do NOT --scrub=bootstrap here: + # despite the --uniqueext, mock's bootstrap scrub removes the CONFIG-LEVEL shared bootstrap + # cache (/var/cache/mock/-bootstrap/, keyed by config name, NOT uniqueext). Doing that + # mid-batch deletes the cache a still-starting sibling is about to bind-mount into its own + # bootstrap root -> `mount rc=32` and a spurious build failure (observed: perl-Sys-Virt's + # buildsrpm raced a faster sibling's post-build bootstrap scrub). The shared bootstrap is + # reclaimed once below, after ALL workers finish, when nothing can be binding it. (my $ps = $pkg) =~ s/[^\w.-]+/-/g; system("mock -r " . sh_quote($mock_cfg) . " --uniqueext " . sh_quote($pkg_uniqueext) - . " --scrub=chroot --scrub=bootstrap > " . sh_quote("$log_dir/scrub-$ps.log") . " 2>&1"); + . " --scrub=chroot > " . sh_quote("$log_dir/scrub-$ps.log") . " 2>&1"); } $pm->finish($ok ? 0 : 1); } $pm->wait_all_children; +# Now that every worker has exited, reclaim the per-uniqueext bootstrap roots + the shared +# config-level bootstrap cache. Serialized and post-join, so no scrub can race a concurrent +# bind (that race is exactly what the per-package note above avoids). Best-effort: the first +# scrub drops /var/cache/mock/-bootstrap; each also removes its uniqueext bootstrap root. +unless ($keep_buildroots) { + for my $idx (0 .. $#packages) { + my $pkg_uniqueext = package_uniqueext($mock_uniqueext, $idx + 1, $packages[$idx]); + (my $ps = $packages[$idx]) =~ s/[^\w.-]+/-/g; + system("mock -r " . sh_quote($mock_cfg) . " --uniqueext " . sh_quote($pkg_uniqueext) + . " --scrub=bootstrap >> " . sh_quote("$log_dir/scrub-$ps.log") . " 2>&1"); + } +} + for my $pkg (@packages) { my $status_file = "$log_dir/$pkg/status.txt"; if (!-f $status_file) { From 40feffc8ce60f23417c64f7ca19889df41ad6048 Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Mon, 10 Aug 2026 11:23:04 -0300 Subject: [PATCH 26/55] fix(mockbuild-all): address PR #62 review (build-number, finalize, run-state, skip-build, goconserver, docs) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reviewed by @viniciusferrao. Each numbered point below is his; the code changes verify + fix it. 1. --build-number over-reach / dry-run / double-stamp - The bump now runs ONLY on a real build: `--dry-run --build-number N` prints what it would stamp and writes nothing (previously it rewrote every spec on disk during a dry run). - Re-stamping is idempotent AND replacing: a re-run in a reused tree with a different --build-number strips the prior .snap. before applying the new one, instead of accumulating a second stamp (…snap57.snap58). Extracted the per-line logic into the unit-testable MockBuildUtils::restamp_release_line and covered it in t/mockbuild-all.t. (The headline "rewrites xcat-core / xCAT-genesis-base.spec" does not occur in the real layout: xcat-core is a sibling of $repo_root, and there is no genesis spec under the dep tree. The legacy nested xcat-source-code case remains a non-CD layout; left as a follow-up.) 3. --finalize-xcat-dep idempotency - cross_copy_genesis compared only SIGMD5 (content), which is blind to signature + index state. It now also treats a same-content-but-UNSIGNED destination rpm as not-up-to-date (new rpm_is_signed helper) so a crash between copy and sign heals on re-run. - finalize_xcat_dep now re-indexes+signs BOTH repos of a touched pair every run, not only when an rpm was copied, so a crash after copy+sign but before createrepo (rpm on disk, absent from repomd) also heals. 4. Stale run-state can mask a failed build - A real build now wipes its per-target $run_root first (run_id is derived from the deterministic commit time, so re-runs reused the same tree). --skip-build keeps the tree; --dry-run writes nothing. - mockbuild-perl-packages.pl clears each package's stale status.txt/error.txt BEFORE building, and the aggregate now treats the child worker's exit code as authoritative: a package is PASS only if its worker exited 0 AND wrote a PASS this run (a stale PASS in a reused log dir no longer counts). 5. --skip-build can publish the wrong artifacts - --skip-build now REQUIRES an explicit --target (without it, all three EL targets collected the same EL-agnostic roots and cross-published them). - Collection is scoped to this target's own per-target $build_root (the same tree a normal build populates), not the legacy build-output/list3/list5/list6 dirs. - The manifest version-pin validation (and the "no manifest section" guard) now also run under --skip-build, so a collection-only publish is validated exactly like a fresh build. 2. goconserver bypassed the CD bump (minimal fix; hermetic rebuild deferred) - goconserver/mockbuild.pl gains --release-suffix, appended to its generated `Release: 4.elN`, and mockbuild-all.pl passes the CD suffix down -- so goconserver's NVR advances per run like every other dep package (an additive publish is no longer a silent no-op on a frozen NVR). - Pinned the clone to an immutable upstream commit instead of the moving `master` (0.3.3 is unreleased -- newest tag is v0.3.2 -- so a SHA pin is required; clone now fetches by ref). - The host build + `go mod tidy` hermeticity concern is a tracked follow-up, not in this change. 7. Docs - BUILD.md: --target is a single value, not repeatable; conserver-xcat is built for every target (not "not required"). POD: --parallel-targets default is 1 = serial, not "auto". - Added a manifest<->docs consistency test (conserver-xcat present in every target section). (6, --max-parallel not a true global cap, is a documented nice-to-have and is left as a follow-up.) Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- BUILD.md | 11 ++++-- MockBuildUtils.pm | 44 ++++++++++++++++++++-- goconserver/mockbuild.pl | 20 +++++++--- mockbuild-all.pl | 75 +++++++++++++++++++++++++++----------- mockbuild-perl-packages.pl | 16 ++++++++ t/mockbuild-all.t | 53 ++++++++++++++++++++++++++- 6 files changed, 184 insertions(+), 35 deletions(-) diff --git a/BUILD.md b/BUILD.md index e74a6a5..c3ea15b 100644 --- a/BUILD.md +++ b/BUILD.md @@ -60,9 +60,10 @@ Each build path uses `mock` for chroot isolation. Top-level steps are paralleliz `packages-manifest.conf` (repo root) declares, per target, exactly which packages are required — one `[]` section (matching `--target`, e.g. `[alma+epel-10-x86_64]`) of `=` lines. For each target, `mockbuild-all.pl` builds **only** the packages -listed for it; a package absent from a target's section is not built for that target (e.g. -`conserver-xcat` is not required by any target, and the per-EL perl set differs because the OS/ -EPEL already provides some modules). The lists were derived empirically — on a clean MN of each +listed for it; a package absent from a target's section is not built for that target (the per-EL +perl set differs because the OS/EPEL already provides some modules). Note `conserver-xcat` is **not** +pulled in by `dnf install xCAT` (goconserver superseded it), yet it is listed in — and therefore +built for — every target, because some deployments still use it. The lists were derived empirically — on a clean MN of each (EL, arch), `dnf install xCAT` from xcat.org latest, and the packages whose `from_repo=xcat-dep` are exactly the required set. See the file header for details. @@ -163,7 +164,9 @@ mock -r ... ``` By default (no `--target`), `mockbuild-all.pl` builds all three EL releases for the host -arch: `rh8`, `rh9`, and `rh10`. Pass `--target` (repeatable) to restrict the set. +arch: `rh8`, `rh9`, and `rh10`. Pass `--target ` to build a single target instead; it +takes **one** value and is not repeatable — run the script once per target to build several, or +omit it to build all three. # Build the Dependency Repository diff --git a/MockBuildUtils.pm b/MockBuildUtils.pm index 30e0cf3..a121900 100644 --- a/MockBuildUtils.pm +++ b/MockBuildUtils.pm @@ -12,7 +12,7 @@ use File::Copy qw(copy); our @EXPORT_OK = qw( sh_quote print_step version_matches required_pkgs have_rpm read_manifest - rpm_version rpm_sigmd5 + rpm_version rpm_sigmd5 rpm_is_signed restamp_release_line cross_copy_genesis finalize_xcat_dep ); @@ -76,6 +76,36 @@ sub rpm_sigmd5 { return $v; } +# rpm_is_signed: does the rpm carry a PGP/GPG header signature? SIGMD5 (above) is content-only and +# is identical whether or not the rpm is signed, so a cross-copied genesis that was copied but not +# yet signed (a crash between the copy and the rpmsign) still matches by SIGMD5. finalize uses this +# to treat such a rpm as NOT up to date so the copy+sign path re-runs and heals it. +sub rpm_is_signed { + my ($f) = @_; + return 0 unless defined $f && -f $f; + my $v = `rpm -qp --qf '%{SIGPGP}%{SIGGPG}' ${\ sh_quote($f)} 2>/dev/null`; + return 0 if !defined $v; + $v =~ s/\(none\)//g; # unsigned rpms report "(none)" for both tags + $v =~ s/\s+//g; + return $v ne '' ? 1 : 0; +} + +# restamp_release_line: given a spec `Release: ...` line and a CD suffix (".snap."), +# return (new_line, changed). Idempotent: a line already ending in exactly $suffix is returned +# unchanged (changed=0). A line carrying a DIFFERENT prior .snap stamp (or several, from an earlier +# corrupted run) has it stripped before the new suffix is appended, so a re-run in a reused tree +# REPLACES the stamp instead of accumulating a second one (…snap...57 -> …snap...58, never +# …snap...57.snap...58). Only the Release token is touched; a non-Release line is returned as-is. +sub restamp_release_line { + my ($line, $suffix) = @_; + return ($line, 0) unless defined $line && $line =~ /^Release:\s*\S/i; + my $qs = quotemeta($suffix); + return ($line, 0) if $line =~ /$qs\s*$/; # already carries THIS suffix + (my $new = $line) =~ s/(?:\.snap\d{12}\.\d+)+(\s*)$/$1/; # drop any prior CD stamp(s) + $new =~ s/(^Release:\s*\S+)/$1$suffix/i; + return ($new, 1); +} + # rpm_version: %{version} of the built binary rpm named under $dir (undef if absent). # Skips src/debug rpms and confirms the rpm's real %{name} matches (glob can over-match). # 'xCAT-genesis-base' matches the arch-suffixed rpm name (xCAT-genesis-base-x86_64 / -ppc64). @@ -149,6 +179,10 @@ sub cross_copy_genesis { # An empty SIGMD5 (unreadable rpm) means "cannot confirm identical" -> refresh rather # than risk skipping on a false match (two '' would otherwise compare equal). if (!-f $dst || $src_sig eq '' || $src_sig ne rpm_sigmd5($dst)) { $up_to_date = 0; last; } + # Content matches, but SIGMD5 cannot see the signature: a crash between the copy and the + # per-rpm sign leaves a same-content-but-UNSIGNED rpm. When a signer is configured, treat + # an unsigned dst as not-up-to-date so the copy+sign path re-runs and signs it. + if ($sign && !rpm_is_signed($dst)) { $up_to_date = 0; last; } } return 0 if $up_to_date; } @@ -202,8 +236,12 @@ sub finalize_xcat_dep { # named xCAT-genesis-base-ppc64-*. Cross-copy both directions. my $to_x86 = cross_copy_genesis($ppcdir, $x86dir, 'ppc64', $sign); my $to_ppc = cross_copy_genesis($x86dir, $ppcdir, 'x86_64', $sign); - $reindex->($x86dir) if $to_x86 && $reindex; - $reindex->($ppcdir) if $to_ppc && $reindex; + # Re-index+sign BOTH repos of the pair every finalize, not only when an rpm was copied this + # run. A crash after a prior run's copy+sign but before its createrepo leaves the genesis rpm + # on disk (so cross_copy_genesis now returns 0) yet ABSENT from repomd.xml -- which no + # signature gate catches. Re-indexing is cheap (these are tiny repos) and idempotent, and it + # heals that partial state; skipped only when no signer/indexer was injected. + if ($reindex) { $reindex->($x86dir); $reindex->($ppcdir); } printf "[finalize] %s: %d ppc64 genesis -> x86_64, %d x86_64 genesis -> ppc64le\n", $osdir, $to_x86, $to_ppc; $pairs++; diff --git a/goconserver/mockbuild.pl b/goconserver/mockbuild.pl index 572743e..e152b09 100755 --- a/goconserver/mockbuild.pl +++ b/goconserver/mockbuild.pl @@ -20,7 +20,11 @@ my $log_dir = "$repo_root/build-logs/list5/goconserver"; my $skip_install = 0; my $version = '0.3.3'; my $go_repo = 'https://github.com/xcat2/goconserver.git'; -my $go_ref = 'master'; +# Immutable pin: goconserver 0.3.3 is unreleased (newest tag v0.3.2) so it lives only on master. +# mockbuild-all.pl passes --go-ref with the canonical pin; this default keeps standalone runs +# reproducible too. `git clone --branch` cannot take a raw SHA, so the clone below fetches by ref. +my $go_ref = '6166fe5ec1c5b3c20475e322a9f0e8e93c87e45f'; +my $release_suffix = ''; # CD Release bump (".snap."); passed by mockbuild-all.pl my $build_timestamp; GetOptions( @@ -33,6 +37,7 @@ GetOptions( 'version=s' => \$version, 'go-repo=s' => \$go_repo, 'go-ref=s' => \$go_ref, + 'release-suffix=s' => \$release_suffix, 'build-timestamp=i' => \$build_timestamp, ) or die usage(); @@ -100,9 +105,14 @@ for my $d (qw(BUILD BUILDROOT RPMS SOURCES SPECS SRPMS)) { print_step("Clone goconserver source"); my $src_dir = "$work_dir/goconserver-src"; -run("git clone --depth 1 --branch " . sh_quote($go_ref) . " " . - sh_quote($go_repo) . " " . sh_quote($src_dir) . - " >" . sh_quote("$log_dir/git-clone.log") . " 2>&1"); +# Fetch the exact pinned ref (a SHA, or a branch/tag). `git clone --branch` rejects a raw SHA, so +# init + shallow fetch the one object + checkout it -- reproducible and immutable, never "latest +# master". (xcat2/goconserver has allowReachableSHA1InWant, so fetching a master-reachable SHA works.) +my $clone_log = sh_quote("$log_dir/git-clone.log"); +run("git init -q " . sh_quote($src_dir) . " >$clone_log 2>&1"); +run("git -C " . sh_quote($src_dir) . " remote add origin " . sh_quote($go_repo) . " >>$clone_log 2>&1"); +run("git -C " . sh_quote($src_dir) . " fetch --depth 1 origin " . sh_quote($go_ref) . " >>$clone_log 2>&1"); +run("git -C " . sh_quote($src_dir) . " checkout -q FETCH_HEAD >>$clone_log 2>&1"); # etcd storage backend has broken deps with modern Go modules; # xCAT only uses file storage, so remove etcd before building. @@ -198,7 +208,7 @@ print_step("Create spec and build RPM"); my $spec_content = <<"SPEC"; Name: goconserver Version: $version -Release: 4.el$rel +Release: 4.el$rel$release_suffix Summary: Console server written in Go for xCAT License: EPL-1.0 URL: https://github.com/xcat2/goconserver diff --git a/mockbuild-all.pl b/mockbuild-all.pl index 0ae4ff2..c439093 100755 --- a/mockbuild-all.pl +++ b/mockbuild-all.pl @@ -7,14 +7,15 @@ use Cwd qw(abs_path cwd); use File::Basename qw(dirname basename); use File::Copy qw(copy); use File::Find qw(find); -use File::Path qw(make_path); +use File::Path qw(make_path remove_tree); use Getopt::Long qw(GetOptions); use Parallel::ForkManager; use POSIX qw(strftime); use FindBin qw($RealBin); use lib $RealBin; use MockBuildUtils qw(sh_quote print_step version_matches required_pkgs have_rpm - read_manifest rpm_version rpm_sigmd5 cross_copy_genesis finalize_xcat_dep); + read_manifest rpm_version rpm_sigmd5 restamp_release_line + cross_copy_genesis finalize_xcat_dep); my $script_dir = abs_path(dirname(__FILE__)); my $repo_root = abs_path($script_dir); @@ -37,6 +38,10 @@ my $build_timestamp; # fresh, monotonic NVR (deploy's additive rsync is a no-op otherwise). NOT applied # to xCAT-genesis-base (built from xcat-core, kept in lockstep with genesis-scripts). my $build_number; +# Pinned goconserver upstream commit (xcat2/goconserver). goconserver 0.3.3 is unreleased (the +# newest tag is v0.3.2), so it exists only on master -- pin an immutable SHA instead of the moving +# branch so the build is reproducible. Bump this deliberately when uptaking a new goconserver. +my $GOCONSERVER_REF = '6166fe5ec1c5b3c20475e322a9f0e8e93c87e45f'; my $skip_install = 0; my $skip_build = 0; my $skip_xcat_dep = 0; @@ -96,6 +101,11 @@ GetOptions( ) or die usage(); die "Run as root (uid=$>)\n" if $> != 0 && !$finalize_xcat_dep; +# --skip-build collects a prior build's artifacts from that build's per-target tree, so it must +# know the target. Without --target the default is "all three EL targets", and each would collect +# the same artifacts and cross-publish them into every repo (foreign-EL / foreign-arch rpms). +die "--skip-build requires an explicit --target (collection is per-target)\n" + if $skip_build && $target eq ''; die "--parallel-builds must be >= 1\n" if defined($parallel_builds) && $parallel_builds < 1; @@ -157,7 +167,13 @@ my $RELEASE_BUMP = ''; if (defined $build_number) { die "--build-number must be a non-negative integer\n" if $build_number < 0; $RELEASE_BUMP = strftime('.snap%Y%m%d%H%M', gmtime($SOURCE_DATE_EPOCH)) . ".$build_number"; - bump_dep_release_suffix($repo_root, $RELEASE_BUMP); + # A dry run must not touch the tree. Report what would be stamped and leave the specs alone; + # $RELEASE_BUMP is still set so the rest of the (no-op) dry-run plan reflects it. + if ($dry_run) { + print "[dry-run] would stamp Release suffix '$RELEASE_BUMP' on xcat-dep specs under $repo_root (no files written)\n"; + } else { + bump_dep_release_suffix($repo_root, $RELEASE_BUMP); + } } # Single output base for every NFS-shared write. Two hosts build in parallel on one NFS by @@ -258,7 +274,6 @@ exit 0; # does not double-stamp). Preserves any %{?dist}/%{?distver} macro already on the line. sub bump_dep_release_suffix { my ($root, $suffix) = @_; - my $qs = quotemeta($suffix); my @specs; find(sub { push @specs, $File::Find::name if /\.spec$/ && -f $_ }, $root); my ($with_release, $bumped, $already) = (0, 0, 0); @@ -271,9 +286,11 @@ sub bump_dep_release_suffix { # case-insensitive: some specs (e.g. Sys-Virt.spec) use a lowercase `release:` next unless $line =~ /^Release:\s*\S/i; $has_release = 1; - if ($line =~ /$qs\s*$/) { last } # already stamped (idempotent / concurrent arch) - $line =~ s/(^Release:\s*\S+)/$1$suffix/i; - $changed = 1; + # restamp_release_line is idempotent (no-op if already carrying $suffix) and strips any + # prior .snap stamp before applying the new one, so a re-run with a different + # --build-number replaces rather than accumulates (unit-tested in t/mockbuild-all.t). + my ($new, $ch) = restamp_release_line($line, $suffix); + if ($ch) { $line = $new; $changed = 1; } last; # only the first Release: line } $with_release++ if $has_release; @@ -312,7 +329,7 @@ sub build_one_target { my %MANIFEST = read_manifest("$repo_root/packages-manifest.conf"); my %req = %{ $MANIFEST{$target} // {} }; die "FATAL: no manifest section for target '$target' in packages-manifest.conf\n" - if !$skip_build && !%req; + if !%req; my $run_root = "$output_root/$run_id"; my $build_root = "$run_root/build-results"; @@ -323,6 +340,16 @@ my $tarball = "$output_root/mockbuild-all-$target-$run_id.tar.gz"; my $srpm_repo_dir = "$run_root/repo-src"; my $srpm_tarball = "$output_root/mockbuild-all-$target-$run_id-srpm.tar.gz"; +# Each real build must start from a clean per-target tree. run_id is derived from the deterministic +# commit timestamp, so re-runs of the same commit resolve to the SAME $run_root -- without a wipe, a +# stale rpm or a stale perl status.txt from an earlier (possibly failed) run could be reused and mask +# a failure (see mockbuild-perl-packages.pl, which reads per-package status files back). --skip-build +# deliberately KEEPS the tree (it collects a prior build's artifacts); --dry-run writes nothing. +if (!$skip_build && !$dry_run && -d $run_root) { + print "Cleaning stale per-target tree before build: $run_root\n"; + remove_tree($run_root); +} + # All dep builders run natively on every arch. xnba-undi and grub2-xcat are noarch packagings of # committed artifacts (an x86 UNDI ROM / the grub2 resource tarball) with no arch-specific build # step, so ppc builds them the same as x86 -- no cross-arch import. @@ -417,6 +444,14 @@ if (!$skip_build) { '--work-dir', sh_quote("/tmp/mockbuild-all-$run_id/$name"), '--build-timestamp', $SOURCE_DATE_EPOCH, ($skip_install ? '--skip-install' : ()), + # goconserver generates its spec at build time (from an upstream clone), so the + # in-tree spec Release bump above cannot reach it. Hand the CD suffix down so its + # NVR advances per run too, and pin the clone to an immutable commit (not the moving + # 'master') so the build is reproducible. + ($name eq 'goconserver' + ? ('--go-ref', sh_quote($GOCONSERVER_REF), + ($RELEASE_BUMP ne '' ? ('--release-suffix', sh_quote($RELEASE_BUMP)) : ())) + : ()), ); push @build_steps, { id => "xcat-dep:$name", @@ -545,16 +580,12 @@ if (!$skip_build) { my $xcat_rpms_dir = "$xcat_src/dist/$target/rpms"; if ($skip_build) { - push @collect_roots, - "$repo_root/build-output/list3/elilo-xcat", - "$repo_root/build-output/list3/grub2-xcat", - "$repo_root/build-output/list3/ipmitool-xcat", - "$repo_root/build-output/list3/syslinux-xcat", - "$repo_root/build-output/list3/xnba-undi", - "$repo_root/build-output/list5/goconserver/$arch", - "$repo_root/goconserver-build-$arch/results/rpm", - "$repo_root/build-output/list6/perl/$arch", - "$repo_root/perl-list6/$arch"; + # Collect THIS target's previously-built artifacts from its own per-target build tree -- the + # same $build_root a normal build populates (collect_rpms recurses). NOT the legacy EL-agnostic + # build-output/list* dirs: those are scoped only by $arch, so an el8 rpm left there would be + # pulled into an el9/el10 repo, and with --target omitted the same rpms would be published into + # every EL repo. (--target is now required for --skip-build, see the option check above.) + push @collect_roots, $build_root; } push @collect_roots, @extra_collect_dirs; @@ -590,8 +621,10 @@ if (!$skip_genesis && !$dry_run) { # Manifest version pins: every required package must be present at its pinned version. A build # that produces a different version (a source version bump not reflected here) fails the run; # a manifest value of '*' accepts any version. Only the Version is pinned, not the Release -# (which carries the per-EL dist tag and the genesis snap timestamp). -if (!$dry_run && !$skip_build) { +# (which carries the per-EL dist tag and the genesis snap timestamp). This also runs under +# --skip-build so a collection-only publish is validated against the target's manifest exactly +# like a fresh build (a stale/foreign-arch collected rpm fails here instead of shipping). +if (!$dry_run) { my @vmiss; # Only validate packages whose builder was NOT skipped -- so a clean --skip-* run does not # fail on packages it deliberately did not build. @@ -856,7 +889,7 @@ Options: --nproc N Parallel jobs for buildrpms.pl (default: 1) --parallel-builds N Max concurrent top-level build steps within one EL target (default: auto) --parallel-targets N Concurrent EL targets (rh8/rh9/rh10). 0/auto = all at once, 1 = serial, - N = cap at N. Each target is fully output-isolated (default: auto) + N = cap at N. Each target is fully output-isolated (default: 1 = serial) --max-parallel N Global cap on concurrent mock builds across ALL targets, to avoid oversubscribing the host. Split evenly across active targets. 0/auto = host nproc (default: auto) diff --git a/mockbuild-perl-packages.pl b/mockbuild-perl-packages.pl index ecd0260..531ca81 100755 --- a/mockbuild-perl-packages.pl +++ b/mockbuild-perl-packages.pl @@ -202,11 +202,13 @@ my @summary_lines; print_step("Build packages"); print "parallel jobs: $jobs\n"; +my %child_rc; # pkg => child exit code; the AUTHORITATIVE pass/fail for that build my $pm = Parallel::ForkManager->new($jobs); $pm->run_on_finish( sub { my ($pid, $exit_code, $ident) = @_; my $label = defined $ident ? $ident : "pid=$pid"; + $child_rc{$ident} = $exit_code if defined $ident; # record it; do not trust status.txt alone my $state = $exit_code == 0 ? 'PASS' : "FAIL(rc=$exit_code)"; print "[$label] $state\n"; } @@ -264,6 +266,17 @@ unless ($keep_buildroots) { for my $pkg (@packages) { my $status_file = "$log_dir/$pkg/status.txt"; + # The child exit code is authoritative: a package is PASS only if its worker exited 0 AND wrote + # a PASS status this run. A missing/non-zero child result is FAIL regardless of any status.txt + # (which could be a stale PASS left in a reused log dir, or unwritten because the worker crashed). + my $rc = $child_rc{$pkg}; + if (!defined $rc || $rc != 0) { + push @failed, $pkg; + push @summary_lines, defined $rc + ? "$pkg FAIL worker exited rc=$rc" + : "$pkg FAIL no worker result recorded"; + next; + } if (!-f $status_file) { push @failed, $pkg; push @summary_lines, "$pkg FAIL missing status file ($status_file)"; @@ -328,6 +341,9 @@ sub build_package { make_path($pkg_run_dir); make_path($pkg_result); make_path($pkg_log); + # Clear any status/error left by an earlier run in a reused log dir BEFORE building, so a crash + # between here and the status write below can never leave a stale PASS the aggregate would trust. + unlink $status_file, "$pkg_log/error.txt"; my $det_mock_cfg = create_deterministic_mock_cfg($mock_cfg, $SOURCE_DATE_EPOCH, $pkg_run_dir); diff --git a/t/mockbuild-all.t b/t/mockbuild-all.t index f43d032..20fdbe3 100644 --- a/t/mockbuild-all.t +++ b/t/mockbuild-all.t @@ -10,8 +10,8 @@ use lib "$RealBin/.."; use File::Temp qw(tempdir); use File::Path qw(make_path); use File::Basename qw(basename); -use MockBuildUtils qw(required_pkgs version_matches rpm_sigmd5 rpm_version - cross_copy_genesis finalize_xcat_dep read_manifest); +use MockBuildUtils qw(required_pkgs version_matches rpm_sigmd5 rpm_version rpm_is_signed + restamp_release_line cross_copy_genesis finalize_xcat_dep read_manifest); # Run a printing sub with STDOUT muted so its progress lines do not pollute TAP. sub quiet(&) { @@ -158,4 +158,53 @@ SPEC like($@, qr/no ppc64le peer repo/, 'finalize error names the missing peer'); } +# ---- restamp_release_line: CD --build-number Release stamping (PR #62 review point 1) ---------- +# A fresh stamp is appended after the Release token, preserving any %{?dist} macro. +{ + my ($l, $ch) = restamp_release_line("Release: 1%{?dist}\n", '.snap202607161200.57'); + is($l, "Release: 1%{?dist}.snap202607161200.57\n", 'stamps a fresh Release, macro preserved'); + is($ch, 1, 'reports changed'); +} +# Idempotent: the exact same suffix is a no-op (concurrent per-arch build / same-tree re-run). +{ + my $line = "Release: 1%{?dist}.snap202607161200.57\n"; + my ($l, $ch) = restamp_release_line($line, '.snap202607161200.57'); + is($l, $line, 're-stamping the SAME suffix is a no-op'); + is($ch, 0, 'reports unchanged'); +} +# A DIFFERENT build-number REPLACES the prior stamp (does not accumulate) -- the double-stamp bug. +{ + my ($l, $ch) = restamp_release_line("Release: 1%{?dist}.snap202607161200.57\n", '.snap202607161200.58'); + is($l, "Release: 1%{?dist}.snap202607161200.58\n", 'a new build-number replaces the old stamp'); + is($ch, 1, 'reports changed'); + unlike($l, qr/\.snap\d{12}\.\d+\.snap/, 'never leaves two stacked .snap stamps'); +} +# Even an already-corrupted (double-stamped) line is healed back to a single stamp. +{ + my ($l) = restamp_release_line("Release: 5.snap202601010000.1.snap202601020000.2\n", '.snap202607161200.9'); + is($l, "Release: 5.snap202607161200.9\n", 'strips multiple stacked prior stamps before re-stamping'); +} +# A non-Release line is never touched. +{ + my ($l, $ch) = restamp_release_line("Version: 0.3.3\n", '.snap202607161200.57'); + is($l, "Version: 0.3.3\n", 'non-Release line untouched'); + is($ch, 0, 'reports unchanged'); +} + +# ---- rpm_is_signed: unreadable / missing -> not signed (used by the finalize idempotency fix) --- +is(rpm_is_signed(undef), 0, 'rpm_is_signed(undef) is 0'); +is(rpm_is_signed("/no/such/file.rpm"), 0, 'rpm_is_signed on a missing file is 0'); + +# ---- manifest <-> docs consistency: conserver-xcat is in EVERY target section (PR #62 point 7c) -- +# BUILD.md documents conserver-xcat as built for every target; guard that the manifest agrees so the +# doc and the manifest can never silently drift apart again. +{ + my %m = read_manifest("$RealBin/../packages-manifest.conf"); + my @targets = sort keys %m; + cmp_ok(scalar(@targets), '>=', 1, 'packages-manifest.conf has at least one target section'); + my @missing = grep { !exists $m{$_}{'conserver-xcat'} } @targets; + is_deeply(\@missing, [], 'conserver-xcat is present in every manifest target section') + or diag("missing conserver-xcat in: @missing"); +} + done_testing; From b2bd440ba0818e468fcaa0c214a61b74a5f22dec Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Mon, 10 Aug 2026 12:46:26 -0300 Subject: [PATCH 27/55] fix(mockbuild-all): address remaining PR #62 review nits (1, 4, 6) Follow-up to 40feffc, addressing the nice-to-have / secondary points from @viniciusferrao's review. 1 (over-reach): the --build-number spec walk now PRUNES a nested `xcat-core`/`xcat-source-code` checkout under $repo_root, so the legacy nested layout can no longer rewrite an xCAT-core spec (e.g. xCAT-genesis-base.spec's dynamic Release). In the normal sibling layout nothing changes. 4 (validate %RELEASE): after the manifest %VERSION pins, when a CD --build-number bump is in effect the run now also asserts the bump actually LANDED in each built dep/perl rpm's %RELEASE (genesis excluded -- it is intentionally not bumped). Catches a silently un-bumped NVR that a Version-only check misses. New MockBuildUtils::rpm_release helper. 6 (--max-parallel a real cap): the perl builder internally forks up to $effective_parallel_builds mock jobs, so running it concurrently with the dep builders let live mock builds reach ~2x the cap. Run the perl builder in its OWN phase, after the (quick) dep builders -- each phase then runs at most $effective_parallel_builds mock builds, so --max-parallel holds, at a small bounded cost. Tests: 45/45 in t/mockbuild-all.t (rpm_release added). Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- MockBuildUtils.pm | 23 ++++++++++++++++++++++- mockbuild-all.pl | 47 +++++++++++++++++++++++++++++++++++++++++------ t/mockbuild-all.t | 5 ++++- 3 files changed, 67 insertions(+), 8 deletions(-) diff --git a/MockBuildUtils.pm b/MockBuildUtils.pm index a121900..d137604 100644 --- a/MockBuildUtils.pm +++ b/MockBuildUtils.pm @@ -12,7 +12,7 @@ use File::Copy qw(copy); our @EXPORT_OK = qw( sh_quote print_step version_matches required_pkgs have_rpm read_manifest - rpm_version rpm_sigmd5 rpm_is_signed restamp_release_line + rpm_version rpm_release rpm_sigmd5 rpm_is_signed restamp_release_line cross_copy_genesis finalize_xcat_dep ); @@ -135,6 +135,27 @@ sub rpm_version { return $v; } +# rpm_release: %{release} of the built binary rpm named under $dir (undef if absent). Same +# name-matching as rpm_version. Used to confirm a CD --build-number/--release-suffix bump actually +# landed in the built rpm's Release (validating %{VERSION} alone can't catch a silently un-bumped NVR). +sub rpm_release { + my ($dir, $name) = @_; + my $glob = ($name eq 'xCAT-genesis-base') + ? "$dir/xCAT-genesis-base-*.rpm" + : "$dir/${name}-*.rpm"; + for my $f (sort glob($glob)) { + next if $f =~ /\.src\.rpm$/ || $f =~ /-debug(?:info|source)-/; + my $n = `rpm -qp --qf '%{name}' ${\ sh_quote($f)} 2>/dev/null`; + my $match = ($name eq 'xCAT-genesis-base') + ? ($n =~ /^xCAT-genesis-base-/) : ($n eq $name); + next unless $match; + my $r = `rpm -qp --qf '%{release}' ${\ sh_quote($f)} 2>/dev/null`; + chomp $r; + return $r if $r ne ''; + } + return undef; +} + # read_manifest: parse packages-manifest.conf into %{ target => { package => version|'*' } }. # INI format: [target] sections; "package=version|*" entries; blank / "#" / ";" lines ignored. # Returns an empty hash if the file is absent (callers that build require a section per target). diff --git a/mockbuild-all.pl b/mockbuild-all.pl index c439093..1b2b3c7 100755 --- a/mockbuild-all.pl +++ b/mockbuild-all.pl @@ -14,7 +14,7 @@ use POSIX qw(strftime); use FindBin qw($RealBin); use lib $RealBin; use MockBuildUtils qw(sh_quote print_step version_matches required_pkgs have_rpm - read_manifest rpm_version rpm_sigmd5 restamp_release_line + read_manifest rpm_version rpm_release rpm_sigmd5 restamp_release_line cross_copy_genesis finalize_xcat_dep); my $script_dir = abs_path(dirname(__FILE__)); @@ -275,7 +275,13 @@ exit 0; sub bump_dep_release_suffix { my ($root, $suffix) = @_; my @specs; - find(sub { push @specs, $File::Find::name if /\.spec$/ && -f $_ }, $root); + # Only stamp xcat-dep's OWN specs. If someone checked xcat-core out NESTED under $repo_root (the + # legacy `xcat-source-code`/`xcat-core` layout), do NOT descend into it -- rewriting a core spec + # (e.g. xCAT-genesis-base.spec's dynamic Release) would break the lockstep with genesis-scripts. + find(sub { + if (-d $_ && ($_ eq 'xcat-core' || $_ eq 'xcat-source-code')) { $File::Find::prune = 1; return; } + push @specs, $File::Find::name if /\.spec$/ && -f $_; + }, $root); my ($with_release, $bumped, $already) = (0, 0, 0); for my $spec (sort @specs) { open my $in, '<', $spec or die "open $spec: $!\n"; @@ -543,10 +549,21 @@ if (!$skip_build) { ($max_build_workers && $max_build_workers >= 1) ? $max_build_workers : defined($parallel_builds) ? $parallel_builds : scalar(@build_steps); - my @failed = run_build_steps_parallel( - steps => \@build_steps, - max_processes => $effective_parallel_builds, - ); + # Make --max-parallel a REAL cap. The perl builder is a single step that internally forks up + # to $effective_parallel_builds mock jobs of its own, so running it concurrently with the dep + # builders pushed live mock builds to ~2x the cap. Run it in its OWN phase, after the dep + # builders (which are quick) -- each phase then runs at most $effective_parallel_builds mock + # builds, so the cap holds, at a small bounded wall-clock cost. (The perl step sets no + # scrub_cfg and scrubs its own chroots; the scrub loop below still covers the dep/genesis steps.) + my @perl_steps = grep { $_->{id} eq 'perl' } @build_steps; + my @nonperl_steps = grep { $_->{id} ne 'perl' } @build_steps; + my @failed; + push @failed, run_build_steps_parallel( + steps => \@nonperl_steps, max_processes => $effective_parallel_builds, + ) if @nonperl_steps; + push @failed, run_build_steps_parallel( + steps => \@perl_steps, max_processes => $effective_parallel_builds, + ) if @perl_steps; # Reclaim each build step's mock chroot now that the step copied its RPMs/logs out to # its --result-dir (collect_rpms reads those, never /var/lib/mock). mock's own cleanup @@ -638,6 +655,24 @@ if (!$dry_run) { die "FATAL: manifest version mismatch for $target:\n " . join("\n ", @vmiss) . "\n" if @vmiss; print "[manifest] version pins satisfied for $target\n"; + + # When a CD --build-number bump is in effect, confirm it actually LANDED in the built rpms' + # Release -- validating %{VERSION} alone can't catch a silently un-bumped NVR (which deploy's + # additive rsync would then dedup away). Every built dep + perl package carries the suffix; + # xCAT-genesis-base is intentionally NOT bumped (kept in lockstep with xcat-core's genesis-scripts). + if ($RELEASE_BUMP ne '') { + my @rmiss; + for my $pkg (required_pkgs([sort keys %req], $skip_genesis, $skip_perl, $skip_xcat_dep)) { + next if $pkg eq 'xCAT-genesis-base'; + my $rel = rpm_release($repo_dir, $pkg); + next if !defined $rel; # a missing rpm was already reported by the version-pin check + push @rmiss, "$pkg: Release '$rel' is missing the CD bump '$RELEASE_BUMP'" + if index($rel, $RELEASE_BUMP) < 0; + } + die "FATAL: --build-number bump '$RELEASE_BUMP' did not land in built rpm(s) for $target:\n " + . join("\n ", @rmiss) . "\n" if @rmiss; + print "[manifest] Release bump '$RELEASE_BUMP' present on all built dep rpms for $target\n"; + } } print_step('Collect source RPM artifacts'); diff --git a/t/mockbuild-all.t b/t/mockbuild-all.t index 20fdbe3..0f791e8 100644 --- a/t/mockbuild-all.t +++ b/t/mockbuild-all.t @@ -10,7 +10,7 @@ use lib "$RealBin/.."; use File::Temp qw(tempdir); use File::Path qw(make_path); use File::Basename qw(basename); -use MockBuildUtils qw(required_pkgs version_matches rpm_sigmd5 rpm_version rpm_is_signed +use MockBuildUtils qw(required_pkgs version_matches rpm_sigmd5 rpm_version rpm_release rpm_is_signed restamp_release_line cross_copy_genesis finalize_xcat_dep read_manifest); # Run a printing sub with STDOUT muted so its progress lines do not pollute TAP. @@ -195,6 +195,9 @@ SPEC is(rpm_is_signed(undef), 0, 'rpm_is_signed(undef) is 0'); is(rpm_is_signed("/no/such/file.rpm"), 0, 'rpm_is_signed on a missing file is 0'); +# ---- rpm_release: absent package -> undef (used by the --build-number bump-landed check) --------- +is(rpm_release(tempdir(CLEANUP => 1), 'nonexistent-pkg'), undef, 'rpm_release is undef when no rpm matches'); + # ---- manifest <-> docs consistency: conserver-xcat is in EVERY target section (PR #62 point 7c) -- # BUILD.md documents conserver-xcat as built for every target; guard that the manifest agrees so the # doc and the manifest can never silently drift apart again. From a5432f28efa06e69d836fca41eacaeaac6f03e0b Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Mon, 10 Aug 2026 13:06:22 -0300 Subject: [PATCH 28/55] fix(goconserver): build inside a mock chroot, deps pinned by go.sum (no vendor tree) (PR #62 review #2) Completes @viniciusferrao's concern #2. Previously goconserver was built on the HOST with a runtime `go mod tidy` against a clone of mutable `master` -- non-reproducible and non-hermetic. - Rewrite goconserver/mockbuild.pl to build the rpm INSIDE a mock chroot via an SRPM: %build compiles in-chroot (BuildRequires: golang, GOTOOLCHAIN=local, CGO_ENABLED=0). - Commit only the pinned module manifest goconserver/gomod/{go.mod,go.sum} (97 lines; go.mod carries the kr/pty -> creack/pty replace). The in-chroot build downloads the modules from the Go proxy (mock networking enabled) but is reproducible because go.sum integrity-checks every module -- no `go mod tidy`, and no committed vendor tree. - goconserver is a CGO-free static binary and el8/el9 chroots ship too old a Go for 0.3.3, so always COMPILE in the el10 chroot for the arch; the Release still carries the target's dist tag (4.el), so every EL repo gets an identical, portable static binary. Verified on the build host: statically linked, no shared-lib deps, correct el tag while built in the el10 chroot. Combined with the immutable-SHA pin + --release-suffix (40feffc), goconserver is now reproducible, built in mock, and advances its NVR per CD run. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- goconserver/gomod/README.md | 14 ++ goconserver/gomod/go.mod | 27 ++++ goconserver/gomod/go.sum | 70 +++++++++ goconserver/mockbuild.pl | 278 ++++++++++++++++++++---------------- 4 files changed, 262 insertions(+), 127 deletions(-) create mode 100644 goconserver/gomod/README.md create mode 100644 goconserver/gomod/go.mod create mode 100644 goconserver/gomod/go.sum diff --git a/goconserver/gomod/README.md b/goconserver/gomod/README.md new file mode 100644 index 0000000..7ee69f2 --- /dev/null +++ b/goconserver/gomod/README.md @@ -0,0 +1,14 @@ +# Pinned Go module manifest for goconserver + +`go.mod` + `go.sum` pin goconserver's Go dependencies so the rpm build is **reproducible without +vendoring the whole dependency tree**. The build runs **inside a mock chroot** (network enabled) and +downloads the modules from the Go proxy at build time; `go.sum` integrity-checks every module, so the +result is deterministic even though the deps are not committed. + +- Generated from **xcat2/goconserver @ 6166fe5ec1c5b3c20475e322a9f0e8e93c87e45f** (the pin in + mockbuild-all.pl / goconserver/mockbuild.pl), with the archived `github.com/kr/pty` replaced by + `github.com/creack/pty@v1.1.21` and the etcd storage backend removed (xCAT uses file storage only). +- To regenerate after bumping the goconserver pin: clone at the new SHA, remove `storage/etcd*`, + `go mod init github.com/xcat2/goconserver`, + `go mod edit -replace github.com/kr/pty=github.com/creack/pty@v1.1.21`, `go mod tidy`, then copy + go.mod/go.sum here. (No `go mod vendor` needed.) diff --git a/goconserver/gomod/go.mod b/goconserver/gomod/go.mod new file mode 100644 index 0000000..092743d --- /dev/null +++ b/goconserver/gomod/go.mod @@ -0,0 +1,27 @@ +module github.com/xcat2/goconserver + +go 1.26.4 + +replace github.com/kr/pty => github.com/creack/pty v1.1.21 + +require ( + github.com/golang/protobuf v1.5.4 + github.com/gorilla/mux v1.8.1 + github.com/kr/pty v0.0.0-00010101000000-000000000000 + github.com/sirupsen/logrus v1.9.4 + github.com/spf13/cobra v1.10.2 + github.com/spf13/pflag v1.0.10 + golang.org/x/crypto v0.54.0 + golang.org/x/net v0.57.0 + google.golang.org/grpc v1.83.0 + gopkg.in/yaml.v2 v2.4.0 +) + +require ( + github.com/inconshreveable/mousetrap v1.1.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/term v0.45.0 // indirect + golang.org/x/text v0.40.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect + google.golang.org/protobuf v1.36.11 // indirect +) diff --git a/goconserver/gomod/go.sum b/goconserver/gomod/go.sum new file mode 100644 index 0000000..a466044 --- /dev/null +++ b/goconserver/gomod/go.sum @@ -0,0 +1,70 @@ +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/creack/pty v1.1.21 h1:1/QdRyBaHHJP61QkWMXlOIBfsgdDeeKfK8SYVUWJKf0= +github.com/creack/pty v1.1.21/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= +github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= +github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= +github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= +github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= +go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= +go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= +go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= +go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= +go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= +go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= +golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= +golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= +golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= +golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= +golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= +golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.83.0 h1:JeNZEKJFbQxArAMl+hiytHauacDNqJUllNfmIMmpqnQ= +google.golang.org/grpc v1.83.0/go.mod h1:kDyl6SKsiHKt0uylY5gtn5cEjkrIOhQOGDgIc4JGwzQ= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/goconserver/mockbuild.pl b/goconserver/mockbuild.pl index e152b09..ed31dcb 100755 --- a/goconserver/mockbuild.pl +++ b/goconserver/mockbuild.pl @@ -22,7 +22,7 @@ my $version = '0.3.3'; my $go_repo = 'https://github.com/xcat2/goconserver.git'; # Immutable pin: goconserver 0.3.3 is unreleased (newest tag v0.3.2) so it lives only on master. # mockbuild-all.pl passes --go-ref with the canonical pin; this default keeps standalone runs -# reproducible too. `git clone --branch` cannot take a raw SHA, so the clone below fetches by ref. +# reproducible too. The committed vendored/ tree (go.mod/go.sum/vendor) corresponds to THIS SHA. my $go_ref = '6166fe5ec1c5b3c20475e322a9f0e8e93c87e45f'; my $release_suffix = ''; # CD Release bump (".snap."); passed by mockbuild-all.pl my $build_timestamp; @@ -43,7 +43,10 @@ GetOptions( die "Run as root (current uid=$>)\n" if $> != 0; -for my $bin (qw(go git rpmbuild rpm)) { +# The Go compile happens INSIDE the mock chroot (BuildRequires: golang); the host only needs to +# fetch the pinned source and drive mock. (No host `go` build any more -- that was the non-hermetic +# path this rewrite removes.) +for my $bin (qw(git rpm mock)) { run("command -v " . sh_quote($bin) . " >/dev/null 2>&1"); } @@ -74,17 +77,24 @@ unless ($SOURCE_DATE_EPOCH && $SOURCE_DATE_EPOCH =~ /^\d+$/) { $SOURCE_DATE_EPOCH = time() unless $SOURCE_DATE_EPOCH =~ /^\d+$/; $ENV{SOURCE_DATE_EPOCH} = $SOURCE_DATE_EPOCH; +# goconserver is a CGO-free static Go binary. el8/el9 chroots ship a Go too old to build 0.3.3, so +# always COMPILE in the el10 chroot for this arch (regardless of the target EL), then ship the static +# binary to every EL repo. The Release still carries the target's dist tag (4.el$rel) so each EL repo +# gets a correctly-named, byte-identical rpm. +(my $build_cfg = $mock_cfg) =~ s/-\d+-/-10-/; + print_step("Configuration"); -print "repo_root: $repo_root\n"; -print "pkg_dir: $pkg_dir\n"; -print "work_dir: $work_dir\n"; -print "result_dir: $result_dir\n"; -print "log_dir: $log_dir\n"; -print "mock_cfg: $mock_cfg\n"; -print "arch: $arch\n"; -print "version: $version\n"; -print "go_repo: $go_repo\n"; -print "go_ref: $go_ref\n"; +print "repo_root: $repo_root\n"; +print "pkg_dir: $pkg_dir\n"; +print "work_dir: $work_dir\n"; +print "result_dir: $result_dir\n"; +print "log_dir: $log_dir\n"; +print "mock_cfg: $mock_cfg (target dist tag: el$rel)\n"; +print "build_cfg: $build_cfg (el10 -- portable static build for arch $arch)\n"; +print "arch: $arch\n"; +print "version: $version\n"; +print "go_ref: $go_ref\n"; +print "release_suffix: " . ($release_suffix ne '' ? $release_suffix : '(none)') . "\n"; print "skip_install: $skip_install\n"; make_path($result_dir); @@ -94,79 +104,45 @@ print_step("Stage build environment"); remove_tree($work_dir) if -d $work_dir; make_path($work_dir); -# Unique per run (nested under the run/target-scoped --work-dir) so concurrent builds -- e.g. -# parallel EL targets on one host -- don't wipe each other. (Was a shared -# /var/tmp/xcat-rpmbuild-goconserver, which collided under parallelism.) -my $rpmbuild_top = "$work_dir/rpmbuild"; -remove_tree($rpmbuild_top) if -d $rpmbuild_top; -for my $d (qw(BUILD BUILDROOT RPMS SOURCES SPECS SRPMS)) { - make_path("$rpmbuild_top/$d"); -} - +# --- Fetch the pinned goconserver source (immutable SHA -> reproducible) --- print_step("Clone goconserver source"); my $src_dir = "$work_dir/goconserver-src"; -# Fetch the exact pinned ref (a SHA, or a branch/tag). `git clone --branch` rejects a raw SHA, so -# init + shallow fetch the one object + checkout it -- reproducible and immutable, never "latest -# master". (xcat2/goconserver has allowReachableSHA1InWant, so fetching a master-reachable SHA works.) my $clone_log = sh_quote("$log_dir/git-clone.log"); run("git init -q " . sh_quote($src_dir) . " >$clone_log 2>&1"); run("git -C " . sh_quote($src_dir) . " remote add origin " . sh_quote($go_repo) . " >>$clone_log 2>&1"); run("git -C " . sh_quote($src_dir) . " fetch --depth 1 origin " . sh_quote($go_ref) . " >>$clone_log 2>&1"); run("git -C " . sh_quote($src_dir) . " checkout -q FETCH_HEAD >>$clone_log 2>&1"); -# etcd storage backend has broken deps with modern Go modules; -# xCAT only uses file storage, so remove etcd before building. +# etcd storage backend has broken deps with modern Go modules; xCAT only uses file storage. unlink "$src_dir/storage/etcd.go"; remove_tree("$src_dir/storage/etcd") if -d "$src_dir/storage/etcd"; +remove_tree("$src_dir/.git") if -d "$src_dir/.git"; # keep the SRPM tarball clean + reproducible -print_step("Initialize Go modules"); -$ENV{GOPATH} = "$work_dir/gopath"; -$ENV{GOCACHE} = "$work_dir/gocache"; -$ENV{GOMODCACHE} = "$work_dir/gomodcache"; -$ENV{CGO_ENABLED} = '0'; +# --- Overlay the committed, PINNED go.mod/go.sum (no vendored tree) --- +# go.mod already replaces the archived github.com/kr/pty with creack/pty (see gomod/README.md), and +# go.sum integrity-checks every module. The in-chroot build downloads the modules from the Go proxy +# (mock networking is enabled) but is reproducible because go.sum pins them -- no `go mod tidy`, and +# no 400k-line vendor tree committed. +print_step("Overlay pinned go.mod/go.sum"); +my $gomod_dir = "$pkg_dir/gomod"; +die "pinned go.mod/go.sum missing under $gomod_dir (regenerate per gomod/README.md)\n" + unless -f "$gomod_dir/go.mod" && -f "$gomod_dir/go.sum"; +copy("$gomod_dir/go.mod", "$src_dir/go.mod") or die "copy go.mod: $!\n"; +copy("$gomod_dir/go.sum", "$src_dir/go.sum") or die "copy go.sum: $!\n"; -# The archived github.com/kr/pty sets SysProcAttr.Ctty to the parent-side fd, -# which modern Go's os/exec rejects with "Setctty set but Ctty not valid in -# child". Replace it with the API-identical maintained fork creack/pty. -run("cd " . sh_quote($src_dir) . " && " . - "go mod init github.com/xcat2/goconserver && " . - "go mod edit -replace github.com/kr/pty=github.com/creack/pty\@v1.1.21 && " . - "go mod tidy" . - " >" . sh_quote("$log_dir/go-mod.log") . " 2>&1"); +# --- Assemble SRPM sources: the source tree (incl. vendor) + the xcat-authored unit + config --- +print_step("Assemble SRPM sources"); +my $srctop = "goconserver-$version"; +my $staged = "$work_dir/$srctop"; +remove_tree($staged) if -d $staged; +run("cp -a " . sh_quote($src_dir) . " " . sh_quote($staged)); +my $sources_dir = "$work_dir/sources"; +make_path($sources_dir); +my $tarball = "$sources_dir/goconserver-$version.tar.gz"; +run("tar --sort=name --owner=0 --group=0 --mtime=\@$SOURCE_DATE_EPOCH" . + " -C " . sh_quote($work_dir) . " -czf " . sh_quote($tarball) . " " . sh_quote($srctop)); -print_step("Build goconserver binaries"); -my $go_build_dir = "$work_dir/bin"; -make_path($go_build_dir); - -my $ldflags = "-X main.Version=$version"; - -run("cd " . sh_quote($src_dir) . " && " . - "go build -trimpath -buildvcs=false -ldflags " . sh_quote($ldflags) . - " -o " . sh_quote("$go_build_dir/goconserver") . " goconserver.go" . - " >" . sh_quote("$log_dir/go-build-server.log") . " 2>&1"); - -run("cd " . sh_quote($src_dir) . " && " . - "go build -trimpath -buildvcs=false -ldflags " . sh_quote($ldflags) . - " -o " . sh_quote("$go_build_dir/congo") . " cmd/congo.go" . - " >" . sh_quote("$log_dir/go-build-client.log") . " 2>&1"); - -die "goconserver binary not built\n" if !-x "$go_build_dir/goconserver"; -die "congo binary not built\n" if !-x "$go_build_dir/congo"; - -print_step("Create source tarball"); -my $payload_dir = "$work_dir/goconserver-$version"; -make_path("$payload_dir/usr/bin"); -make_path("$payload_dir/usr/lib/systemd/system"); -make_path("$payload_dir/etc/goconserver"); - -copy("$go_build_dir/goconserver", "$payload_dir/usr/bin/goconserver") - or die "copy goconserver: $!\n"; -copy("$go_build_dir/congo", "$payload_dir/usr/bin/congo") - or die "copy congo: $!\n"; -chmod 0755, "$payload_dir/usr/bin/goconserver"; -chmod 0755, "$payload_dir/usr/bin/congo"; - -write_file("$payload_dir/usr/lib/systemd/system/goconserver.service", <<'SERVICE'); +write_file("$sources_dir/goconserver.service", <<'SERVICE'); [Unit] Description=goconserver console server After=network.target @@ -181,13 +157,11 @@ StateDirectory=goconserver WantedBy=multi-user.target SERVICE -# The goconserver binary parses server.conf as YAML. Ship a VALID YAML default: the old INI-style -# ([server]\nhost = ...) is read by the YAML parser as a sequence -> `panic: cannot unmarshal !!seq into -# common.ServerConfig` at startup -> systemd rate-limits the service to `failed`. On an xCAT MN, -# xCAT::Goconserver.pm overwrites this with a cert-enabled config, so this default only has to PARSE and -# start (no SSL here -- the xcat certs don't exist until xCAT is configured). Keys/ports mirror the schema -# xCAT itself writes (api 12429, console 12430). -write_file("$payload_dir/etc/goconserver/server.conf", <<'CONF'); +# The goconserver binary parses server.conf as YAML. Ship a VALID YAML default (the old INI-style +# [server] block is read by the YAML parser as a sequence -> `panic: cannot unmarshal !!seq` at +# startup -> systemd rate-limits the service to `failed`). On an xCAT MN, xCAT::Goconserver.pm +# overwrites this with a cert-enabled config; this default only has to PARSE and start. +write_file("$sources_dir/server.conf", <<'CONF'); global: host: 0.0.0.0 logfile: /var/log/goconserver/server.log @@ -199,13 +173,13 @@ console: log_timestamp: true CONF -my $tarball = "$rpmbuild_top/SOURCES/goconserver-$version.tar.gz"; -run("tar --sort=name --owner=0 --group=0 --mtime=\@$SOURCE_DATE_EPOCH" . - " -C " . sh_quote($work_dir) . " -czf " . sh_quote($tarball) . - " goconserver-$version"); - -print_step("Create spec and build RPM"); -my $spec_content = <<"SPEC"; +# --- Spec: the Go compile runs in %build INSIDE the chroot, offline, from the vendored tree --- +print_step("Write spec"); +my $spec_file = "$work_dir/goconserver.spec"; +write_file($spec_file, <<"SPEC"); +# Go binaries carry no useful DWARF debugsource; the empty debuginfo subpackage otherwise fails +# packaging ("Empty %files debugsourcefiles.list"). Disable it. +%global debug_package %{nil} Name: goconserver Version: $version Release: 4.el$rel$release_suffix @@ -215,25 +189,33 @@ URL: https://github.com/xcat2/goconserver BuildArch: $arch Source0: goconserver-%{version}.tar.gz +Source1: goconserver.service +Source2: server.conf + +BuildRequires: golang %description goconserver is a scalable console server written in Go. It provides console logging and management for xCAT cluster nodes. %prep -%setup -n goconserver-%{version} +%setup -q -n goconserver-%{version} + +%build +# Compile in-chroot. Modules are downloaded from the Go proxy at build time (mock networking is on) +# but PINNED + integrity-checked by the committed go.sum, so the build is reproducible without a +# vendored tree. GOTOOLCHAIN=local pins the chroot's Go (never auto-downloads a toolchain). +export GOFLAGS=-mod=mod GOTOOLCHAIN=local CGO_ENABLED=0 +export GOCACHE=%{_builddir}/.gocache GOPATH=%{_builddir}/.gopath GOMODCACHE=%{_builddir}/.gomodcache +go build -trimpath -buildvcs=false -ldflags "-X main.Version=%{version}" -o goconserver goconserver.go +go build -trimpath -buildvcs=false -ldflags "-X main.Version=%{version}" -o congo cmd/congo.go %install -mkdir -p %{buildroot}/usr/bin -mkdir -p %{buildroot}/usr/lib/systemd/system -mkdir -p %{buildroot}/etc/goconserver -mkdir -p %{buildroot}/var/log/goconserver -mkdir -p %{buildroot}/var/lib/goconserver - -install -m 755 usr/bin/goconserver %{buildroot}/usr/bin/goconserver -install -m 755 usr/bin/congo %{buildroot}/usr/bin/congo -install -m 644 usr/lib/systemd/system/goconserver.service %{buildroot}/usr/lib/systemd/system/goconserver.service -install -m 644 etc/goconserver/server.conf %{buildroot}/etc/goconserver/server.conf +install -Dm0755 goconserver %{buildroot}/usr/bin/goconserver +install -Dm0755 congo %{buildroot}/usr/bin/congo +install -Dm0644 %{SOURCE1} %{buildroot}/usr/lib/systemd/system/goconserver.service +install -Dm0644 %{SOURCE2} %{buildroot}/etc/goconserver/server.conf +mkdir -p %{buildroot}/var/log/goconserver %{buildroot}/var/lib/goconserver %files /usr/bin/goconserver @@ -244,53 +226,75 @@ install -m 644 etc/goconserver/server.conf %{buildroot}/etc/goconserver/server.c %dir /var/lib/goconserver %changelog -* Thu Jul 23 2026 xCAT build - 0.3.3-4.el10 -- Ship /etc/goconserver/server.conf in YAML (the format the goconserver binary parses) instead of the - old INI [server] style, which the YAML parser reads as a sequence -> panic (cannot unmarshal !!seq) at - startup -> systemd rate-limits the service to failed before xCAT can convert the config. Fixes console - provisioning (makegocons) on the management node. -* Mon Jun 08 2026 xCAT EL10 build - 0.3.3-2.el10 -- Replace archived github.com/kr/pty with github.com/creack/pty to fix - "Setctty set but Ctty not valid in child" console fork failure on modern Go. +* Mon Aug 10 2026 xCAT build - $version-4.el$rel +- Build inside a mock chroot (no host build). Modules are downloaded at build time but pinned + + integrity-checked by a committed go.sum (no `go mod tidy`, no vendored tree). Compiled in the + el10 chroot for the arch and shipped to every EL repo (CGO-free static binary). +- Ship /etc/goconserver/server.conf as YAML (the format the goconserver binary parses). +- Replace archived github.com/kr/pty with github.com/creack/pty (console fork on modern Go). SPEC -my $spec_file = "$rpmbuild_top/SPECS/goconserver.spec"; -write_file($spec_file, $spec_content); +# --- Build in the el10 chroot for this arch --- +my $mock_uniqueext_opt = $mock_uniqueext ne '' ? ' --uniqueext ' . sh_quote($mock_uniqueext) : ''; +print_step("Mock config check"); +run("mock -r " . sh_quote($build_cfg) . $mock_uniqueext_opt . " --print-root-path >/dev/null"); +my $det_cfg = create_deterministic_mock_cfg($build_cfg, $SOURCE_DATE_EPOCH, $work_dir); -run( - "rpmbuild --define " . sh_quote("_topdir $rpmbuild_top") . +print_step("Build SRPM with mock"); +my $srpm_out = "$work_dir/srpm"; +make_path($srpm_out); +run("mock -r " . sh_quote($det_cfg) . $mock_uniqueext_opt . + " --buildsrpm --spec " . sh_quote($spec_file) . + " --sources " . sh_quote($sources_dir) . " --define " . sh_quote("use_source_date_epoch_as_buildtime 1") . " --define " . sh_quote("clamp_mtime_to_source_date_epoch 1") . " --define " . sh_quote("_buildhost xcat-build") . - " -ba " . sh_quote($spec_file) . - " >" . sh_quote("$log_dir/rpmbuild.log") . " 2>&1" -); + " --resultdir " . sh_quote($srpm_out) . + " >" . sh_quote("$log_dir/mock-buildsrpm.log") . " 2>&1"); +my @srpms = sort glob("$srpm_out/goconserver-*.src.rpm"); +die "SRPM not generated in $srpm_out\n" if !@srpms; +my $srpm = $srpms[-1]; +print "SRPM: $srpm\n"; + +print_step("Rebuild RPM with mock (offline, in-chroot go build)"); +my $rpm_out = "$work_dir/rpm"; +make_path($rpm_out); +run("mock -r " . sh_quote($det_cfg) . $mock_uniqueext_opt . + " --rebuild " . sh_quote($srpm) . + " --define " . sh_quote("use_source_date_epoch_as_buildtime 1") . + " --define " . sh_quote("clamp_mtime_to_source_date_epoch 1") . + " --define " . sh_quote("_buildhost xcat-build") . + " --resultdir " . sh_quote($rpm_out) . + " >" . sh_quote("$log_dir/mock-rebuild.log") . " 2>&1"); print_step("Collect results"); -for my $rpm (glob("$rpmbuild_top/RPMS/*/*.rpm"), glob("$rpmbuild_top/SRPMS/*.rpm")) { +my @arch_rpms = sort grep { !/\.src\.rpm$/ } glob("$rpm_out/goconserver-*.$arch.rpm"); +die "No goconserver $arch rpm generated in $rpm_out\n" if !@arch_rpms; +for my $rpm (@arch_rpms, glob("$rpm_out/*.src.rpm")) { my $dest = "$result_dir/" . basename($rpm); copy($rpm, $dest) or die "Failed to copy $rpm to $dest: $!\n"; print "Copied: $dest\n"; } +for my $log (qw(build.log root.log state.log)) { + my $s = "$rpm_out/$log"; + copy($s, "$log_dir/mock-$log") if -f $s; +} + +# Reclaim goconserver's own build chroot. mockbuild-all's scrub keys on the TARGET cfg (el$rel), not +# the el10 build cfg used here, so scrub it ourselves to avoid leaking /var/lib/mock. Best-effort. +system("mock -r " . sh_quote($build_cfg) . $mock_uniqueext_opt . + " --scrub=chroot >" . sh_quote("$log_dir/mock-scrub.log") . " 2>&1"); if (!$skip_install) { print_step("Install and smoke test"); - my @built = glob("$rpmbuild_top/RPMS/$arch/goconserver-*.rpm"); - die "No arch RPM found\n" if !@built; - my $main_rpm = $built[0]; - - run("dnf -y install " . sh_quote($main_rpm) . - " >" . sh_quote("$log_dir/dnf-install.log") . " 2>&1"); - + my $main_rpm = $arch_rpms[0]; + run("dnf -y install " . sh_quote($main_rpm) . " >" . sh_quote("$log_dir/dnf-install.log") . " 2>&1"); die "Missing /usr/bin/goconserver\n" if !-x '/usr/bin/goconserver'; die "Missing /usr/bin/congo\n" if !-x '/usr/bin/congo'; - my $rc_help = run_rc("goconserver -h >" . sh_quote("$log_dir/smoke-help.log") . " 2>&1"); die "goconserver -h failed (rc=$rc_help)\n" if $rc_help > 1; - my $rc_congo = run_rc("congo -h >" . sh_quote("$log_dir/smoke-congo.log") . " 2>&1"); die "congo -h failed (rc=$rc_congo)\n" if $rc_congo > 1; - print "Smoke tests passed.\n"; } @@ -302,18 +306,23 @@ sub usage { return <<"USAGE"; Usage: $0 [options] -Build goconserver RPM from source. +Build the goconserver RPM inside a mock chroot: fetch the pinned source, overlay the committed +go.mod/go.sum, and compile IN-CHROOT (modules downloaded at build time but pinned by go.sum -- no +vendored tree, no `go mod tidy`). The compile runs in the el10 chroot for the host arch (goconserver +is a CGO-free static binary; el8/el9 ship too old a Go), and the rpm is tagged with the target EL +(4.el) so every EL repo gets an identical static binary. Options: --work-dir PATH Working directory (default: /tmp/goconserver-mockbuild) - --mock-cfg NAME Mock config name (auto-detected if omitted) - --mock-uniqueext STR Mock uniqueext value (for compatibility with mockbuild-all.pl) + --mock-cfg NAME Target mock config (sets the EL dist tag; the build runs in its el10 peer) + --mock-uniqueext STR Mock uniqueext (for concurrency isolation under mockbuild-all.pl) --result-dir PATH Output directory for RPMs --log-dir PATH Output directory for logs --skip-install Skip dnf install + smoke tests --version VER Version string (default: 0.3.3) --go-repo URL Git repo URL (default: github.com/xcat2/goconserver) - --go-ref REF Git ref to build (default: master) + --go-ref REF Git ref/SHA to build (default: the pinned commit) + --release-suffix STR Appended to Release for CD (e.g. .snap.) --build-timestamp EPOCH SOURCE_DATE_EPOCH for deterministic builds USAGE } @@ -374,3 +383,18 @@ sub resolve_mock_cfg { } return "${os_id}+epel-${rel}-${arch}"; } + +sub create_deterministic_mock_cfg { + my ($base_cfg, $epoch, $dir) = @_; + my $cfg_path = "$dir/mock-deterministic.cfg"; + open my $fh, '>', $cfg_path or die "Cannot write $cfg_path: $!\n"; + print $fh "include('/etc/mock/${base_cfg}.cfg')\n"; + print $fh "config_opts['environment']['SOURCE_DATE_EPOCH'] = '$epoch'\n"; + print $fh "config_opts['environment']['ZERO_AR_DATE'] = '1'\n"; + # Allow network during %build so `go build` can download the (go.sum-pinned) modules -- we build + # against the proxy rather than committing a vendored tree. + print $fh "config_opts['rpmbuild_networking'] = True\n"; + print $fh "config_opts['use_host_resolv'] = True\n"; + close $fh; + return $cfg_path; +} From 9d32988ae26ba7bc4f8655948211fd297dd47b00 Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Tue, 11 Aug 2026 22:45:38 -0300 Subject: [PATCH 29/55] fix(goconserver): scrub the bootstrap chroot too, not just the chroot goconserver always compiles in the el10 chroot for the arch (el8/el9 ship a Go too old for 0.3.3), so mockbuild-all's post-build scrub -- which keys on the target cfg (el8/el9) -- cannot reach it, and goconserver self-scrubs its el10 build chroot. But it ran only 'mock --scrub=chroot', leaving the ~190 MiB bootstrap-image tree '-bootstrap-' behind. One survived per target per run and piled up in /var/lib/mock -- part of the disk leak that filled the x86 build host to 99% and flaked a build (VersatusHPC/xcat-core#51). Add '--scrub=bootstrap' so goconserver reclaims its whole chroot, matching mockbuild-all's scrub_buildroot. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- goconserver/mockbuild.pl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/goconserver/mockbuild.pl b/goconserver/mockbuild.pl index ed31dcb..2d4be86 100755 --- a/goconserver/mockbuild.pl +++ b/goconserver/mockbuild.pl @@ -282,8 +282,11 @@ for my $log (qw(build.log root.log state.log)) { # Reclaim goconserver's own build chroot. mockbuild-all's scrub keys on the TARGET cfg (el$rel), not # the el10 build cfg used here, so scrub it ourselves to avoid leaking /var/lib/mock. Best-effort. +# Scrub BOTH the chroot and its bootstrap: the el10 build cfg is bootstrap-image based, so +# --scrub=chroot alone leaves the ~190 MiB -bootstrap- tree behind (it accumulated +# one per target per run in /var/lib/mock -- the disk leak of VersatusHPC/xcat-core#51). system("mock -r " . sh_quote($build_cfg) . $mock_uniqueext_opt . - " --scrub=chroot >" . sh_quote("$log_dir/mock-scrub.log") . " 2>&1"); + " --scrub=chroot --scrub=bootstrap >" . sh_quote("$log_dir/mock-scrub.log") . " 2>&1"); if (!$skip_install) { print_step("Install and smoke test"); From 441f13c0343e074eab9215a87c44ee19bdb10e51 Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Wed, 12 Aug 2026 11:41:46 -0300 Subject: [PATCH 30/55] =?UTF-8?q?fix(build):=20address=20code=20review=20?= =?UTF-8?q?=E2=80=94=20zero-tolerance=20comment,=20gpg=20quoting,=20manife?= =?UTF-8?q?st-derived=20required=20set,=20testable=20release=20bump?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to the @viniciusferrao review of the EL matrix build: - Rewrite the stale run_build_steps_parallel comment that still described the removed 'tolerate genesis failure' workaround; the code is strict zero-tolerance (xcat-core #7696 made buildrpms.pl exit 0 iff it built the genesis rpm), so the comment now matches. - sh_quote the operator-supplied --gpg-key-name at every rpmsign/gpg site (was interpolated raw into the shell). - Derive assert_required_deps' required set from the target's packages-manifest.conf section (the single source of truth) instead of a second hard-coded list that could drift. - Move bump_dep_release_suffix into MockBuildUtils (pure, arg-driven) and add a File::Temp fixture test (stamp, xcat-core prune, no-Release skip, idempotency) -- the paths the review asked to cover. Its temp file now carries hostname+pid so the two arch build hosts can't collide on the shared NFS tree. prove t/mockbuild-all.t: 50/50. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- MockBuildUtils.pm | 59 +++++++++++++++++++++++++++- mockbuild-all.pl | 99 +++++++++++++---------------------------------- t/mockbuild-all.t | 40 ++++++++++++++++++- 3 files changed, 124 insertions(+), 74 deletions(-) diff --git a/MockBuildUtils.pm b/MockBuildUtils.pm index d137604..d2387e8 100644 --- a/MockBuildUtils.pm +++ b/MockBuildUtils.pm @@ -8,12 +8,14 @@ use warnings; use Exporter 'import'; use File::Basename qw(basename); use File::Copy qw(copy); +use File::Find; +use Sys::Hostname; our @EXPORT_OK = qw( sh_quote print_step version_matches required_pkgs have_rpm read_manifest rpm_version rpm_release rpm_sigmd5 rpm_is_signed restamp_release_line - cross_copy_genesis finalize_xcat_dep + cross_copy_genesis finalize_xcat_dep bump_dep_release_suffix ); # sh_quote: single-quote a string for safe use in a shell command. @@ -272,4 +274,59 @@ sub finalize_xcat_dep { print_step('Finalize complete'); } +# bump_dep_release_suffix: append $suffix (e.g. ".snap202607161200.57") to the Release: line of +# every xcat-dep package spec under $repo_root, so the CD build stamps a fresh, monotonic NVR. +# Idempotent: a spec already carrying this exact suffix is left alone (a re-run in the same tree +# does not double-stamp). Preserves any %{?dist}/%{?distver} macro already on the line. Returns the +# count of specs newly stamped. Dies only if NO spec under $repo_root carries a Release: line. +# Pure (takes everything as args) so t/mockbuild-all.t can exercise it directly. +sub bump_dep_release_suffix { + my ($repo_root, $suffix) = @_; + my @specs; + # Only stamp xcat-dep's OWN specs. If someone checked xcat-core out NESTED under $repo_root (the + # legacy `xcat-source-code`/`xcat-core` layout), do NOT descend into it -- rewriting a core spec + # (e.g. xCAT-genesis-base.spec's dynamic Release) would break the lockstep with genesis-scripts. + find(sub { + if (-d $_ && ($_ eq 'xcat-core' || $_ eq 'xcat-source-code')) { $File::Find::prune = 1; return; } + push @specs, $File::Find::name if /\.spec$/ && -f $_; + }, $repo_root); + my ($with_release, $bumped, $already) = (0, 0, 0); + for my $spec (sort @specs) { + open my $in, '<', $spec or die "open $spec: $!\n"; + my @lines = <$in>; + close $in; + my ($has_release, $changed) = (0, 0); + for my $line (@lines) { + # case-insensitive: some specs (e.g. Sys-Virt.spec) use a lowercase `release:` + next unless $line =~ /^Release:\s*\S/i; + $has_release = 1; + # restamp_release_line is idempotent (no-op if already carrying $suffix) and strips any + # prior .snap stamp before applying the new one, so a re-run with a different + # --build-number replaces rather than accumulates (unit-tested in t/mockbuild-all.t). + my ($new, $ch) = restamp_release_line($line, $suffix); + if ($ch) { $line = $new; $changed = 1; } + last; # only the first Release: line + } + $with_release++ if $has_release; + $already++ if $has_release && !$changed; + next unless $changed; + # atomic write (temp + rename) so a concurrent per-arch build on the shared NFS tree never + # sees a torn spec; identical suffix -> identical content, so last-writer-wins is safe. The + # temp name carries the hostname AND pid: the two arch build hosts share the NFS tree and can + # reuse the same pid, so pid alone could collide across hosts. + my $tmp = "$spec.bump." . hostname() . ".$$"; + open my $out, '>', $tmp or die "open> $tmp: $!\n"; + print {$out} @lines; + close $out; + rename $tmp, $spec or die "rename $tmp -> $spec: $!\n"; + $bumped++; + } + print "Release bump '$suffix': $bumped newly stamped, $already already stamped, of $with_release spec(s) with a Release line under $repo_root\n"; + # Only a genuine "no dep specs at all" is fatal. All-already-stamped is the expected idempotent + # case (re-run in the same tree, or the other arch bumped first) -- NOT an error. + die "FATAL: --build-number given but NO spec carried a Release: line under $repo_root (wrong tree?)\n" + if $with_release == 0; + return $bumped; +} + 1; diff --git a/mockbuild-all.pl b/mockbuild-all.pl index 1b2b3c7..c2d0cc9 100755 --- a/mockbuild-all.pl +++ b/mockbuild-all.pl @@ -15,7 +15,7 @@ use FindBin qw($RealBin); use lib $RealBin; use MockBuildUtils qw(sh_quote print_step version_matches required_pkgs have_rpm read_manifest rpm_version rpm_release rpm_sigmd5 restamp_release_line - cross_copy_genesis finalize_xcat_dep); + cross_copy_genesis finalize_xcat_dep bump_dep_release_suffix); my $script_dir = abs_path(dirname(__FILE__)); my $repo_root = abs_path($script_dir); @@ -151,7 +151,7 @@ if ($finalize_xcat_dep) { sign => ($gpg_sign ? sub { my ($rpm) = @_; local $ENV{GNUPGHOME} = $gpg_home if $gpg_home; - run_simple(qq(rpmsign --define "%_gpg_name $gpg_key_name" --addsign ) . sh_quote($rpm)); + run_simple("rpmsign --define " . sh_quote("%_gpg_name $gpg_key_name") . " --addsign " . sh_quote($rpm)); } : undef), reindex => \&reindex_and_sign_repo, ); @@ -268,56 +268,6 @@ die "FATAL: $tgt_fail target(s) failed\n" if $tgt_fail; print_step('All targets completed'); exit 0; -# Append $suffix (e.g. ".snap202607161200.57") to the Release: line of every xcat-dep -# package spec under $root, so the CD build stamps a fresh, monotonic NVR. Idempotent: -# a spec already carrying this exact suffix is left alone (so a re-run in the same tree -# does not double-stamp). Preserves any %{?dist}/%{?distver} macro already on the line. -sub bump_dep_release_suffix { - my ($root, $suffix) = @_; - my @specs; - # Only stamp xcat-dep's OWN specs. If someone checked xcat-core out NESTED under $repo_root (the - # legacy `xcat-source-code`/`xcat-core` layout), do NOT descend into it -- rewriting a core spec - # (e.g. xCAT-genesis-base.spec's dynamic Release) would break the lockstep with genesis-scripts. - find(sub { - if (-d $_ && ($_ eq 'xcat-core' || $_ eq 'xcat-source-code')) { $File::Find::prune = 1; return; } - push @specs, $File::Find::name if /\.spec$/ && -f $_; - }, $root); - my ($with_release, $bumped, $already) = (0, 0, 0); - for my $spec (sort @specs) { - open my $in, '<', $spec or die "open $spec: $!\n"; - my @lines = <$in>; - close $in; - my ($has_release, $changed) = (0, 0); - for my $line (@lines) { - # case-insensitive: some specs (e.g. Sys-Virt.spec) use a lowercase `release:` - next unless $line =~ /^Release:\s*\S/i; - $has_release = 1; - # restamp_release_line is idempotent (no-op if already carrying $suffix) and strips any - # prior .snap stamp before applying the new one, so a re-run with a different - # --build-number replaces rather than accumulates (unit-tested in t/mockbuild-all.t). - my ($new, $ch) = restamp_release_line($line, $suffix); - if ($ch) { $line = $new; $changed = 1; } - last; # only the first Release: line - } - $with_release++ if $has_release; - $already++ if $has_release && !$changed; - next unless $changed; - # atomic write (temp + rename) so a concurrent per-arch build on the shared NFS tree never - # sees a torn spec; identical suffix -> identical content, so last-writer-wins is safe. - my $tmp = "$spec.bump.$$"; - open my $out, '>', $tmp or die "open> $tmp: $!\n"; - print {$out} @lines; - close $out; - rename $tmp, $spec or die "rename $tmp -> $spec: $!\n"; - $bumped++; - } - print "Release bump '$suffix': $bumped newly stamped, $already already stamped, of $with_release spec(s) with a Release line under $root\n"; - # Only a genuine "no dep specs at all" is fatal. All-already-stamped is the expected idempotent - # case (re-run in the same tree, or the other arch bumped first) -- NOT an error. - die "FATAL: --build-number given but NO spec carried a Release: line under $root (wrong tree?)\n" - if $with_release == 0; -} - # Build a single target into its own build-output/ tree and return # { repo_dir, rel }. Everything below through the summary is per-target work. sub build_one_target { @@ -782,7 +732,12 @@ sub deploy_target { copy($rpm, "$dest/" . basename($rpm)) or die "Failed to copy $rpm -> $dest: $!\n"; } - assert_required_deps($dest); + # Derive the required package set from THIS target's manifest section (the single source of + # truth), dropping any package whose builder was skipped, and assert each landed in the repo. + my %MAN = read_manifest("$repo_root/packages-manifest.conf"); + my %req = %{ $MAN{$tgt} // {} }; + my @required = required_pkgs([sort keys %req], $skip_genesis, $skip_perl, $skip_xcat_dep); + assert_required_deps($dest, \@required); sign_and_index_repo($dest); write_dep_repo_metadata($dest, $rel); my $n = scalar(grep { !/\.src\.rpm$/ } glob("$dest/*.rpm")); @@ -804,7 +759,7 @@ sub sign_and_index_repo { my @rpms = grep { !/\.src\.rpm$/ } glob("$dir/*.rpm"); if ($gpg_sign && @rpms) { local $ENV{GNUPGHOME} = $gpg_home if $gpg_home; - run_simple(qq(rpmsign --define "%_gpg_name $gpg_key_name" --addsign ) + run_simple("rpmsign --define " . sh_quote("%_gpg_name $gpg_key_name") . " --addsign " . join(' ', map { sh_quote($_) } @rpms)); } run_simple(createrepo_c_cmd($dir)); @@ -812,8 +767,8 @@ sub sign_and_index_repo { local $ENV{GNUPGHOME} = $gpg_home if $gpg_home; my $repomd = "$dir/repodata/repomd.xml"; unlink "$repomd.asc" if -f "$repomd.asc"; - run_simple(qq(gpg -a --detach-sign --default-key "$gpg_key_name" ) . sh_quote($repomd)); - run_simple(qq(gpg -a --export "$gpg_key_name" > ) . sh_quote("$repomd.key")); + run_simple("gpg -a --detach-sign --default-key " . sh_quote($gpg_key_name) . ' ' . sh_quote($repomd)); + run_simple("gpg -a --export " . sh_quote($gpg_key_name) . " > " . sh_quote("$repomd.key")); } } @@ -883,8 +838,8 @@ sub reindex_and_sign_repo { local $ENV{GNUPGHOME} = $gpg_home if $gpg_home; my $repomd = "$dir/repodata/repomd.xml"; unlink "$repomd.asc" if -f "$repomd.asc"; - run_simple(qq(gpg -a --detach-sign --default-key "$gpg_key_name" ) . sh_quote($repomd)); - run_simple(qq(gpg -a --export "$gpg_key_name" > ) . sh_quote("$repomd.key")); + run_simple("gpg -a --detach-sign --default-key " . sh_quote($gpg_key_name) . ' ' . sh_quote($repomd)); + run_simple("gpg -a --export " . sh_quote($gpg_key_name) . " > " . sh_quote("$repomd.key")); } } @@ -1045,11 +1000,12 @@ sub run_build_steps_parallel { return if !@{$steps}; # Returns the ids of any steps that failed; the caller (build_one_target) enforces - # zero-tolerance -- any failed manifest package fails the whole run. We build only packages - # required for the target (per packages-manifest.conf), so there is no "expected to fail on this - # arch/el" case left to tolerate. genesis is the sole exception the CALLER handles: xcat-core's - # buildrpms.pl exits non-zero on an unrelated post-build xCAT-release-latest cp even when the - # genesis rpm IS built, so the caller treats genesis as failed only if its rpm is absent. + # zero-tolerance -- ANY failed step fails the whole run, genesis included, with no special-case. + # We build only packages required for the target (per packages-manifest.conf), so there is no + # "expected to fail on this arch/el" case left to tolerate. There is likewise no genesis + # exception: since xcat-core #7696, buildrpms.pl exits 0 iff it produced the genesis rpm, so a + # non-zero genesis exit is a real failure (the old "tolerate if the rpm is already present" + # workaround is gone -- a stale artifact must never mask a failed build). if ($dry_run || $max_processes <= 1 || @{$steps} == 1) { my @failed; for my $step (@{$steps}) { @@ -1125,20 +1081,19 @@ sub run_build_steps_parallel { # assert_required_deps: the per-EL dep repo is unusable without these, so a MISSING one is -# fatal even though individual builder failures are tolerated above. genesis-base is required -# unless --skip-genesis. +# fatal even though individual builder failures are tolerated above. The required set is derived +# by the caller from this target's packages-manifest.conf section (the single source of truth) and +# passed in as $required_ref, rather than duplicated as a hard-coded list here. sub assert_required_deps { - my ($dir) = @_; + my ($dir, $required_ref) = @_; # xCAT Requires all of these on every arch, and every one of them builds natively on every # arch (the noarch deps -- grub2-xcat, xnba-undi -- just repackage committed artifacts), so # a self-sufficient per-arch build produces the whole set with no cross-arch import. # elilo-xcat is noarch but xCAT hard-requires it (Requires: elilo-xcat >= 3.14-6) on EVERY arch, - # so a missing elilo makes the whole dep repo uninstallable -- it MUST be required here, not - # silently tolerated (it builds from a tracked prebuilt on ppc64le/EL8, compiled elsewhere). - # A package whose builder was skipped is not required (else a clean --skip-* run fails). - my @all = qw(elilo-xcat ipmitool-xcat syslinux-xcat grub2-xcat xnba-undi - perl-IO-Stty perl-HTTP-Async perl-Net-HTTPS-NB xCAT-genesis-base); - my @req = required_pkgs(\@all, $skip_genesis, $skip_perl, $skip_xcat_dep); + # so a missing elilo makes the whole dep repo uninstallable -- it is listed in every manifest + # target section, so it is always part of the required set below (not silently tolerated). + # A package whose builder was skipped is not required (the caller applies required_pkgs()). + my @req = @$required_ref; my @missing = grep { !have_rpm($dir, $_) } @req; die "FATAL: required deps missing from $dir: @missing\n" if @missing; print "[deps] required set present in $dir: @req\n"; diff --git a/t/mockbuild-all.t b/t/mockbuild-all.t index 0f791e8..c810338 100644 --- a/t/mockbuild-all.t +++ b/t/mockbuild-all.t @@ -11,7 +11,8 @@ use File::Temp qw(tempdir); use File::Path qw(make_path); use File::Basename qw(basename); use MockBuildUtils qw(required_pkgs version_matches rpm_sigmd5 rpm_version rpm_release rpm_is_signed - restamp_release_line cross_copy_genesis finalize_xcat_dep read_manifest); + restamp_release_line cross_copy_genesis finalize_xcat_dep read_manifest + bump_dep_release_suffix); # Run a printing sub with STDOUT muted so its progress lines do not pollute TAP. sub quiet(&) { @@ -210,4 +211,41 @@ is(rpm_release(tempdir(CLEANUP => 1), 'nonexistent-pkg'), undef, 'rpm_release is or diag("missing conserver-xcat in: @missing"); } +# ---- bump_dep_release_suffix: stamps xcat-dep specs, prunes nested xcat-core, idempotent -------- +# Reviewer asked for a test on this path. It walks a tree, stamps the first Release: line of every +# xcat-dep spec, prunes a nested xcat-core/ checkout, and is idempotent on a re-run. +{ + my $tmp = tempdir(CLEANUP => 1); + # (a) a top-level dep spec that MUST be stamped + open my $a, '>', "$tmp/a.spec" or die; + print $a "Name: a\nVersion: 1.0\nRelease: 5%{?dist}\n"; + close $a; + # (b) a spec NESTED under xcat-core/ that MUST be pruned (left untouched) + make_path("$tmp/xcat-core"); + open my $b, '>', "$tmp/xcat-core/b.spec" or die; + print $b "Name: b\nVersion: 1.0\nRelease: 9\n"; + close $b; + # (c) a spec with no Release: line at all (ignored, never stamped) + open my $c, '>', "$tmp/c.spec" or die; + print $c "Name: c\nVersion: 1.0\n"; + close $c; + + my $n = quiet { bump_dep_release_suffix($tmp, '.snap202601010000') }; + is($n, 1, 'bump_dep_release_suffix stamps exactly the one dep spec with a Release line'); + + my $a_after = do { open my $fh, '<', "$tmp/a.spec" or die; local $/; <$fh> }; + like($a_after, qr/^Release: 5%\{\?dist\}\.snap202601010000$/m, + 'a.spec Release now carries the CD suffix, macro preserved'); + + my $b_after = do { open my $fh, '<', "$tmp/xcat-core/b.spec" or die; local $/; <$fh> }; + is($b_after, "Name: b\nVersion: 1.0\nRelease: 9\n", + 'nested xcat-core/b.spec is pruned and left untouched'); + + # A SECOND call is idempotent: nothing newly stamped, a.spec content unchanged. + my $n2 = quiet { bump_dep_release_suffix($tmp, '.snap202601010000') }; + is($n2, 0, 'a second bump_dep_release_suffix call stamps nothing (idempotent)'); + my $a_again = do { open my $fh, '<', "$tmp/a.spec" or die; local $/; <$fh> }; + is($a_again, $a_after, 'a.spec content unchanged on the idempotent second call'); +} + done_testing; From 25dfc3957ab343bb39fb422236473520e0e98998 Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Wed, 12 Aug 2026 15:29:02 -0300 Subject: [PATCH 31/55] feat(build): manifest-driven repo completeness + signature gate, auto-run after build Adds a real gate on the BUILT per-target repo, using packages-manifest.conf as the single source of truth, layered so the decision is pure and unit-tested: - MockBuildUtils::verify_repo_packages(\%expected,\%present) -- pure completeness (MISSING / VERSION vs the manifest pins); verify_repo_signature(\%expected,\%observed) -- pure signature identity (UNSIGNED / WRONGKEY). Both unit-tested (happy+sad). - mockbuild-all.pl does the IO and calls both from one sub, verify_target_repo: reads the manifest, enumerates the repo via rpm_version, resolves --gpg-key-name to a primary-key fingerprint and extracts repomd.xml.asc's actual signer (VALIDSIG), then merges the two pure results and dies listing every problem. - Runs AUTOMATICALLY at the end of deploy_target (after sign+index), replacing the old assert_required_deps + inline version-pin loop with one consolidated gate; suppressible with --no-verify-repo. Also a standalone build-free '--verify-repo=' mode (manifest from repo_root, gpg from --gpg-key-name/--gpg-home; target derived from the rh/ path or --target). prove t/mockbuild-all.t: 68/68 (was 50). gpg round-trip smoke-tested: right key -> OK, wrong key -> WRONGKEY, missing .asc -> UNSIGNED. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- MockBuildUtils.pm | 48 +++++++++++ mockbuild-all.pl | 215 +++++++++++++++++++++++++++++++++------------- t/mockbuild-all.t | 61 ++++++++++++- 3 files changed, 262 insertions(+), 62 deletions(-) diff --git a/MockBuildUtils.pm b/MockBuildUtils.pm index d2387e8..f444a44 100644 --- a/MockBuildUtils.pm +++ b/MockBuildUtils.pm @@ -14,6 +14,7 @@ use Sys::Hostname; our @EXPORT_OK = qw( sh_quote print_step version_matches required_pkgs have_rpm read_manifest + verify_repo_packages verify_repo_signature rpm_version rpm_release rpm_sigmd5 rpm_is_signed restamp_release_line cross_copy_genesis finalize_xcat_dep bump_dep_release_suffix ); @@ -60,6 +61,53 @@ sub required_pkgs { } @$pkgs; } +# verify_repo_packages: the PURE completeness-decision layer of the repo gate. Given the manifest's +# %expected { pkg => version-pin } and the %present { pkg => version-found-or-undef } actually in a +# built repo, return a list of human-readable problem strings (empty list = every package present at +# its pin): +# "MISSING (manifest requires )" when %present has no (defined) version for +# "VERSION : repo has , manifest pins " when present but !version_matches(got, pin) +# Uses version_matches (same semantics as the in-line manifest pin loop), so a '*' or glob pin is +# accepted exactly as there. No file/manifest I/O here -- the disk layer builds %present and passes +# both hashes in, keeping this unit-testable in isolation. +sub verify_repo_packages { + my ($expected, $present) = @_; + my @problems; + for my $pkg (sort keys %$expected) { + my $pin = $expected->{$pkg}; + my $got = $present->{$pkg}; + if (!defined $got) { + push @problems, "MISSING $pkg (manifest requires " . (defined($pin) ? $pin : '*') . ")"; + } elsif (!version_matches($got, $pin)) { + push @problems, "VERSION $pkg: repo has $got, manifest pins $pin"; + } + } + return @problems; +} + +# verify_repo_signature: the PURE signature-decision layer of the repo gate. Given %expected +# { unit => expected signing-key identity } and %observed { unit => key that ACTUALLY signed (a +# string the script extracts from gpg), or undef/'' when unsigned / verification failed }, return a +# list of problem strings (empty = every unit signed by the expected key). For EL the single unit is +# 'repomd'. This does a plain string compare only -- it invokes NO gpg: the caller runs gpg --verify, +# extracts the observed key id, and resolves --gpg-key-name to the SAME identity form before calling. +# "UNSIGNED (expected )" when %observed is absent/empty for +# "WRONGKEY : signed by , expected " when both defined but differ +sub verify_repo_signature { + my ($expected, $observed) = @_; + my @problems; + for my $unit (sort keys %$expected) { + my $exp = $expected->{$unit}; + my $obs = $observed->{$unit}; + if (!defined($obs) || $obs eq '') { + push @problems, "UNSIGNED $unit (expected " . (defined($exp) ? $exp : '') . ")"; + } elsif (defined($exp) && $obs ne $exp) { + push @problems, "WRONGKEY $unit: signed by $obs, expected $exp"; + } + } + return @problems; +} + # have_rpm: is there a non-src rpm named -... under $dir? sub have_rpm { my ($dir, $name) = @_; diff --git a/mockbuild-all.pl b/mockbuild-all.pl index c2d0cc9..b2ca160 100755 --- a/mockbuild-all.pl +++ b/mockbuild-all.pl @@ -14,7 +14,8 @@ use POSIX qw(strftime); use FindBin qw($RealBin); use lib $RealBin; use MockBuildUtils qw(sh_quote print_step version_matches required_pkgs have_rpm - read_manifest rpm_version rpm_release rpm_sigmd5 restamp_release_line + read_manifest verify_repo_packages verify_repo_signature + rpm_version rpm_release rpm_sigmd5 restamp_release_line cross_copy_genesis finalize_xcat_dep bump_dep_release_suffix); my $script_dir = abs_path(dirname(__FILE__)); @@ -63,6 +64,13 @@ my $force_unlock = 0; my $finalize_xcat_dep = 0; my $x86_64_repo = ''; my $ppc64le_repo = ''; +# --verify-repo=: standalone, build-free completeness + signature gate over one already-built +# per-target repo (see verify_target_repo). Empty means "not in standalone verify mode". The target +# is derived from the repo path (.../rh/ -> alma+epel--) or taken from --target. +my $verify_repo = ''; +# --no-verify-repo suppresses the AUTOMATIC post-build gate deploy_target runs after each target is +# finalized+signed (for iteration/debug). Verification is ON by default. +my $no_verify_repo = 0; my $HELD_LOCK; # path of the output lock this process owns (for cleanup on exit) my $LOCK_OWNER_PID; # pid that created the lock; forked children must NOT remove it @@ -79,6 +87,8 @@ GetOptions( 'finalize-xcat-dep!' => \$finalize_xcat_dep, 'x86_64-repo=s' => \$x86_64_repo, 'ppc64le-repo=s' => \$ppc64le_repo, + 'verify-repo=s' => \$verify_repo, + 'no-verify-repo!' => \$no_verify_repo, 'target=s' => \$target, 'nproc=i' => \$nproc, 'parallel-builds=i' => \$parallel_builds, @@ -100,7 +110,7 @@ GetOptions( 'dry-run!' => \$dry_run, ) or die usage(); -die "Run as root (uid=$>)\n" if $> != 0 && !$finalize_xcat_dep; +die "Run as root (uid=$>)\n" if $> != 0 && !$finalize_xcat_dep && !$verify_repo; # --skip-build collects a prior build's artifacts from that build's per-target tree, so it must # know the target. Without --target the default is "all three EL targets", and each would collect # the same artifacts and cross-publish them into every repo (foreign-EL / foreign-arch rpms). @@ -127,6 +137,23 @@ if ($run_id eq '') { $run_id = strftime('%Y%m%d-%H%M%S', gmtime($SOURCE_DATE_EPOCH)); } +# --verify-repo=: a distinct, build-free completeness + signature gate over ONE already-built +# per-target repo. The value is just the repo dir; the manifest comes from the script's existing +# resolution (repo_root/packages-manifest.conf) and the gpg key/home from --gpg-key-name/--gpg-home. +# The target is derived from the repo path (.../rh/ -> alma+epel--) unless --target +# is given. Delegates the whole check to verify_target_repo (the SAME gate the auto-run uses), so it +# exits 0 when complete or dies listing every problem. Runs alone -- no build, no lock, no root. +if ($verify_repo ne '') { + require_command('rpm'); + my $rdir = abs_path($verify_repo) or die "--verify-repo repo '$verify_repo' not found\n"; + die "--verify-repo repo '$rdir' is not a directory\n" if !-d $rdir; + my $tgt = $target ne '' ? $target : derive_target_from_repo_path($rdir); + die "--verify-repo: cannot derive a target from repo path '$rdir'; pass --target\n" + if !defined($tgt) || $tgt eq ''; + verify_target_repo($rdir, $tgt); # manifest defaults to repo_root/packages-manifest.conf + exit 0; +} + # --finalize-xcat-dep: a distinct, build-free mode. After BOTH arch build hosts have # produced their per-EL repos (each carrying only its own xCAT-genesis-base), the x86_64 # repo must ALSO ship the noarch xCAT-genesis-base-ppc64 (so an x86_64 MN can netboot ppc @@ -585,44 +612,25 @@ if (!$skip_genesis && !$dry_run) { } } -# Manifest version pins: every required package must be present at its pinned version. A build -# that produces a different version (a source version bump not reflected here) fails the run; -# a manifest value of '*' accepts any version. Only the Version is pinned, not the Release -# (which carries the per-EL dist tag and the genesis snap timestamp). This also runs under -# --skip-build so a collection-only publish is validated against the target's manifest exactly -# like a fresh build (a stale/foreign-arch collected rpm fails here instead of shipping). -if (!$dry_run) { - my @vmiss; - # Only validate packages whose builder was NOT skipped -- so a clean --skip-* run does not - # fail on packages it deliberately did not build. +# Repo completeness -- every required package present at its pinned version (a '*' pin accepts any), +# missing packages included -- is now gated ONCE, centrally, in deploy_target via verify_target_repo +# (the single consolidated gate; it also runs under --skip-build and validates the deployed repo). +# The only per-build check kept here is the CD --build-number bump: confirm it actually LANDED in the +# built rpms' Release, since validating %{VERSION} alone can't catch a silently un-bumped NVR (which +# deploy's additive rsync would then dedup away). Every built dep + perl package carries the suffix; +# xCAT-genesis-base is intentionally NOT bumped (kept in lockstep with xcat-core's genesis-scripts). +if (!$dry_run && $RELEASE_BUMP ne '') { + my @rmiss; for my $pkg (required_pkgs([sort keys %req], $skip_genesis, $skip_perl, $skip_xcat_dep)) { - my $want = $req{$pkg}; - next if !defined($want) || $want eq '*'; - my $got = rpm_version($repo_dir, $pkg); - if (!defined $got) { push @vmiss, "$pkg: not built"; } - elsif (!version_matches($got, $want)) { push @vmiss, "$pkg: built $got, manifest pins $want"; } - } - die "FATAL: manifest version mismatch for $target:\n " . join("\n ", @vmiss) . "\n" - if @vmiss; - print "[manifest] version pins satisfied for $target\n"; - - # When a CD --build-number bump is in effect, confirm it actually LANDED in the built rpms' - # Release -- validating %{VERSION} alone can't catch a silently un-bumped NVR (which deploy's - # additive rsync would then dedup away). Every built dep + perl package carries the suffix; - # xCAT-genesis-base is intentionally NOT bumped (kept in lockstep with xcat-core's genesis-scripts). - if ($RELEASE_BUMP ne '') { - my @rmiss; - for my $pkg (required_pkgs([sort keys %req], $skip_genesis, $skip_perl, $skip_xcat_dep)) { - next if $pkg eq 'xCAT-genesis-base'; - my $rel = rpm_release($repo_dir, $pkg); - next if !defined $rel; # a missing rpm was already reported by the version-pin check - push @rmiss, "$pkg: Release '$rel' is missing the CD bump '$RELEASE_BUMP'" - if index($rel, $RELEASE_BUMP) < 0; - } - die "FATAL: --build-number bump '$RELEASE_BUMP' did not land in built rpm(s) for $target:\n " - . join("\n ", @rmiss) . "\n" if @rmiss; - print "[manifest] Release bump '$RELEASE_BUMP' present on all built dep rpms for $target\n"; + next if $pkg eq 'xCAT-genesis-base'; + my $rel = rpm_release($repo_dir, $pkg); + next if !defined $rel; # a missing rpm is caught by the completeness gate in deploy_target + push @rmiss, "$pkg: Release '$rel' is missing the CD bump '$RELEASE_BUMP'" + if index($rel, $RELEASE_BUMP) < 0; } + die "FATAL: --build-number bump '$RELEASE_BUMP' did not land in built rpm(s) for $target:\n " + . join("\n ", @rmiss) . "\n" if @rmiss; + print "[manifest] Release bump '$RELEASE_BUMP' present on all built dep rpms for $target\n"; } print_step('Collect source RPM artifacts'); @@ -732,14 +740,13 @@ sub deploy_target { copy($rpm, "$dest/" . basename($rpm)) or die "Failed to copy $rpm -> $dest: $!\n"; } - # Derive the required package set from THIS target's manifest section (the single source of - # truth), dropping any package whose builder was skipped, and assert each landed in the repo. - my %MAN = read_manifest("$repo_root/packages-manifest.conf"); - my %req = %{ $MAN{$tgt} // {} }; - my @required = required_pkgs([sort keys %req], $skip_genesis, $skip_perl, $skip_xcat_dep); - assert_required_deps($dest, \@required); sign_and_index_repo($dest); write_dep_repo_metadata($dest, $rel); + # Automatic completeness + signature gate on the finalized, signed repo -- the single + # consolidated gate (verify_target_repo, the same one --verify-repo runs). Asserts every + # manifest-required package is present at its pinned version AND that the repomd signature + # verifies. Suppressible with --no-verify-repo for iteration/debug. + verify_target_repo($dest, $tgt) unless $no_verify_repo; my $n = scalar(grep { !/\.src\.rpm$/ } glob("$dest/*.rpm")); print "Deployed rh$rel/$arch: $n rpms\n"; } @@ -871,6 +878,15 @@ Options: (issue #7610). Honors --gpg-sign/--gpg-key-name/--gpg-home. Use alone. --x86_64-repo PATH (finalize) x86_64 repo root holding /x86_64 (e.g. rh9/x86_64) --ppc64le-repo PATH (finalize) ppc64le repo root holding /ppc64le + --verify-repo PATH Standalone completeness + signature gate over the per-target repo at PATH + (builds nothing). Asserts every package packages-manifest.conf requires for + the target is present at a version satisfying its pin AND that the repomd + is signed by --gpg-key-name; exits 0 if complete, or lists each MISSING/ + VERSION/UNSIGNED/WRONGKEY problem and fails. The target is derived from the path + (.../rh/ -> alma+epel--) unless --target is given; the + manifest and gpg key/home come from the usual options. Use alone. + --no-verify-repo Suppress the AUTOMATIC post-build completeness+signature gate that runs + after each target's repo is finalized (default: verification ON) --gpg-sign Sign rpms + repomd.xml of each per-EL repo --gpg-key-name NAME GPG key name (default: "xCAT Signing Key") --gpg-home PATH GNUPGHOME for signing (default: system keyring) @@ -1080,23 +1096,100 @@ sub run_build_steps_parallel { -# assert_required_deps: the per-EL dep repo is unusable without these, so a MISSING one is -# fatal even though individual builder failures are tolerated above. The required set is derived -# by the caller from this target's packages-manifest.conf section (the single source of truth) and -# passed in as $required_ref, rather than duplicated as a hard-coded list here. -sub assert_required_deps { - my ($dir, $required_ref) = @_; - # xCAT Requires all of these on every arch, and every one of them builds natively on every - # arch (the noarch deps -- grub2-xcat, xnba-undi -- just repackage committed artifacts), so - # a self-sufficient per-arch build produces the whole set with no cross-arch import. - # elilo-xcat is noarch but xCAT hard-requires it (Requires: elilo-xcat >= 3.14-6) on EVERY arch, - # so a missing elilo makes the whole dep repo uninstallable -- it is listed in every manifest - # target section, so it is always part of the required set below (not silently tolerated). - # A package whose builder was skipped is not required (the caller applies required_pkgs()). - my @req = @$required_ref; - my @missing = grep { !have_rpm($dir, $_) } @req; - die "FATAL: required deps missing from $dir: @missing\n" if @missing; - print "[deps] required set present in $dir: @req\n"; +# repo_present_versions: thin disk layer for the repo gate. Given a built repo dir and the list of +# required package names, return %present = (name => rpm_version($dir, $name)) for each -- reusing the +# EXISTING rpm_version so genesis's arch-suffixed naming resolves exactly as the in-line manifest pin +# check does. rpm_version returns undef for an absent package, which verify_repo_packages then reports +# as MISSING. Pure disk read; the decision itself lives in verify_repo_packages. +sub repo_present_versions { + my ($dir, $names) = @_; + my %present; + $present{$_} = rpm_version($dir, $_) for @$names; + return %present; +} + +# gpg_key_fingerprint: resolve a gpg key NAME (e.g. "xCAT Signing Key") to its primary-key +# fingerprint in the given keyring, so the expected and observed signing identities are compared in +# the SAME form (a fingerprint). Falls back to the name itself when it cannot be resolved. +sub gpg_key_fingerprint { + my ($keyname, $home) = @_; + my $h = ($home ne '') ? ' --homedir ' . sh_quote($home) : ''; + my $out = `gpg$h --with-colons --fingerprint --list-keys ${\ sh_quote($keyname)} 2>/dev/null`; + for my $line (split /\n/, $out // '') { + return $1 if $line =~ /^fpr:+([0-9A-Fa-f]+):/; # first fpr = primary key fingerprint + } + return $keyname; +} + +# repomd_observed_signer: run gpg --verify on the detached repomd signature and extract the identity +# of the key that actually signed it, as a primary-key fingerprint (the last field of the VALIDSIG +# status line). Returns '' when the .asc is absent or verification fails (both read as "unsigned"). +sub repomd_observed_signer { + my ($asc, $file, $home) = @_; + return '' unless -f $asc && -f $file; + my $h = ($home ne '') ? ' --homedir ' . sh_quote($home) : ''; + my $out = `gpg$h --status-fd=1 --verify ${\ sh_quote($asc)} ${\ sh_quote($file)} 2>/dev/null`; + for my $line (split /\n/, $out // '') { + # VALIDSIG ; the trailing field is the primary fpr. + if ($line =~ /^\[GNUPG:\]\s+VALIDSIG\s+(.*\S)\s*$/) { + my @f = split ' ', $1; + return $f[-1]; + } + } + return ''; +} + +# verify_target_repo: the completeness + signature gate for ONE built per-target repo -- the single +# source of truth for "is this repo shippable?", replacing the old assert_required_deps + in-line +# version-pin loop. It does the IO (manifest parse, rpm_version, gpg --verify) and delegates every +# DECISION to the two PURE helpers: verify_repo_packages (missing/version) and verify_repo_signature +# (unsigned/wrongkey). Both problem lists are merged. Prints a one-line OK, or dies listing every +# problem. Both the automatic post-build gate (deploy_target) and the standalone --verify-repo mode +# call this, so there is exactly one gate implementation. +sub verify_target_repo { + my ($dir, $tgt, $manifest) = @_; + $manifest //= "$repo_root/packages-manifest.conf"; + my %MAN = read_manifest($manifest); + my %req = %{ $MAN{$tgt} // {} }; + die "FATAL: no manifest section for target '$tgt' in $manifest\n" if !%req; + # Skip flags default 0 -> the full required set. A package whose builder was skipped is not required. + my @names = required_pkgs([sort keys %req], $skip_genesis, $skip_perl, $skip_xcat_dep); + my %present = repo_present_versions($dir, \@names); + my %expected = map { $_ => $req{$_} } @names; + my @problems = verify_repo_packages(\%expected, \%present); + + # Signature gate: the IO (gpg) lives here; the decision is the pure verify_repo_signature. The + # pipeline always signs, so a signed repo's repomd MUST be signed by --gpg-key-name. We resolve + # that key to a fingerprint and extract the fingerprint that actually signed repomd, then compare. + # Skipped with a printed note only when no gpg key/home is configured (nothing to check against). + if ($gpg_sign || $gpg_home ne '') { + require_command('gpg'); + my $repomd = "$dir/repodata/repomd.xml"; + my $asc = "$repomd.asc"; + my %exp_sig = ('repomd' => gpg_key_fingerprint($gpg_key_name, $gpg_home)); + my %obs_sig = ('repomd' => repomd_observed_signer($asc, $repomd, $gpg_home)); + push @problems, verify_repo_signature(\%exp_sig, \%obs_sig); + } else { + print "[verify-repo] $tgt: no gpg key/home configured -- skipping repomd signature check\n"; + } + + if (@problems) { + print " - $_\n" for @problems; + die "FATAL: repo INCOMPLETE for $tgt at $dir (" . scalar(@problems) . " problem(s))\n"; + } + print "[verify-repo] $tgt complete: " . scalar(@names) + . " required packages present + version-pinned in $dir\n"; + return 1; +} + +# derive_target_from_repo_path: map a deployed per-target repo path .../rh/ to its manifest +# target section name alma+epel--. Returns undef when the path lacks that rh/ tail, +# so the standalone --verify-repo mode can require an explicit --target instead. +sub derive_target_from_repo_path { + my ($dir) = @_; + return undef unless defined $dir; + return "alma+epel-$1-$2" if $dir =~ m{/rh(\d+)/([^/]+)/*$}; + return undef; } sub collect_rpms { diff --git a/t/mockbuild-all.t b/t/mockbuild-all.t index c810338..52fe8b6 100644 --- a/t/mockbuild-all.t +++ b/t/mockbuild-all.t @@ -12,7 +12,7 @@ use File::Path qw(make_path); use File::Basename qw(basename); use MockBuildUtils qw(required_pkgs version_matches rpm_sigmd5 rpm_version rpm_release rpm_is_signed restamp_release_line cross_copy_genesis finalize_xcat_dep read_manifest - bump_dep_release_suffix); + verify_repo_packages verify_repo_signature bump_dep_release_suffix); # Run a printing sub with STDOUT muted so its progress lines do not pollute TAP. sub quiet(&) { @@ -248,4 +248,63 @@ is(rpm_release(tempdir(CLEANUP => 1), 'nonexistent-pkg'), undef, 'rpm_release is is($a_again, $a_after, 'a.spec content unchanged on the idempotent second call'); } +# ---- verify_repo_packages: pure repo-completeness decision (MISSING + VERSION + wildcard) --------- +# The gate's completeness layer: given manifest pins and the versions actually present in a repo, +# return the list of problems (empty = complete). No I/O -- exercised directly with plain hashes. +{ + # happy: every required package present, one exact-pinned + one wildcard -> no problems. + my @p = verify_repo_packages({ a => '1.0', b => '*' }, { a => '1.0', b => '9.9' }); + is_deeply(\@p, [], 'verify_repo_packages: all present + pins satisfied -> 0 problems'); + + # missing: present lacks 'a' entirely -> exactly one MISSING problem naming 'a'. + my @m = verify_repo_packages({ a => '1.0', b => '*' }, { b => '9.9' }); + is(scalar(@m), 1, 'verify_repo_packages: an absent package yields exactly one problem'); + like($m[0], qr/^MISSING a\b/, 'verify_repo_packages: absent package reported as MISSING '); + + # missing via explicit undef present value is treated the same as absent. + my @mu = verify_repo_packages({ a => '1.0' }, { a => undef }); + is(scalar(@mu), 1, 'verify_repo_packages: undef present version counts as MISSING'); + like($mu[0], qr/^MISSING a\b/, 'verify_repo_packages: undef present version reported as MISSING'); + + # version: present but the wrong version -> exactly one VERSION problem naming 'a'. + my @v = verify_repo_packages({ a => '1.0' }, { a => '2.0' }); + is(scalar(@v), 1, 'verify_repo_packages: a mismatched version yields exactly one problem'); + like($v[0], qr/^VERSION a\b/, 'verify_repo_packages: version mismatch reported as VERSION '); + + # wildcard: a '*' pin accepts any present version -> no problem. + my @w = verify_repo_packages({ c => '*' }, { c => '0.0.1' }); + is_deeply(\@w, [], "verify_repo_packages: '*' pin accepts any present version"); + + # combined: one MISSING and one VERSION -> two problems (sorted by package name: a before b). + my @c = verify_repo_packages({ a => '1.0', b => '2.0' }, { b => '9.9' }); + is(scalar(@c), 2, 'verify_repo_packages: one MISSING + one VERSION -> two problems'); + like($c[0], qr/^MISSING a\b/, 'verify_repo_packages: combined case reports MISSING a'); + like($c[1], qr/^VERSION b\b/, 'verify_repo_packages: combined case reports VERSION b'); +} + +# ---- verify_repo_signature: pure signature decision (match / unsigned / wrongkey) ---------------- +# Given the expected signing-key identity per unit and the key that actually signed, return the list +# of problems (empty = every unit signed by the expected key). Plain string compare -- no gpg here. +{ + # happy: repomd signed by exactly the expected key -> no problems. + my @ok = verify_repo_signature({ repomd => 'KEYFPR' }, { repomd => 'KEYFPR' }); + is_deeply(\@ok, [], 'verify_repo_signature: observed == expected -> 0 problems'); + + # unsigned: observed empty -> one UNSIGNED problem naming the unit + expected key. + my @us = verify_repo_signature({ repomd => 'KEYFPR' }, { repomd => '' }); + is(scalar(@us), 1, 'verify_repo_signature: empty observed yields exactly one problem'); + like($us[0], qr/^UNSIGNED repomd\b/, 'verify_repo_signature: empty observed reported as UNSIGNED'); + like($us[0], qr/expected KEYFPR/, 'verify_repo_signature: UNSIGNED names the expected key'); + + # unsigned via explicit undef observed is treated the same as empty. + my @uu = verify_repo_signature({ repomd => 'KEYFPR' }, { repomd => undef }); + like($uu[0], qr/^UNSIGNED repomd\b/, 'verify_repo_signature: undef observed reported as UNSIGNED'); + + # wrongkey: signed, but by a different key -> one WRONGKEY problem naming both. + my @wk = verify_repo_signature({ repomd => 'GOODFPR' }, { repomd => 'EVILFPR' }); + is(scalar(@wk), 1, 'verify_repo_signature: a mismatched signer yields exactly one problem'); + like($wk[0], qr/^WRONGKEY repomd: signed by EVILFPR, expected GOODFPR$/, + 'verify_repo_signature: mismatch reported as WRONGKEY : signed by , expected '); +} + done_testing; From bebc39090799180417ea5d65ed75e36b701eaff5 Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Wed, 12 Aug 2026 16:43:16 -0300 Subject: [PATCH 32/55] fix(build): reject expired/revoked signatures in the repo gate; drop unused import Review follow-up on the completeness+signature gate: - repomd_observed_signer keyed off VALIDSIG, which gpg also emits for an EXPIRED or REVOKED key (and an expired signature) -- so a no-longer-trustworthy signature would PASS the gate. Reject EXPKEYSIG/REVKEYSIG/EXPSIG explicitly before accepting VALIDSIG. - drop the now-unused have_rpm import (its only caller, assert_required_deps, was replaced by verify_target_repo). prove t/mockbuild-all.t: 68/68. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- mockbuild-all.pl | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/mockbuild-all.pl b/mockbuild-all.pl index b2ca160..675460c 100755 --- a/mockbuild-all.pl +++ b/mockbuild-all.pl @@ -13,7 +13,7 @@ use Parallel::ForkManager; use POSIX qw(strftime); use FindBin qw($RealBin); use lib $RealBin; -use MockBuildUtils qw(sh_quote print_step version_matches required_pkgs have_rpm +use MockBuildUtils qw(sh_quote print_step version_matches required_pkgs read_manifest verify_repo_packages verify_repo_signature rpm_version rpm_release rpm_sigmd5 restamp_release_line cross_copy_genesis finalize_xcat_dep bump_dep_release_suffix); @@ -1128,8 +1128,12 @@ sub repomd_observed_signer { my ($asc, $file, $home) = @_; return '' unless -f $asc && -f $file; my $h = ($home ne '') ? ' --homedir ' . sh_quote($home) : ''; - my $out = `gpg$h --status-fd=1 --verify ${\ sh_quote($asc)} ${\ sh_quote($file)} 2>/dev/null`; - for my $line (split /\n/, $out // '') { + my $out = `gpg$h --status-fd=1 --verify ${\ sh_quote($asc)} ${\ sh_quote($file)} 2>/dev/null` // ''; + # An EXPIRED or REVOKED key, or an expired signature, still emits VALIDSIG -- reject those + # explicitly so a no-longer-trustworthy signature is a problem, not a pass. A fully-good signature + # emits GOODSIG; the degraded cases emit EXPKEYSIG/REVKEYSIG/EXPSIG instead. + return '' if $out =~ /^\[GNUPG:\]\s+(?:EXPKEYSIG|REVKEYSIG|EXPSIG)\b/m; + for my $line (split /\n/, $out) { # VALIDSIG ; the trailing field is the primary fpr. if ($line =~ /^\[GNUPG:\]\s+VALIDSIG\s+(.*\S)\s*$/) { my @f = split ' ', $1; From e856a2d378aecfd4f69ceca4d0141fe5c1edd063 Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Wed, 12 Aug 2026 17:11:39 -0300 Subject: [PATCH 33/55] fix(build): repo gate signature must strictly match the CLI key; document gate in BUILD.md Per review of the gate design: - Signature: fail (SIGKEY) if --gpg-key-name cannot be resolved to a fingerprint, so the gate always confirms the repo was signed by EXACTLY the CLI key -- never a soft pass. (EL already dies loudly on a duplicate rpm version via rpm_version; Ubuntu now matches.) - Document the gate + its intentional idiosyncrasies (what 'the repo' is, duplicate=hard error, signature identity) in BUILD.md. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- BUILD.md | 29 +++++++++++++++++++++++++++++ mockbuild-all.pl | 17 ++++++++++++----- 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/BUILD.md b/BUILD.md index c3ea15b..40ee7e8 100644 --- a/BUILD.md +++ b/BUILD.md @@ -320,6 +320,35 @@ prove t/ # or: perl t/mockbuild-all.t The RPM-identity / `cross_copy_genesis` cases build tiny fixture rpms and are skipped automatically if `rpmbuild` is unavailable. +# Repository verification gate + +After each per-target repo is built + signed, `mockbuild-all.pl` runs a **manifest-driven gate** that +fails the build if the published repo is incomplete or mis-signed. It uses `packages-manifest.conf` as +the single source of truth and is layered so the decision logic is pure and unit-tested +(`MockBuildUtils::verify_repo_packages` / `verify_repo_signature`), separate from the disk/gpg I/O. + +- **Runs automatically** at the end of `deploy_target` (per `rh/` cell). Suppress with + `--no-verify-repo`. Verify an already-built repo out of band with `--verify-repo=` + (target derived from the `rh/` path, or pass `--target`; manifest from `/ + packages-manifest.conf`; key/home from `--gpg-key-name`/`--gpg-home`). +- **Completeness:** every package the target's manifest section requires (after `required_pkgs` + skip-filtering) must be present with a version satisfying its pin. +- **Signature:** the repo's `repodata/repomd.xml.asc` must be a *good* signature whose **primary-key + fingerprint equals the fingerprint of `--gpg-key-name`** — i.e. the repo was signed by exactly the + CLI key. Expired/revoked keys and expired signatures are rejected (not just `VALIDSIG`). If the CLI + key cannot be resolved to a fingerprint (not in the keyring) the gate fails (`SIGKEY`), never passes. + +**Semantic idiosyncrasies (intentional, and mirrored in the Ubuntu `sbuild-all.pl` gate):** + +- **What "the repo" is:** the EL gate reads the **binary rpm files** in the per-target dir (via + `rpm_version`); the Ubuntu gate reads the **published `binary-/Packages` index**. Both check + the artifact that ships; they differ only in the RHEL-vs-Debian notion of "the repository". +- **Duplicate = hard error:** if a required package appears with **two distinct versions** (a stale + artifact not cleaned before the build), the gate **dies loudly** rather than silently picking one — + identical on both EL (`rpm_version`) and Ubuntu (`parse_packages_index`). +- **Version pins** are the manifest's *upstream* version; the Debian gate strips the epoch/revision + (`deb_upstream_version`) before comparing, the EL gate compares `%{version}` directly. + # References - [mock project repository](https://github.com/rpm-software-management/mock) diff --git a/mockbuild-all.pl b/mockbuild-all.pl index 675460c..76979a9 100755 --- a/mockbuild-all.pl +++ b/mockbuild-all.pl @@ -1168,11 +1168,18 @@ sub verify_target_repo { # Skipped with a printed note only when no gpg key/home is configured (nothing to check against). if ($gpg_sign || $gpg_home ne '') { require_command('gpg'); - my $repomd = "$dir/repodata/repomd.xml"; - my $asc = "$repomd.asc"; - my %exp_sig = ('repomd' => gpg_key_fingerprint($gpg_key_name, $gpg_home)); - my %obs_sig = ('repomd' => repomd_observed_signer($asc, $repomd, $gpg_home)); - push @problems, verify_repo_signature(\%exp_sig, \%obs_sig); + my $repomd = "$dir/repodata/repomd.xml"; + my $asc = "$repomd.asc"; + my $exp_fpr = gpg_key_fingerprint($gpg_key_name, $gpg_home); + # STRICT: the CLI key MUST resolve to a fingerprint so we can confirm it signed the repo. If it + # does not (not in the keyring), we cannot verify -> hard fail, never a presence-only pass. + if ($exp_fpr !~ /^[0-9A-Fa-f]{16,}$/) { + push @problems, "SIGKEY: cannot resolve --gpg-key-name '$gpg_key_name' to a fingerprint (in the $gpg_home keyring?)"; + } else { + my %exp_sig = ('repomd' => $exp_fpr); + my %obs_sig = ('repomd' => repomd_observed_signer($asc, $repomd, $gpg_home)); + push @problems, verify_repo_signature(\%exp_sig, \%obs_sig); + } } else { print "[verify-repo] $tgt: no gpg key/home configured -- skipping repomd signature check\n"; } From bf5bcee9fcd8ddcd7086adec9bbc5f36ace3d32d Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Wed, 12 Aug 2026 22:13:25 -0300 Subject: [PATCH 34/55] fix(build): resolve the gpg key to exactly one fingerprint or hard-fail SIGKEY Re-review polish: gpg_key_fingerprint now returns the primary-key fingerprint only when EXACTLY ONE key matches --gpg-key-name (undef if absent or ambiguous), and the gate keys its SIGKEY hard-fail off 'undef' rather than 'looks hex' -- so a name matching multiple keys, or an unresolvable key, can never be silently accepted. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- mockbuild-all.pl | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/mockbuild-all.pl b/mockbuild-all.pl index 76979a9..6da98f4 100755 --- a/mockbuild-all.pl +++ b/mockbuild-all.pl @@ -1114,11 +1114,19 @@ sub repo_present_versions { sub gpg_key_fingerprint { my ($keyname, $home) = @_; my $h = ($home ne '') ? ' --homedir ' . sh_quote($home) : ''; - my $out = `gpg$h --with-colons --fingerprint --list-keys ${\ sh_quote($keyname)} 2>/dev/null`; - for my $line (split /\n/, $out // '') { - return $1 if $line =~ /^fpr:+([0-9A-Fa-f]+):/; # first fpr = primary key fingerprint + my $out = `gpg$h --with-colons --fingerprint --list-keys ${\ sh_quote($keyname)} 2>/dev/null` // ''; + # Collect the PRIMARY-key fingerprint of every key matching $keyname (the fpr line right after a + # 'pub' record; subkey fprs follow 'sub' and are ignored). Return undef -- not a guess -- when the + # key is absent (unresolved) or when MORE THAN ONE key matches the name (ambiguous): the caller + # then hard-fails SIGKEY rather than comparing against a possibly-wrong key. + my (@fprs, $want); + for my $line (split /\n/, $out) { + if ($line =~ /^pub:/) { $want = 1; } + elsif ($line =~ /^sub:/) { $want = 0; } + elsif ($want && $line =~ /^fpr:+([0-9A-Fa-f]+):/) { push @fprs, $1; $want = 0; } } - return $keyname; + return undef if @fprs != 1; + return $fprs[0]; } # repomd_observed_signer: run gpg --verify on the detached repomd signature and extract the identity @@ -1171,10 +1179,11 @@ sub verify_target_repo { my $repomd = "$dir/repodata/repomd.xml"; my $asc = "$repomd.asc"; my $exp_fpr = gpg_key_fingerprint($gpg_key_name, $gpg_home); - # STRICT: the CLI key MUST resolve to a fingerprint so we can confirm it signed the repo. If it - # does not (not in the keyring), we cannot verify -> hard fail, never a presence-only pass. - if ($exp_fpr !~ /^[0-9A-Fa-f]{16,}$/) { - push @problems, "SIGKEY: cannot resolve --gpg-key-name '$gpg_key_name' to a fingerprint (in the $gpg_home keyring?)"; + # STRICT: the CLI key MUST resolve to exactly one fingerprint so we can confirm it signed the + # repo. Undef => absent or ambiguous in the keyring -> we cannot verify -> hard fail, never a + # presence-only pass. + if (!defined $exp_fpr) { + push @problems, "SIGKEY: cannot resolve --gpg-key-name '$gpg_key_name' to a single fingerprint (in the $gpg_home keyring?)"; } else { my %exp_sig = ('repomd' => $exp_fpr); my %obs_sig = ('repomd' => repomd_observed_signer($asc, $repomd, $gpg_home)); From d8333008cf3e8aa183faeacd158648f13422b6a1 Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Wed, 12 Aug 2026 22:21:27 -0300 Subject: [PATCH 35/55] feat(build): --finalize-xcat-dep self-verifies each re-signed repo (gate by default) The Cross-arch genesis / finalize step re-indexes + re-signs each per-EL repo AFTER the per-target deploy_target gate ran, so the auto-run never saw the final shipped state. Run the same manifest completeness + signature gate at the end of --finalize-xcat-dep over every finalized rh/ cell, so the build script verifies its own FINAL output by default -- no external --verify-repo call needed. Suppressible with --no-verify-repo. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- mockbuild-all.pl | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/mockbuild-all.pl b/mockbuild-all.pl index 6da98f4..e7a0ddd 100755 --- a/mockbuild-all.pl +++ b/mockbuild-all.pl @@ -182,6 +182,26 @@ if ($finalize_xcat_dep) { } : undef), reindex => \&reindex_and_sign_repo, ); + + # finalize just RE-INDEXED + RE-SIGNED each per-EL repo and cross-copied the foreign-arch genesis + # in -- i.e. it produced the FINAL shipped state, which the per-target gate in deploy_target (run + # earlier, pre-finalize) never saw. So run the SAME manifest completeness + signature gate here, on + # every finalized cell, so the build script verifies its own final output by default (no external + # --verify-repo needed). Suppressible with --no-verify-repo. + unless ($no_verify_repo) { + my %seen; + for my $root ($x86, $ppc) { + my @cells = (glob("$root/rh*/x86_64"), glob("$root/rh*/ppc64le")); + for my $d (sort @cells) { + next unless -d $d; + my $abs = abs_path($d); + next if $seen{$abs}++; + my $tgt = derive_target_from_repo_path($abs) + or die "FATAL: finalize verify -- cannot derive target from '$abs'\n"; + verify_target_repo($abs, $tgt); + } + } + } exit 0; } From 411da1539eb890ae896cfbf1f1db58a7440674ad Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Thu, 13 Aug 2026 06:29:42 -0300 Subject: [PATCH 36/55] fix(xcat-dep): scrub the shared genesis mock chroot before each build The xCAT-genesis-base build runs buildrpms.pl without a per-run --mock-uniqueext, so its mock chroot (xCAT-genesis-base-) is shared across CD runs. mock only scrubs a chroot on a SUCCESSFUL build, so a killed or failFast-interrupted prior run leaves the chroot stunted (missing /bin/sh), and mock REUSES that corpse on the next run, which then dies with FileNotFoundError: '/bin/sh' during genesis -- failing an otherwise-healthy build until an operator manually scrubs the chroot. Scrub the genesis chroot before building it: a lock-safe, best-effort 'mock --scrub=chroot --scrub=bootstrap' (skipped if a concurrent build holds the lock, a no-op on the first run before the config exists). Any corpse from an interrupted run is dropped and mock recreates the chroot fresh from the cached root, so an interrupted build can no longer poison the next one. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- mockbuild-all.pl | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/mockbuild-all.pl b/mockbuild-all.pl index e7a0ddd..a0cbd41 100755 --- a/mockbuild-all.pl +++ b/mockbuild-all.pl @@ -520,7 +520,15 @@ if (!$skip_build) { # buildrpms.pl's rpmdev-setuptree only runs during env setup, not per build, so create the # rpmbuild tree ourselves for this per-target HOME (else $HOME/rpmbuild/SOURCES is missing). my $mktree = join(' ', map { sh_quote("$genesis_home/rpmbuild/$_") } qw(SOURCES SPECS BUILD BUILDROOT RPMS SRPMS)); - my $cmd = "mkdir -p $mktree && HOME=" . sh_quote($genesis_home) . ' ' . join(' ', + # The genesis chroot (xCAT-genesis-base-) is SHARED across runs -- buildrpms.pl builds it + # without a per-run --mock-uniqueext. mock's post-build scrub only runs on SUCCESS, so a killed + # or failFast-interrupted prior run leaves the chroot stunted (missing /bin/sh), and mock REUSES + # the corpse on the next run -> "FileNotFoundError: '/bin/sh'". Scrub it FIRST (lock-safe -- mock + # refuses if a concurrent build holds it -- and best-effort: a no-op on the first run before the + # config exists) so mock recreates the chroot fresh from the cached root (fast, no re-bootstrap). + my $genesis_scrub = "{ mock -r " . sh_quote("xCAT-genesis-base-$target") + . " --scrub=chroot --scrub=bootstrap >/dev/null 2>&1 || true; }"; + my $cmd = "mkdir -p $mktree && $genesis_scrub && HOME=" . sh_quote($genesis_home) . ' ' . join(' ', 'perl', sh_quote("$xcat_src/buildrpms.pl"), '--package', 'xCAT-genesis-base', '--target', sh_quote($target), From 9bf3f7ad228f686cee06be8f6092b9627ff8a44a Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Thu, 13 Aug 2026 07:34:09 -0300 Subject: [PATCH 37/55] fix(xcat-dep): run mock builds in an rslave mount namespace to protect the host cgroup mock mounts /sys/fs/cgroup into every build chroot. On the systemd build hosts every mount carries `shared` propagation, so the chroot cgroup joins the same peer group as the host's own /sys/fs/cgroup. When mock tears a chroot down -- its post-build --scrub, or an aborted build's cleanup -- the cgroup unmount PROPAGATES back through the shared peer group and unmounts the HOST's /sys/fs/cgroup. Every subsequent mock (and even new login sessions) then fails with 'Failed to determine whether the unified cgroups hierarchy is used: No medium found', wedging the whole build host until an operator remounts cgroup2. ppc64le is hit hardest because it leaks corpse chroot mounts on abort, but x86_64 shares the identical shared-cgroup exposure and is one bad abort away from the same failure. Re-exec mockbuild-all.pl inside a private mount namespace made rslave (unshare --mount --propagation slave): the namespace still sees host mounts one-way, but nothing mock mounts or unmounts can propagate out to the host, so a chroot teardown can no longer unmount the host cgroup. The namespace also auto-reaps every mount mock leaks when the process exits, so an aborted build no longer strands corpse mounts under /var/lib/mock. Best-effort and guarded: only re-execs as root with unshare present, and MOCKBUILD_ALL_MOUNTNS prevents a re-exec loop. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- mockbuild-all.pl | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/mockbuild-all.pl b/mockbuild-all.pl index a0cbd41..27688f6 100755 --- a/mockbuild-all.pl +++ b/mockbuild-all.pl @@ -18,6 +18,36 @@ use MockBuildUtils qw(sh_quote print_step version_matches required_pkgs rpm_version rpm_release rpm_sigmd5 restamp_release_line cross_copy_genesis finalize_xcat_dep bump_dep_release_suffix); +# --- Mount-namespace isolation: guard the host cgroup against mock teardown propagation ---------- +# mock mounts /sys/fs/cgroup into every build chroot. On these systemd build hosts every mount is +# `shared`, so the chroot's cgroup joins the HOST's cgroup peer group. When mock tears a chroot down +# -- its post-build --scrub, or an aborted build's cleanup -- the unmount PROPAGATES back through the +# shared peer group and unmounts the HOST's /sys/fs/cgroup, after which every later mock (and even new +# login sessions) dies with "Failed to determine whether the unified cgroups hierarchy is used: No +# medium found". This bit ppc hardest (it leaks corpse chroot mounts on abort) but x86 shares the same +# shared-cgroup exposure. Re-exec inside a private mount namespace made rslave +# (`unshare --mount --propagation slave`): the namespace still sees host mounts (slave = one-way), but +# nothing mock mounts/unmounts can propagate OUT to the host. As a bonus the namespace tears down every +# mount mock leaks when we exit, so an aborted build can no longer leave corpse mounts under +# /var/lib/mock. Best-effort: only as root (needs CAP_SYS_ADMIN) and only if `unshare` exists; +# otherwise warn loudly and continue unisolated. MOCKBUILD_ALL_MOUNTNS guards against a re-exec loop. +unless ($ENV{MOCKBUILD_ALL_MOUNTNS}) { + if ($> != 0) { + warn "WARN: not root -- skipping mount-namespace isolation (host-cgroup propagation guard); " + . "run as root in CI so mock chroot teardown cannot unmount the host /sys/fs/cgroup\n"; + } elsif (system('sh', '-c', 'command -v unshare >/dev/null 2>&1') != 0) { + warn "WARN: 'unshare' not found -- skipping mount-namespace isolation; mock chroot teardown " + . "may unmount the host /sys/fs/cgroup on a shared-propagation host\n"; + } else { + $ENV{MOCKBUILD_ALL_MOUNTNS} = 1; + my @reexec = ('unshare', '--mount', '--propagation', 'slave', '--', $^X, $0, @ARGV); + exec { $reexec[0] } @reexec; + # exec only returns on failure -- fall through and run unisolated rather than abort the build. + warn "WARN: exec unshare failed ($!) -- continuing without mount-namespace isolation\n"; + delete $ENV{MOCKBUILD_ALL_MOUNTNS}; + } +} + my $script_dir = abs_path(dirname(__FILE__)); my $repo_root = abs_path($script_dir); my $xcat_src = "$repo_root/../xcat-core"; From 6e51366cef385490c76539491277addfe0a9c75d Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Thu, 13 Aug 2026 10:05:09 -0300 Subject: [PATCH 38/55] fix(build): --verify-repo must assert the repomd signature; abs-path the re-exec Two review follow-ups to the EL matrix build: 1. Standalone --verify-repo silently PASSED the signature check when no --gpg-key-name/--gpg-home was configured, contradicting its documented contract (it advertises a signature assertion). verify_target_repo now takes a sig_required flag, set for the standalone mode, so a missing signing identity is a loud SIGKEY failure instead of a false pass. The pipeline auto-runs are unaffected (they always pass --gpg-sign). 2. The mount-namespace re-exec used a bare $0, which the re-exec'd perl could fail to find if the script was invoked via a $PATH name from a different cwd (exec succeeds, so the unisolated fallback never runs and the build dies hard). Resolve $0 to an absolute path first. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- mockbuild-all.pl | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/mockbuild-all.pl b/mockbuild-all.pl index 27688f6..6580167 100755 --- a/mockbuild-all.pl +++ b/mockbuild-all.pl @@ -40,7 +40,10 @@ unless ($ENV{MOCKBUILD_ALL_MOUNTNS}) { . "may unmount the host /sys/fs/cgroup on a shared-propagation host\n"; } else { $ENV{MOCKBUILD_ALL_MOUNTNS} = 1; - my @reexec = ('unshare', '--mount', '--propagation', 'slave', '--', $^X, $0, @ARGV); + # Absolute path to self so the re-exec'd perl finds the script regardless of cwd or a bare + # $PATH invocation (unshare does not change cwd, but $0 may be relative or a bare name). + my $self = abs_path($0) // $0; + my @reexec = ('unshare', '--mount', '--propagation', 'slave', '--', $^X, $self, @ARGV); exec { $reexec[0] } @reexec; # exec only returns on failure -- fall through and run unisolated rather than abort the build. warn "WARN: exec unshare failed ($!) -- continuing without mount-namespace isolation\n"; @@ -180,7 +183,9 @@ if ($verify_repo ne '') { my $tgt = $target ne '' ? $target : derive_target_from_repo_path($rdir); die "--verify-repo: cannot derive a target from repo path '$rdir'; pass --target\n" if !defined($tgt) || $tgt eq ''; - verify_target_repo($rdir, $tgt); # manifest defaults to repo_root/packages-manifest.conf + # sig_required=1: a standalone verify MUST assert the repomd signature (its documented contract), + # never silently skip it when no gpg key/home is configured (that would be a false PASS on sigs). + verify_target_repo($rdir, $tgt, undef, 1); # manifest defaults to repo_root/packages-manifest.conf exit 0; } @@ -1217,7 +1222,7 @@ sub repomd_observed_signer { # problem. Both the automatic post-build gate (deploy_target) and the standalone --verify-repo mode # call this, so there is exactly one gate implementation. sub verify_target_repo { - my ($dir, $tgt, $manifest) = @_; + my ($dir, $tgt, $manifest, $sig_required) = @_; $manifest //= "$repo_root/packages-manifest.conf"; my %MAN = read_manifest($manifest); my %req = %{ $MAN{$tgt} // {} }; @@ -1247,6 +1252,10 @@ sub verify_target_repo { my %obs_sig = ('repomd' => repomd_observed_signer($asc, $repomd, $gpg_home)); push @problems, verify_repo_signature(\%exp_sig, \%obs_sig); } + } elsif ($sig_required) { + # Standalone --verify-repo advertises a signature check; with no keyring we cannot resolve the + # CLI key or read the signer, so refuse rather than silently pass (which would be a false PASS). + push @problems, "SIGKEY: --verify-repo requires --gpg-key-name + --gpg-home to check the repomd signature (none configured)"; } else { print "[verify-repo] $tgt: no gpg key/home configured -- skipping repomd signature check\n"; } From dcd193a4f9a8fc174ed9e616922d5f65780b5a30 Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Thu, 13 Aug 2026 10:51:13 -0300 Subject: [PATCH 39/55] docs(build): note pyodbc is distro-provided (python3-pyodbc), not built by xcat-dep The legacy pyodbc/ RPM spec is kept for reference but pyodbc is absent from every target's manifest because modern EL ships python3-pyodbc from appstream/EPEL. Document this so the omission (vs the 2.16/2.17 xcat-dep repos) is not mistaken for a gap. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- BUILD.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/BUILD.md b/BUILD.md index 40ee7e8..94f52c7 100644 --- a/BUILD.md +++ b/BUILD.md @@ -67,6 +67,10 @@ built for — every target, because some deployments still use it. The lists wer (EL, arch), `dnf install xCAT` from xcat.org latest, and the packages whose `from_repo=xcat-dep` are exactly the required set. See the file header for details. +`pyodbc` is intentionally **not** built or listed in any target's manifest: modern EL provides +`python3-pyodbc` from appstream/EPEL, so xcat-dep no longer ships its own. The legacy `pyodbc/` +directory (an old `pyodbc-3.0.7` RPM spec) is kept for historical reference only. + Build failures are **not tolerated**: any required (manifest) package that fails to build fails the whole run. From 06ffc659445d96847db81ce0741a7f8de88eb98f Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Thu, 13 Aug 2026 11:07:54 -0300 Subject: [PATCH 40/55] docs(build): correct scrub lock semantics; bound genesis scrub; skip re-exec in build-free modes Review follow-ups (all low-impact): - The genesis pre-build scrub and scrub_buildroot comments claimed mock 'refuses and we skip' on a held buildroot lock; mock actually BLOCKS until release. Correct the wording and wrap the genesis scrub in 'timeout 300' so even a pathological stale-lock wait can never hang the build (the CD topology never runs a second same-target build concurrently, so it is belt-and-suspenders). - Drop the inaccurate 'no re-bootstrap' note: --scrub=bootstrap intentionally forces a re-bootstrap from the root cache (that bootstrap is the leak). - The mount-namespace re-exec ran for every invocation, printing a spurious non-root warning for the documented no-root, build-free modes (--verify-repo, --finalize-xcat-dep). Skip the re-exec for those (no mock, no cgroup exposure). - Soften the abs_path($0) comment: it resolves a relative $0 against cwd but does not search $PATH. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- mockbuild-all.pl | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/mockbuild-all.pl b/mockbuild-all.pl index 6580167..7d9b958 100755 --- a/mockbuild-all.pl +++ b/mockbuild-all.pl @@ -31,7 +31,10 @@ use MockBuildUtils qw(sh_quote print_step version_matches required_pkgs # mount mock leaks when we exit, so an aborted build can no longer leave corpse mounts under # /var/lib/mock. Best-effort: only as root (needs CAP_SYS_ADMIN) and only if `unshare` exists; # otherwise warn loudly and continue unisolated. MOCKBUILD_ALL_MOUNTNS guards against a re-exec loop. -unless ($ENV{MOCKBUILD_ALL_MOUNTNS}) { +# Build-free modes (--verify-repo, --finalize-xcat-dep) run no mock and are documented no-root, so they +# skip the re-exec entirely -- no cgroup exposure, and no spurious non-root warning. +my $mountns_build_free = grep { /^--(?:verify-repo|finalize-xcat-dep)(?:=|$)/ } @ARGV; +unless ($ENV{MOCKBUILD_ALL_MOUNTNS} || $mountns_build_free) { if ($> != 0) { warn "WARN: not root -- skipping mount-namespace isolation (host-cgroup propagation guard); " . "run as root in CI so mock chroot teardown cannot unmount the host /sys/fs/cgroup\n"; @@ -40,8 +43,9 @@ unless ($ENV{MOCKBUILD_ALL_MOUNTNS}) { . "may unmount the host /sys/fs/cgroup on a shared-propagation host\n"; } else { $ENV{MOCKBUILD_ALL_MOUNTNS} = 1; - # Absolute path to self so the re-exec'd perl finds the script regardless of cwd or a bare - # $PATH invocation (unshare does not change cwd, but $0 may be relative or a bare name). + # Absolute path to self (resolved against cwd, which unshare preserves) so the re-exec'd perl + # finds a relative $0. A bare $PATH-only $0 isn't resolved (abs_path doesn't search $PATH), but + # a shell invocation yields a full $0 there anyway. my $self = abs_path($0) // $0; my @reexec = ('unshare', '--mount', '--propagation', 'slave', '--', $^X, $self, @ARGV); exec { $reexec[0] } @reexec; @@ -558,10 +562,14 @@ if (!$skip_build) { # The genesis chroot (xCAT-genesis-base-) is SHARED across runs -- buildrpms.pl builds it # without a per-run --mock-uniqueext. mock's post-build scrub only runs on SUCCESS, so a killed # or failFast-interrupted prior run leaves the chroot stunted (missing /bin/sh), and mock REUSES - # the corpse on the next run -> "FileNotFoundError: '/bin/sh'". Scrub it FIRST (lock-safe -- mock - # refuses if a concurrent build holds it -- and best-effort: a no-op on the first run before the - # config exists) so mock recreates the chroot fresh from the cached root (fast, no re-bootstrap). - my $genesis_scrub = "{ mock -r " . sh_quote("xCAT-genesis-base-$target") + # the corpse on the next run -> "FileNotFoundError: '/bin/sh'". Scrub it FIRST (best-effort: a + # no-op on the first run before the config exists) so mock recreates the chroot from the cached + # root; --scrub=bootstrap goes too (the per-uniqueext bootstrap is the leak), so the fresh chroot + # re-bootstraps from the root cache. mock takes the buildroot lock for --scrub and BLOCKS (not + # skips) if a concurrent build holds it, but within a run the scrub is sequential before the + # build and the CD topology never runs a second same-target build at once; `timeout` bounds even + # a pathological wait so a stale lock can never hang the build. + my $genesis_scrub = "{ timeout 300 mock -r " . sh_quote("xCAT-genesis-base-$target") . " --scrub=chroot --scrub=bootstrap >/dev/null 2>&1 || true; }"; my $cmd = "mkdir -p $mktree && $genesis_scrub && HOME=" . sh_quote($genesis_home) . ' ' . join(' ', 'perl', sh_quote("$xcat_src/buildrpms.pl"), @@ -1049,8 +1057,9 @@ sub run_step { } } -# Scrub a single mock buildroot via mock's own lock-safe --scrub. Never rm: if a concurrent build -# still holds the chroot lock, mock refuses and we skip it. Failures (already scrubbed, locked, or +# Scrub a single mock buildroot via mock's own --scrub (never rm). mock takes the buildroot lock for +# the scrub, so a concurrent build holding it makes mock BLOCK until release rather than corrupt a live +# chroot (this can never rm a chroot out from under a running build). Failures (already scrubbed or # config missing) are tolerated -- a cleanup hiccup must never fail the build. Scrubs both the # build chroot and its per-uniqueext bootstrap chroot (each build step gets its own bootstrap, so # both must go or /var/lib/mock still leaks). The shared root cache under /var/cache/mock is kept, From 8a8789abe145fe7acb5ab59aaa2e10b220e8fa1c Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Thu, 13 Aug 2026 11:12:16 -0300 Subject: [PATCH 41/55] docs(build): add a Packages notes section (pyodbc distro-provided; conserver vs goconserver) Move the pyodbc note under a dedicated 'Packages notes' section and add a note that conserver is shipped for completeness/backward-compat only: core packages depend on goconserver, so conserver is not pulled in as a dependency -- to use it you must install it explicitly, then disable the goconserver service and enable the conserver service. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- BUILD.md | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/BUILD.md b/BUILD.md index 94f52c7..f6f082e 100644 --- a/BUILD.md +++ b/BUILD.md @@ -67,10 +67,6 @@ built for — every target, because some deployments still use it. The lists wer (EL, arch), `dnf install xCAT` from xcat.org latest, and the packages whose `from_repo=xcat-dep` are exactly the required set. See the file header for details. -`pyodbc` is intentionally **not** built or listed in any target's manifest: modern EL provides -`python3-pyodbc` from appstream/EPEL, so xcat-dep no longer ships its own. The legacy `pyodbc/` -directory (an old `pyodbc-3.0.7` RPM spec) is kept for historical reference only. - Build failures are **not tolerated**: any required (manifest) package that fails to build fails the whole run. @@ -86,6 +82,16 @@ the whole run. 8. Tarball creation for both repo trees 9. Summary generation (`summary.txt`) +# Packages notes + +- **`pyodbc`** is intentionally not built or listed in any target's manifest: modern EL provides + `python3-pyodbc` from appstream/EPEL, so xcat-dep no longer ships its own. The legacy `pyodbc/` + directory (an old `pyodbc-3.0.7` RPM spec) is kept for historical reference only. +- **`conserver`** was replaced by `goconserver` but is provided for completeness and backward + compatibility. Core packages depend on `goconserver`; to use `conserver` you must install it + explicitly (it is **not** pulled in as a dependency), disable the `goconserver` service and enable + the `conserver` service. + # Skip and Control Flags Use these flags to skip specific operations: From a58d6feadd0f29df2798716ab94e525a3a798b58 Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Thu, 13 Aug 2026 12:25:53 -0300 Subject: [PATCH 42/55] docs(build): fix genesis-scrub comment (genesis has no uniqueext); name conserver-xcat in BUILD.md Landing-review nits: the genesis pre-scrub comment said 'per-uniqueext bootstrap' but the genesis chroot carries no --mock-uniqueext; and the Packages note said 'conserver' where the shipped package is 'conserver-xcat' ('dnf install conserver' would pull the distro package, not xCAT's). Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- BUILD.md | 8 ++++---- mockbuild-all.pl | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/BUILD.md b/BUILD.md index f6f082e..cea63b1 100644 --- a/BUILD.md +++ b/BUILD.md @@ -87,10 +87,10 @@ the whole run. - **`pyodbc`** is intentionally not built or listed in any target's manifest: modern EL provides `python3-pyodbc` from appstream/EPEL, so xcat-dep no longer ships its own. The legacy `pyodbc/` directory (an old `pyodbc-3.0.7` RPM spec) is kept for historical reference only. -- **`conserver`** was replaced by `goconserver` but is provided for completeness and backward - compatibility. Core packages depend on `goconserver`; to use `conserver` you must install it - explicitly (it is **not** pulled in as a dependency), disable the `goconserver` service and enable - the `conserver` service. +- **`conserver-xcat`** was replaced by `goconserver` but is provided for completeness and backward + compatibility. Core packages depend on `goconserver`; to use conserver you must install + `conserver-xcat` explicitly (it is **not** pulled in as a dependency), disable the `goconserver` + service and enable the `conserver` service. # Skip and Control Flags diff --git a/mockbuild-all.pl b/mockbuild-all.pl index 7d9b958..5956a98 100755 --- a/mockbuild-all.pl +++ b/mockbuild-all.pl @@ -564,8 +564,8 @@ if (!$skip_build) { # or failFast-interrupted prior run leaves the chroot stunted (missing /bin/sh), and mock REUSES # the corpse on the next run -> "FileNotFoundError: '/bin/sh'". Scrub it FIRST (best-effort: a # no-op on the first run before the config exists) so mock recreates the chroot from the cached - # root; --scrub=bootstrap goes too (the per-uniqueext bootstrap is the leak), so the fresh chroot - # re-bootstraps from the root cache. mock takes the buildroot lock for --scrub and BLOCKS (not + # root; --scrub=bootstrap goes too (the genesis bootstrap chroot is part of the leak; genesis + # carries no --mock-uniqueext), so the fresh chroot re-bootstraps from the root cache. mock takes the buildroot lock for --scrub and BLOCKS (not # skips) if a concurrent build holds it, but within a run the scrub is sequential before the # build and the CD topology never runs a second same-target build at once; `timeout` bounds even # a pathological wait so a stale lock can never hang the build. From eb7fc47bf055fc2fbb2bfcc79256ddd9d2a24f3b Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Fri, 21 Aug 2026 18:04:41 -0300 Subject: [PATCH 43/55] fix(xcat-dep): test the skip-mode completeness gate composition PR #62 review R3.3 objected that the --skip-genesis/--skip-perl/--skip-xcat-dep modes still validated every manifest package, so a clean skip run failed the completeness gate. The gate was reworked to derive its expected set with required_pkgs(...skip flags), but the fixtures tested required_pkgs and verify_repo_packages only in isolation -- not the composition that the gate actually performs. Add a fixture that feeds the required_pkgs-filtered expected set into verify_repo_packages with the skipped package absent from the repo, asserting no MISSING is reported; a paired unfiltered case proves the skip filter is load-bearing (the same absent package IS flagged without it). Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- t/mockbuild-all.t | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/t/mockbuild-all.t b/t/mockbuild-all.t index 52fe8b6..76076c9 100644 --- a/t/mockbuild-all.t +++ b/t/mockbuild-all.t @@ -39,6 +39,24 @@ is_deeply([required_pkgs(\@all, 0, 0, 1)], [qw(perl-IO-Stty perl-Sys-Virt xCAT-g is_deeply([required_pkgs(\@all, 1, 1, 1)], [], 'all skips -> nothing required (a clean skip run validates nothing)'); +# ---- gate composition: a clean --skip-* run does not flag the skipped package as MISSING ------ +# (PR #62 R3.3) verify_target_repo derives its expected set with required_pkgs(...skip flags), so a +# package a skip mode intentionally did not build must NOT be reported missing by the completeness +# gate. This tests the composition (required_pkgs -> verify_repo_packages), not either half alone. +{ + my %pins = ('elilo-xcat' => '3.14', 'perl-IO-Stty' => '0.04', 'xCAT-genesis-base' => '2.*'); + my $present = { 'elilo-xcat' => '3.14', 'perl-IO-Stty' => '0.04' }; # --skip-genesis: no genesis rpm built + my @keep = required_pkgs([sort keys %pins], 1, 0, 0); # skip_genesis + my %expected = map { $_ => $pins{$_} } @keep; + is_deeply([verify_repo_packages(\%expected, $present)], [], + 'gate: --skip-genesis run with genesis absent reports no MISSING (skipped pkg not required)'); + + # Sanity: WITHOUT the skip filter that same absent genesis IS flagged -- proving the filter is load-bearing. + my @unfiltered = verify_repo_packages(\%pins, $present); + is(scalar(@unfiltered), 1, 'gate: unfiltered, the absent genesis is flagged MISSING'); + like($unfiltered[0], qr/^MISSING xCAT-genesis-base\b/, 'gate: the flag names the absent genesis'); +} + # ---- version_matches: exact + shell-glob pins ------------------------------------------------ ok( version_matches('2.19.0', '2.*'), '2.* matches 2.19.0'); ok( version_matches('2.18.2', '2.*'), '2.* matches 2.18.2 (walks with xcat-core)'); From e898911431e487b6fcf0614605105d6420dad33a Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Fri, 21 Aug 2026 18:19:57 -0300 Subject: [PATCH 44/55] fix(goconserver): correct stale "vendored tree" comments PR #62 review flagged doc drift. The goconserver build was moved off a committed vendor/ tree to modules fetched from the Go proxy and pinned by a committed go.sum, but three comments still described a vendored tree (and one called the compile "offline", which it is not -- mock networking is on and the proxy is reachable). Align the comments with the actual no-vendor, go.sum-pinned build. Comment-only; no behavior change. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- goconserver/mockbuild.pl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/goconserver/mockbuild.pl b/goconserver/mockbuild.pl index 2d4be86..c6d3ec8 100755 --- a/goconserver/mockbuild.pl +++ b/goconserver/mockbuild.pl @@ -22,7 +22,7 @@ my $version = '0.3.3'; my $go_repo = 'https://github.com/xcat2/goconserver.git'; # Immutable pin: goconserver 0.3.3 is unreleased (newest tag v0.3.2) so it lives only on master. # mockbuild-all.pl passes --go-ref with the canonical pin; this default keeps standalone runs -# reproducible too. The committed vendored/ tree (go.mod/go.sum/vendor) corresponds to THIS SHA. +# reproducible too. The committed go.mod/go.sum (no vendor tree) correspond to THIS SHA. my $go_ref = '6166fe5ec1c5b3c20475e322a9f0e8e93c87e45f'; my $release_suffix = ''; # CD Release bump (".snap."); passed by mockbuild-all.pl my $build_timestamp; @@ -130,7 +130,7 @@ die "pinned go.mod/go.sum missing under $gomod_dir (regenerate per gomod/README. copy("$gomod_dir/go.mod", "$src_dir/go.mod") or die "copy go.mod: $!\n"; copy("$gomod_dir/go.sum", "$src_dir/go.sum") or die "copy go.sum: $!\n"; -# --- Assemble SRPM sources: the source tree (incl. vendor) + the xcat-authored unit + config --- +# --- Assemble SRPM sources: the source tree (go.mod/go.sum, no vendor) + the xcat-authored unit + config --- print_step("Assemble SRPM sources"); my $srctop = "goconserver-$version"; my $staged = "$work_dir/$srctop"; @@ -173,7 +173,7 @@ console: log_timestamp: true CONF -# --- Spec: the Go compile runs in %build INSIDE the chroot, offline, from the vendored tree --- +# --- Spec: the Go compile runs in %build INSIDE the chroot; modules fetched from the proxy, pinned by go.sum --- print_step("Write spec"); my $spec_file = "$work_dir/goconserver.spec"; write_file($spec_file, <<"SPEC"); From 5addcaa976c7e6ee3f5b54c0aa12eaf3c33dd8b1 Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Fri, 21 Aug 2026 18:42:28 -0300 Subject: [PATCH 45/55] fix(xcat-dep): drop host-install smoke; fix ppc mock-root collision PR #62 review raised two build-side problems: 1. The child builders installed each freshly built RPM onto the build host ("dnf -y install" + a run smoke). Because mockbuild-all builds el8/el9/el10 on one host, this installs a foreign-EL RPM into the host RPM database and corrupts it. The real install-and-run verification already happens in the CI Test phase (cluster-test.pl boots a matching MN and installs xCAT + the freshly built xcat-dep there), so the host-side smoke was redundant. Remove the install/smoke step from every builder and the perl aggregate builder, and drop the now-dead --skip-install flag (builders, mockbuild-all.pl, and the pipeline invocations). The perl builder's --jobs 1 throttle existed only to avoid host dnf-lock contention during that install, so it goes too (perl packages build in parallel again). Also drop goconserver's now-unused run_rc. 2. build_mock_uniqueext truncated the run id by keeping the LAST 24 chars, which for the 7-char "ppc64le" arch dropped the leading EL digit -- so alma+epel-{8,9,10}-ppc64le collapsed to the same run part. goconserver compiles every EL in the el10 chroot (build_cfg rewritten to -10-), so the chroot name is identical across the three ELs and the uniqueext was the only thing keeping their mock roots apart: with parallel targets the three ppc goconserver builds raced in one root. Keep a readable leading token AND append a short digest of the full id so distinct ids always yield distinct uniqueext. Moved the helper into MockBuildUtils.pm and added fixtures (distinct per EL on a long run id, both arches, determinism). Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- BUILD.md | 38 +++++++++---------------- MockBuildUtils.pm | 39 +++++++++++++++++++++++++ conserver/mockbuild.pl | 24 +++------------- elilo/mockbuild.pl | 35 ----------------------- goconserver/mockbuild.pl | 24 ---------------- grub2-xcat/mockbuild.pl | 46 ------------------------------ ipmitool/mockbuild.pl | 58 -------------------------------------- mockbuild-all.pl | 30 ++------------------ mockbuild-perl-packages.pl | 25 ---------------- syslinux/mockbuild.pl | 46 ------------------------------ t/mockbuild-all.t | 32 ++++++++++++++++++++- xnba/mockbuild.pl | 4 --- 12 files changed, 90 insertions(+), 311 deletions(-) diff --git a/BUILD.md b/BUILD.md index cea63b1..b969225 100644 --- a/BUILD.md +++ b/BUILD.md @@ -75,12 +75,16 @@ the whole run. 1. Optional chroot cleanup (`--scrub-all-chroots`) 2. Parallel build execution — only the target's manifest packages; any failure fails the run 3. Post-build chroot scrub — reclaims each build step's mock chroot (unless `--keep-buildroots`) -4. Optional install/smoke checks inside child builders (disabled with `--skip-install`) -5. Binary RPM collection into `repo//` -6. Source RPM collection into `repo-src/` -7. `createrepo --update` on both repo trees -8. Tarball creation for both repo trees -9. Summary generation (`summary.txt`) +4. Binary RPM collection into `repo//` +5. Source RPM collection into `repo-src/` +6. `createrepo --update` on both repo trees +7. Tarball creation for both repo trees +8. Summary generation (`summary.txt`) + +The build process does **not** install any built RPM onto the build host. Installing an EL8/EL9 +package on the (single, possibly EL10) build host corrupts the host RPM database; the real +install-and-run verification happens in the CI's separate Test phase (`cluster-test.pl` boots a +matching MN and installs xCAT + the freshly built xcat-dep there). # Packages notes @@ -96,8 +100,6 @@ the whole run. Use these flags to skip specific operations: -- `--skip-install` - - Skips install/smoke checks performed by child builder scripts after RPM build. - `--skip-genesis` - Skips the `xCAT-genesis-base` build (`/buildrpms.pl --package xCAT-genesis-base`). - `--skip-xcat-dep` @@ -194,15 +196,15 @@ perl ./mockbuild-all.pl \ Notes: -- Install/smoke checks run by default inside child builders. -- Add `--skip-install` to skip those checks. +- The build never installs a built RPM onto the build host (see above); install-and-run + verification is the CI Test phase's job. - Add `--skip-genesis` to skip the `xCAT-genesis-base` build (the only step that invokes `/buildrpms.pl`). - `` is optional; when omitted it is timestamp-based. # Common Build Modes -xcat-dep repo (with install/smoke checks): +xcat-dep repo: ```bash cd @@ -212,17 +214,6 @@ perl ./mockbuild-all.pl \ --scrub-all-chroots ``` -xcat-dep repo (skip install/smoke checks): - -```bash -cd -perl ./mockbuild-all.pl \ - --repo-root \ - --xcat-source \ - --scrub-all-chroots \ - --skip-install -``` - Dependency repo without the `xCAT-genesis-base` build: ```bash @@ -231,8 +222,7 @@ perl ./mockbuild-all.pl \ --repo-root \ --xcat-source \ --scrub-all-chroots \ - --skip-genesis \ - --skip-install + --skip-genesis ``` Collection-only pass from existing build artifacts: diff --git a/MockBuildUtils.pm b/MockBuildUtils.pm index f444a44..45879d7 100644 --- a/MockBuildUtils.pm +++ b/MockBuildUtils.pm @@ -10,6 +10,7 @@ use File::Basename qw(basename); use File::Copy qw(copy); use File::Find; use Sys::Hostname; +use Digest::MD5 qw(md5_hex); our @EXPORT_OK = qw( sh_quote print_step @@ -17,6 +18,7 @@ our @EXPORT_OK = qw( verify_repo_packages verify_repo_signature rpm_version rpm_release rpm_sigmd5 rpm_is_signed restamp_release_line cross_copy_genesis finalize_xcat_dep bump_dep_release_suffix + build_mock_uniqueext ); # sh_quote: single-quote a string for safe use in a shell command. @@ -377,4 +379,41 @@ sub bump_dep_release_suffix { return $bumped; } +# build_mock_uniqueext: a mock --uniqueext UNIQUE per (run, build-step) so concurrent mock builds +# never share a chroot root (/var/lib/mock/-). $run is the per-target run id +# (e.g. "alma+epel-8-ppc64le-"), $seq orders the step, $label names the package. +# +# The run id must NOT be blindly tail-truncated. The per-target id leads with the EL/arch token, and +# for the 7-char "ppc64le" arch the EL digit is exactly what falls off the front of a keep-the-last-24 +# truncation -- so alma+epel-{8,9,10}-ppc64le all collapse to the same run part. That is catastrophic +# for goconserver, which compiles EVERY EL in the el10 chroot (build_cfg rewritten to -10-): the +# chroot NAME is then identical across the three ELs, and the uniqueext is the ONLY thing keeping +# their roots apart, so three concurrent el8/el9/el10 ppc64le goconserver builds race in one root. +# When the id is too long, keep a readable leading token AND append a short digest of the FULL id, so +# distinct ids always yield distinct uniqueext regardless of where in the string they differ. +sub build_mock_uniqueext { + my ($run, $seq, $label) = @_; + + my $run_part = defined($run) ? $run : 'run'; + $run_part =~ s/[^A-Za-z0-9_.-]+/-/g; + $run_part =~ s/^-+|-+$//g; + $run_part = 'run' if $run_part eq ''; + if (length($run_part) > 24) { + my $digest = substr(md5_hex($run_part), 0, 8); + (my $head = substr($run_part, 0, 15)) =~ s/-+$//; + $run_part = "$head-$digest"; + } + + my $label_part = defined($label) ? $label : 'step'; + $label_part =~ s/[^A-Za-z0-9_.-]+/-/g; + $label_part =~ s/^-+|-+$//g; + $label_part = 'step' if $label_part eq ''; + $label_part = substr($label_part, 0, 20) if length($label_part) > 20; + + my $idx = defined($seq) ? int($seq) : 0; + $idx = 0 if $idx < 0; + + return sprintf("mba-%02d-%s-%s", $idx, $run_part, $label_part); +} + 1; diff --git a/conserver/mockbuild.pl b/conserver/mockbuild.pl index 6d175e6..e03c959 100755 --- a/conserver/mockbuild.pl +++ b/conserver/mockbuild.pl @@ -1,11 +1,10 @@ #!/usr/bin/perl # # mockbuild.pl - build conserver-xcat (the traditional C conserver, 8.2.1) for one -# mock target and smoke-test the resulting binaries. Mirrors the other xcat-dep +# mock target. Mirrors the other xcat-dep # builders (goconserver/mockbuild.pl, ipmitool/mockbuild.pl): stage sources + spec, -# build a SRPM, `mock --rebuild` it in the target chroot, copy the RPMs to -# --result-dir, then (unless --skip-install) install into the chroot and run -# `console -V` / `conserver -V` to confirm the binaries work. +# build a SRPM, `mock --rebuild` it in the target chroot, and copy the RPMs to +# --result-dir. # # conserver is NOT part of the default mockbuild-all.pl dep set (xCAT uses goconserver), # so this builder is standalone. Usage: @@ -28,7 +27,6 @@ my $mock_cfg = ''; my $mock_uniqueext = ''; my $result_dir = "$script_dir/../build-output/list-conserver/conserver"; my $log_dir = "$script_dir/../build-logs/list-conserver/conserver"; -my $skip_install = 0; my $build_timestamp; GetOptions( @@ -37,7 +35,6 @@ GetOptions( 'mock-uniqueext=s' => \$mock_uniqueext, 'result-dir=s' => \$result_dir, 'log-dir=s' => \$log_dir, - 'skip-install!' => \$skip_install, 'build-timestamp=i' => \$build_timestamp, ) or die usage(); @@ -112,19 +109,6 @@ for my $r (@rpms) { } print "built: " . join(', ', map { basename($_) } @rpms) . "\n"; -# ---- smoke test: install into the chroot and run the binaries --------------- -unless ($skip_install) { - print "== Smoke test (install + run) ==\n"; - run("mock -r " . sh_quote($mock_cfg) . $uniq . " --install " . sh_quote($main) - . " > " . sh_quote("$log_dir/install.log") . " 2>&1"); - # console (client) and conserver (daemon) both print their version to stderr/stdout. - my $smoke = capture("mock -r " . sh_quote($mock_cfg) . $uniq - . " --chroot -- " . sh_quote('/usr/bin/console -V 2>&1; /usr/sbin/conserver -V 2>&1') - . " 2>&1"); - print " output: $smoke\n"; - die "FATAL: smoke test did not report version $version\n" unless $smoke =~ /\Q$version\E/; - print "smoke test PASSED (console/conserver report $version)\n"; -} print "DONE: conserver-xcat $version for $mock_cfg -> $result_dir\n"; # ---- helpers ---------------------------------------------------------------- @@ -143,5 +127,5 @@ sub capture { sub sh_quote { my ($s) = @_; $s =~ s/'/'\\''/g; return "'$s'"; } sub usage { return "usage: mockbuild.pl --mock-cfg [--result-dir DIR] [--work-dir DIR]\n" - . " [--log-dir DIR] [--mock-uniqueext EXT] [--skip-install]\n"; + . " [--log-dir DIR] [--mock-uniqueext EXT]\n"; } diff --git a/elilo/mockbuild.pl b/elilo/mockbuild.pl index d6d7b5e..663145d 100755 --- a/elilo/mockbuild.pl +++ b/elilo/mockbuild.pl @@ -19,7 +19,6 @@ my $mock_cfg = ''; my $mock_uniqueext = ''; my $result_dir = "$repo_root/build-output/list3/elilo-xcat"; my $log_dir = "$repo_root/build-logs/list3/elilo-xcat"; -my $skip_install = 0; my $build_timestamp; GetOptions( @@ -29,7 +28,6 @@ GetOptions( 'mock-uniqueext=s' => \$mock_uniqueext, 'result-dir=s' => \$result_dir, 'log-dir=s' => \$log_dir, - 'skip-install!' => \$skip_install, 'build-timestamp=i' => \$build_timestamp, ) or die usage(); @@ -79,7 +77,6 @@ print "log_dir: $log_dir\n"; print "mock_cfg: $mock_cfg\n"; print "mock_uniqueext: " . ($mock_uniqueext ne '' ? $mock_uniqueext : '(none)') . "\n"; print "source_file:$source_file\n"; -print "skip_install: $skip_install\n"; make_path($result_dir); make_path($log_dir); @@ -207,37 +204,6 @@ for my $log (qw(build.log root.log state.log hw_info.log installed_pkgs.log)) { or die "Failed to copy $src to $log_dir: $!\n"; } -if (!$skip_install) { - print_step("Install RPM and run smoke tests"); - run("dnf -y install " . sh_quote($main_rpm)); - - my $efi_file = '/tftpboot/xcat/elilo-x64.efi'; - die "Missing installed EFI binary: $efi_file\n" if !-f $efi_file; - - my $file_log = "$log_dir/smoke-file.log"; - my $qf_log = "$log_dir/smoke-rpm-qf.log"; - my $rc_file = run_capture_rc("file $efi_file", $file_log); - my $rc_qf = run_capture_rc("rpm -qf $efi_file", $qf_log); - - die "Smoke check failed: file returned $rc_file\n" if $rc_file != 0; - die "Smoke check failed: rpm -qf returned $rc_qf\n" if $rc_qf != 0; - - my $file_out = slurp($file_log); - my $qf_out = slurp($qf_log); - - die "EFI file signature check failed:\n$file_out\n" - if $file_out !~ /(EFI application|PE32\+ executable)/i; - die "Installed file is not owned by elilo-xcat:\n$qf_out\n" - if $qf_out !~ /^elilo-xcat-/m; - - my $summary = "$log_dir/smoke-summary.txt"; - open my $sfh, '>', $summary or die "Cannot write $summary: $!\n"; - print {$sfh} "efi_file=$efi_file\n"; - print {$sfh} "rc_file=$rc_file\n"; - print {$sfh} "rc_qf=$rc_qf\n"; - close $sfh; -} - print_step("Completed"); print "Main RPM: $main_rpm\n"; print "Artifacts: $result_dir\n"; @@ -253,7 +219,6 @@ Usage: $0 [options] --mock-uniqueext TXT Optional mock --uniqueext suffix to isolate concurrent builds --result-dir PATH Output RPM/SRPM directory (default: $result_dir) --log-dir PATH Log directory (default: $log_dir) - --skip-install Skip dnf install + smoke tests --build-timestamp EPOCH Unix timestamp for SOURCE_DATE_EPOCH (deterministic builds) USAGE } diff --git a/goconserver/mockbuild.pl b/goconserver/mockbuild.pl index c6d3ec8..d9c9630 100755 --- a/goconserver/mockbuild.pl +++ b/goconserver/mockbuild.pl @@ -17,7 +17,6 @@ my $mock_cfg = ''; my $mock_uniqueext = ''; my $result_dir = "$repo_root/build-output/list5/goconserver"; my $log_dir = "$repo_root/build-logs/list5/goconserver"; -my $skip_install = 0; my $version = '0.3.3'; my $go_repo = 'https://github.com/xcat2/goconserver.git'; # Immutable pin: goconserver 0.3.3 is unreleased (newest tag v0.3.2) so it lives only on master. @@ -33,7 +32,6 @@ GetOptions( 'mock-uniqueext=s' => \$mock_uniqueext, 'result-dir=s' => \$result_dir, 'log-dir=s' => \$log_dir, - 'skip-install!' => \$skip_install, 'version=s' => \$version, 'go-repo=s' => \$go_repo, 'go-ref=s' => \$go_ref, @@ -95,7 +93,6 @@ print "arch: $arch\n"; print "version: $version\n"; print "go_ref: $go_ref\n"; print "release_suffix: " . ($release_suffix ne '' ? $release_suffix : '(none)') . "\n"; -print "skip_install: $skip_install\n"; make_path($result_dir); make_path($log_dir); @@ -288,19 +285,6 @@ for my $log (qw(build.log root.log state.log)) { system("mock -r " . sh_quote($build_cfg) . $mock_uniqueext_opt . " --scrub=chroot --scrub=bootstrap >" . sh_quote("$log_dir/mock-scrub.log") . " 2>&1"); -if (!$skip_install) { - print_step("Install and smoke test"); - my $main_rpm = $arch_rpms[0]; - run("dnf -y install " . sh_quote($main_rpm) . " >" . sh_quote("$log_dir/dnf-install.log") . " 2>&1"); - die "Missing /usr/bin/goconserver\n" if !-x '/usr/bin/goconserver'; - die "Missing /usr/bin/congo\n" if !-x '/usr/bin/congo'; - my $rc_help = run_rc("goconserver -h >" . sh_quote("$log_dir/smoke-help.log") . " 2>&1"); - die "goconserver -h failed (rc=$rc_help)\n" if $rc_help > 1; - my $rc_congo = run_rc("congo -h >" . sh_quote("$log_dir/smoke-congo.log") . " 2>&1"); - die "congo -h failed (rc=$rc_congo)\n" if $rc_congo > 1; - print "Smoke tests passed.\n"; -} - print_step("Completed"); print "Results in: $result_dir\n"; exit 0; @@ -321,7 +305,6 @@ Options: --mock-uniqueext STR Mock uniqueext (for concurrency isolation under mockbuild-all.pl) --result-dir PATH Output directory for RPMs --log-dir PATH Output directory for logs - --skip-install Skip dnf install + smoke tests --version VER Version string (default: 0.3.3) --go-repo URL Git repo URL (default: github.com/xcat2/goconserver) --go-ref REF Git ref/SHA to build (default: the pinned commit) @@ -352,13 +335,6 @@ sub run { } } -sub run_rc { - my ($cmd) = @_; - print "+ $cmd\n"; - my $rc = system($cmd); - return $rc == -1 ? 255 : ($rc >> 8); -} - sub capture { my ($cmd) = @_; my $out = `$cmd`; diff --git a/grub2-xcat/mockbuild.pl b/grub2-xcat/mockbuild.pl index a7848c9..a3d8ad9 100755 --- a/grub2-xcat/mockbuild.pl +++ b/grub2-xcat/mockbuild.pl @@ -20,7 +20,6 @@ my $mock_cfg = ''; my $mock_uniqueext = ''; my $result_dir = "$repo_root/build-output/list3/grub2-xcat"; my $log_dir = "$repo_root/build-logs/list3/grub2-xcat"; -my $skip_install = 0; my $build_timestamp; GetOptions( @@ -30,7 +29,6 @@ GetOptions( 'mock-uniqueext=s' => \$mock_uniqueext, 'result-dir=s' => \$result_dir, 'log-dir=s' => \$log_dir, - 'skip-install!' => \$skip_install, 'build-timestamp=i' => \$build_timestamp, ) or die usage(); @@ -88,7 +86,6 @@ print "result_dir: $result_dir\n"; print "log_dir: $log_dir\n"; print "mock_cfg: $mock_cfg\n"; print "mock_uniqueext: " . ($mock_uniqueext ne '' ? $mock_uniqueext : '(none)') . "\n"; -print "skip_install: $skip_install\n"; make_path($result_dir); make_path($log_dir); @@ -218,48 +215,6 @@ for my $log (qw(build.log root.log state.log hw_info.log installed_pkgs.log)) { or die "Failed to copy $src to $log_dir: $!\n"; } -if (!$skip_install) { - print_step("Install RPM and run smoke tests"); - run("dnf -y install " . sh_quote($main_rpm)); - - my $core = '/tftpboot/boot/grub2/powerpc-ieee1275/core.elf'; - my $grub2ppc = '/tftpboot/boot/grub2/grub2.ppc'; - die "Missing installed core image: $core\n" if !-f $core; - die "Missing installed post script output: $grub2ppc\n" if !-f $grub2ppc; - - my $file_core_log = "$log_dir/smoke-file-core.log"; - my $file_ppc_log = "$log_dir/smoke-file-grub2ppc.log"; - my $qf_log = "$log_dir/smoke-rpm-qf.log"; - - my $rc_file_core = run_capture_rc("file $core", $file_core_log); - my $rc_file_ppc = run_capture_rc("file $grub2ppc", $file_ppc_log); - my $rc_qf = run_capture_rc("rpm -qf $core", $qf_log); - my $rc_cmp = run_capture_rc("cmp -s $core $grub2ppc", "$log_dir/smoke-cmp.log"); - - die "Smoke check failed: file core returned $rc_file_core\n" if $rc_file_core != 0; - die "Smoke check failed: file grub2.ppc returned $rc_file_ppc\n" if $rc_file_ppc != 0; - die "Smoke check failed: rpm -qf returned $rc_qf\n" if $rc_qf != 0; - die "Smoke check failed: core.elf and grub2.ppc differ (cmp rc=$rc_cmp)\n" if $rc_cmp != 0; - - my $core_out = slurp($file_core_log); - my $qf_out = slurp($qf_log); - - die "Core image signature check failed:\n$core_out\n" - if $core_out !~ /ELF|data/i; - die "Installed core image is not owned by grub2-xcat:\n$qf_out\n" - if $qf_out !~ /^grub2-xcat-/m; - - my $summary = "$log_dir/smoke-summary.txt"; - open my $sfh, '>', $summary or die "Cannot write $summary: $!\n"; - print {$sfh} "core=$core\n"; - print {$sfh} "grub2ppc=$grub2ppc\n"; - print {$sfh} "rc_file_core=$rc_file_core\n"; - print {$sfh} "rc_file_ppc=$rc_file_ppc\n"; - print {$sfh} "rc_qf=$rc_qf\n"; - print {$sfh} "rc_cmp=$rc_cmp\n"; - close $sfh; -} - print_step("Completed"); print "Main RPM: $main_rpm\n"; print "Artifacts: $result_dir\n"; @@ -275,7 +230,6 @@ Usage: $0 [options] --mock-uniqueext TXT Optional mock --uniqueext suffix to isolate concurrent builds --result-dir PATH Output RPM/SRPM directory (default: $result_dir) --log-dir PATH Log directory (default: $log_dir) - --skip-install Skip dnf install + smoke tests --build-timestamp EPOCH Unix timestamp for deterministic builds (SOURCE_DATE_EPOCH) USAGE } diff --git a/ipmitool/mockbuild.pl b/ipmitool/mockbuild.pl index f02f2f2..ba599b2 100755 --- a/ipmitool/mockbuild.pl +++ b/ipmitool/mockbuild.pl @@ -19,7 +19,6 @@ my $mock_cfg = ''; my $mock_uniqueext = ''; my $result_dir = "$repo_root/build-output/list3/ipmitool-xcat"; my $log_dir = "$repo_root/build-logs/list3/ipmitool-xcat"; -my $skip_install = 0; my $build_timestamp; GetOptions( @@ -29,7 +28,6 @@ GetOptions( 'mock-uniqueext=s' => \$mock_uniqueext, 'result-dir=s' => \$result_dir, 'log-dir=s' => \$log_dir, - 'skip-install!' => \$skip_install, 'build-timestamp=i' => \$build_timestamp, ) or die usage(); @@ -79,7 +77,6 @@ print "log_dir: $log_dir\n"; print "mock_cfg: $mock_cfg\n"; print "mock_uniqueext: " . ($mock_uniqueext ne '' ? $mock_uniqueext : '(none)') . "\n"; print "source_file:$source_file\n"; -print "skip_install: $skip_install\n"; print "SOURCE_DATE_EPOCH: $SOURCE_DATE_EPOCH\n"; make_path($result_dir); @@ -206,60 +203,6 @@ for my $log (qw(build.log root.log state.log hw_info.log installed_pkgs.log)) { or die "Failed to copy $src to $log_dir: $!\n"; } -if (!$skip_install) { - print_step("Install RPM and run smoke tests"); - run("dnf -y install " . sh_quote($main_rpm)); - - my $bin = '/opt/xcat/bin/ipmitool-xcat'; - die "Missing installed binary: $bin\n" if !-x $bin; - - my $help_short_log = "$log_dir/smoke-help-short.log"; - my $help_long_log = "$log_dir/smoke-help-long.log"; - my $version_log = "$log_dir/smoke-version.log"; - my $open_log = "$log_dir/smoke-open-mc-info.log"; - my $ldd_log = "$log_dir/smoke-ldd.log"; - - my $rc_help_short = run_capture_rc("$bin -h", $help_short_log); - my $rc_help_long = run_capture_rc("$bin --help", $help_long_log); - my $rc_version = run_capture_rc("$bin -V", $version_log); - my $rc_open = run_capture_rc("$bin -I open mc info", $open_log); - my $rc_ldd = run_capture_rc("ldd $bin", $ldd_log); - - die "Smoke check failed: -h returned $rc_help_short\n" if $rc_help_short != 0; - die "Smoke check failed: -V returned $rc_version\n" if $rc_version != 0; - die "Smoke check failed: ldd returned $rc_ldd\n" if $rc_ldd != 0; - - my $help_short_out = slurp($help_short_log); - my $help_long_out = slurp($help_long_log); - my $version_out = slurp($version_log); - my $open_out = slurp($open_log); - my $ldd_out = slurp($ldd_log); - - die "Short help output does not contain usage text\n" - if $help_short_out !~ /usage:/i; - die "Long help output does not contain usage text\n" - if $help_long_out !~ /usage:/i; - die "Long help returned unexpected rc=$rc_help_long (expected 0 or 1)\n" - if $rc_help_long != 0 && $rc_help_long != 1; - die "Version output missing expected version string\n" - if $version_out !~ /ipmitool-xcat version \Q$version\E/i; - die "ldd output missing libcrypto dependency\n" - if $ldd_out !~ /libcrypto/; - if ($rc_open != 0 && $open_out !~ m{Could not open device|/dev/ipmi}) { - die "IPMI probe failed with unexpected output:\n$open_out\n"; - } - - my $summary = "$log_dir/smoke-summary.txt"; - open my $sfh, '>', $summary or die "Cannot write $summary: $!\n"; - print {$sfh} "binary=$bin\n"; - print {$sfh} "rc_help_short=$rc_help_short\n"; - print {$sfh} "rc_help_long=$rc_help_long\n"; - print {$sfh} "rc_version=$rc_version\n"; - print {$sfh} "rc_open=$rc_open\n"; - print {$sfh} "rc_ldd=$rc_ldd\n"; - close $sfh; -} - print_step("Completed"); print "Main RPM: $main_rpm\n"; print "Artifacts: $result_dir\n"; @@ -275,7 +218,6 @@ Usage: $0 [options] --mock-uniqueext TXT Optional mock --uniqueext suffix to isolate concurrent builds --result-dir PATH Output RPM/SRPM directory (default: $result_dir) --log-dir PATH Log directory (default: $log_dir) - --skip-install Skip dnf install + smoke tests --build-timestamp N Unix epoch for SOURCE_DATE_EPOCH (deterministic builds) USAGE } diff --git a/mockbuild-all.pl b/mockbuild-all.pl index 5956a98..0d3fd52 100755 --- a/mockbuild-all.pl +++ b/mockbuild-all.pl @@ -16,7 +16,8 @@ use lib $RealBin; use MockBuildUtils qw(sh_quote print_step version_matches required_pkgs read_manifest verify_repo_packages verify_repo_signature rpm_version rpm_release rpm_sigmd5 restamp_release_line - cross_copy_genesis finalize_xcat_dep bump_dep_release_suffix); + cross_copy_genesis finalize_xcat_dep bump_dep_release_suffix + build_mock_uniqueext); # --- Mount-namespace isolation: guard the host cgroup against mock teardown propagation ---------- # mock mounts /sys/fs/cgroup into every build chroot. On these systemd build hosts every mount is @@ -80,7 +81,6 @@ my $build_number; # newest tag is v0.3.2), so it exists only on master -- pin an immutable SHA instead of the moving # branch so the build is reproducible. Bump this deliberately when uptaking a new goconserver. my $GOCONSERVER_REF = '6166fe5ec1c5b3c20475e322a9f0e8e93c87e45f'; -my $skip_install = 0; my $skip_build = 0; my $skip_xcat_dep = 0; my $skip_perl = 0; @@ -134,7 +134,6 @@ GetOptions( 'run-id=s' => \$run_id, 'build-timestamp=i' => \$build_timestamp, 'build-number=i' => \$build_number, - 'skip-install!' => \$skip_install, 'skip-build!' => \$skip_build, 'skip-xcat-dep!' => \$skip_xcat_dep, 'skip-perl!' => \$skip_perl, @@ -443,7 +442,6 @@ print "skip_build: $skip_build\n"; print "skip_xcat_dep: $skip_xcat_dep\n"; print "skip_perl: $skip_perl\n"; print "skip_genesis: $skip_genesis\n"; -print "skip_install: $skip_install\n"; print "skip_createrepo: $skip_createrepo\n"; print "skip_tarball: $skip_tarball\n"; print "scrub_all_chroots:$scrub_all_chroots\n"; @@ -485,7 +483,6 @@ if (!$skip_build) { # host-local, run-scoped work dir so /tmp doesn't collide between runs '--work-dir', sh_quote("/tmp/mockbuild-all-$run_id/$name"), '--build-timestamp', $SOURCE_DATE_EPOCH, - ($skip_install ? '--skip-install' : ()), # goconserver generates its spec at build time (from an upstream clone), so the # in-tree spec Release bump above cannot reach it. Hand the CD suffix down so its # NVR advances per run too, and pin the clone to an immutable commit (not the moving @@ -529,7 +526,6 @@ if (!$skip_build) { # packages; the srpm-mode ones (HTML-Form, IO-Stty, Net-Telnet) build from a # committed .src.rpm, so hand the suffix down for the builder to re-stamp them. ($RELEASE_BUMP ne '' ? ('--release-suffix', sh_quote($RELEASE_BUMP)) : ()), - ($skip_install ? '--skip-install' : ()), ($keep_buildroots ? '--keep-buildroots' : ()), ); push @build_steps, { @@ -972,7 +968,6 @@ Options: 0/auto = host nproc (default: auto) --run-id ID Run identifier suffix (default: derived from build timestamp) --build-timestamp EPOCH Unix epoch for deterministic builds (default: Gitepoch or git log) - --skip-install Skip install/smoke tests in child builder scripts --skip-build Skip all build steps and only collect/create repo/tarballs --skip-xcat-dep Skip xcat-dep mockbuild.pl package steps --skip-perl Skip perl package build step @@ -1408,27 +1403,6 @@ sub resolve_mock_cfg { . "(tried /etc/mock/${os_id}+epel-${rel}-${arch}.cfg and /etc/mock/${short}+epel-${rel}-${arch}.cfg)\n"; } -sub build_mock_uniqueext { - my ($run, $seq, $label) = @_; - - my $run_part = defined($run) ? $run : 'run'; - $run_part =~ s/[^A-Za-z0-9_.-]+/-/g; - $run_part =~ s/^-+|-+$//g; - $run_part = 'run' if $run_part eq ''; - $run_part = substr($run_part, -24) if length($run_part) > 24; - - my $label_part = defined($label) ? $label : 'step'; - $label_part =~ s/[^A-Za-z0-9_.-]+/-/g; - $label_part =~ s/^-+|-+$//g; - $label_part = 'step' if $label_part eq ''; - $label_part = substr($label_part, 0, 20) if length($label_part) > 20; - - my $idx = defined($seq) ? int($seq) : 0; - $idx = 0 if $idx < 0; - - return sprintf("mba-%02d-%s-%s", $idx, $run_part, $label_part); -} - sub resolve_xcat_source { my ($requested, $root) = @_; # Prefer the sibling ../xcat-core (the real layout: source/xcat-core beside source/xcat-dep) diff --git a/mockbuild-perl-packages.pl b/mockbuild-perl-packages.pl index 531ca81..3822b3f 100755 --- a/mockbuild-perl-packages.pl +++ b/mockbuild-perl-packages.pl @@ -18,8 +18,6 @@ my $result_dir = ''; my $log_dir = ''; my $packages_csv = ''; my $jobs = 0; -my $skip_install = 0; -my $allow_erasing = 0; my $build_timestamp; # CD version bump: appended to the Release of the srpm-mode packages (HTML-Form, IO-Stty, # Net-Telnet), which build from a committed .src.rpm and so are NOT covered by mockbuild-all's @@ -35,8 +33,6 @@ GetOptions( 'log-dir=s' => \$log_dir, 'packages=s' => \$packages_csv, 'jobs=i' => \$jobs, - 'skip-install!' => \$skip_install, - 'allow-erasing!' => \$allow_erasing, 'build-timestamp=i' => \$build_timestamp, 'release-suffix=s' => \$release_suffix, ) or die usage(); @@ -170,10 +166,6 @@ $jobs = 1 if $jobs < 1; if (@packages && $jobs > scalar(@packages)) { $jobs = scalar(@packages); } -if (!$skip_install && $jobs > 1) { - print "INFO: --skip-install is disabled; forcing --jobs 1 to avoid host dnf lock contention\n"; - $jobs = 1; -} make_path($result_dir); make_path($log_dir); @@ -189,8 +181,6 @@ print "mock_cfg: $mock_cfg\n"; print "mock_uniqueext: " . ($mock_uniqueext ne '' ? $mock_uniqueext : '(none)') . "\n"; print "packages: " . join(', ', @packages) . "\n"; print "jobs: $jobs\n"; -print "skip_install:$skip_install\n"; -print "allow_erasing:$allow_erasing\n"; print "release_suffix:" . ($release_suffix ne '' ? $release_suffix : '(none)') . "\n"; print_step("Mock config check"); @@ -230,8 +220,6 @@ for my $idx (0 .. $#packages) { mock_cfg => $mock_cfg, mock_uniqueext => $pkg_uniqueext, arch => $arch, - skip_install => $skip_install, - allow_erasing => $allow_erasing, release_suffix => $release_suffix, ); unless ($keep_buildroots) { @@ -328,8 +316,6 @@ sub build_package { my $mock_cfg = $args{mock_cfg}; my $mock_uniqueext = $args{mock_uniqueext}; my $arch = $args{arch}; - my $skip_install = $args{skip_install}; - my $allow_erasing = $args{allow_erasing}; my $release_suffix = $args{release_suffix}; my $pkg_run_dir = "$work_dir/$pkg"; @@ -506,15 +492,6 @@ sub build_package { } } - if (!$skip_install) { - my $install_cmd = "dnf -y install "; - $install_cmd .= "--allowerasing " if $allow_erasing; - run($install_cmd . sh_quote($main_rpm)); - my $module = $cfg->{module}; - my $rc_mod = run_capture_rc("perl -M$module -e 1", "$pkg_log/smoke-perl-module.log"); - die "Perl module import failed for $pkg ($module), rc=$rc_mod\n" if $rc_mod != 0; - } - $summary = "$pkg PASS main_rpm=" . basename($main_rpm); $ok = 1; }; @@ -561,8 +538,6 @@ Usage: $0 [options] --build-timestamp EPOCH Unix epoch for SOURCE_DATE_EPOCH (deterministic builds) --release-suffix STR CD bump appended to the Release of the srpm-mode packages that build from a committed .src.rpm (HTML-Form, IO-Stty, Net-Telnet) - --skip-install Skip dnf install + perl module import checks - --allow-erasing Allow dnf to erase conflicting packages during install smoke tests USAGE } diff --git a/syslinux/mockbuild.pl b/syslinux/mockbuild.pl index ca82bd8..dda65f3 100755 --- a/syslinux/mockbuild.pl +++ b/syslinux/mockbuild.pl @@ -19,7 +19,6 @@ my $mock_cfg = ''; my $mock_uniqueext = ''; my $result_dir = "$repo_root/build-output/list3/syslinux-xcat"; my $log_dir = "$repo_root/build-logs/list3/syslinux-xcat"; -my $skip_install = 0; my $build_timestamp; GetOptions( @@ -29,7 +28,6 @@ GetOptions( 'mock-uniqueext=s' => \$mock_uniqueext, 'result-dir=s' => \$result_dir, 'log-dir=s' => \$log_dir, - 'skip-install!' => \$skip_install, 'build-timestamp=i' => \$build_timestamp, ) or die usage(); @@ -87,7 +85,6 @@ print "version: $version\n"; print "mock_cfg: $mock_cfg\n"; print "mock_uniqueext: " . ($mock_uniqueext ne '' ? $mock_uniqueext : '(none)') . "\n"; print "source_file:$source_file\n"; -print "skip_install: $skip_install\n"; make_path($result_dir); make_path($log_dir); @@ -226,48 +223,6 @@ for my $log (qw(build.log root.log state.log hw_info.log installed_pkgs.log)) { or die "Failed to copy $src to $log_dir: $!\n"; } -if (!$skip_install) { - print_step("Install RPM(s) and run smoke tests"); - run("dnf -y install " . sh_quote($xcat_rpm)); - - my $pxe_file = '/opt/xcat/share/xcat/netboot/syslinux/pxelinux.0'; - die "Missing installed PXE file: $pxe_file\n" if !-f $pxe_file; - - my $file_log = "$log_dir/smoke-file.log"; - my $qf_log = "$log_dir/smoke-rpm-qf.log"; - my $rc_file = run_capture_rc("file $pxe_file", $file_log); - my $rc_qf = run_capture_rc("rpm -qf $pxe_file", $qf_log); - - die "Smoke check failed: file returned $rc_file\n" if $rc_file != 0; - die "Smoke check failed: rpm -qf returned $rc_qf\n" if $rc_qf != 0; - - my $qf_out = slurp($qf_log); - die "Installed file is not owned by syslinux-xcat:\n$qf_out\n" - if $qf_out !~ /^syslinux-xcat-/m; - - # EL10 hosts may already carry syslinux-nonlinux, which conflicts with - # directly installing the rebuilt syslinux payload. The xcat subpackage - # is the artifact we need to smoke-test on the host; repository-level - # dependency validation happens later in the full install phase. - my $syslinux_help_log = "$log_dir/smoke-syslinux-help.log"; - if (-x '/usr/bin/syslinux') { - my $rc_help = run_capture_rc("/usr/bin/syslinux --help", $syslinux_help_log); - my $help_out = slurp($syslinux_help_log); - die "syslinux --help returned unexpected rc=$rc_help\n" - if $rc_help != 0 && $rc_help != 1; - die "syslinux --help output missing expected usage text\n" - if $help_out !~ /usage|syslinux/i; - } - - my $summary = "$log_dir/smoke-summary.txt"; - open my $sfh, '>', $summary or die "Cannot write $summary: $!\n"; - print {$sfh} "pxe_file=$pxe_file\n"; - print {$sfh} "rc_file=$rc_file\n"; - print {$sfh} "rc_qf=$rc_qf\n"; - print {$sfh} "main_rpm_available=" . ($main_rpm ? 1 : 0) . "\n"; - close $sfh; -} - print_step("Completed"); print "syslinux-xcat RPM: $xcat_rpm\n"; print "Artifacts: $result_dir\n"; @@ -284,7 +239,6 @@ Usage: $0 [options] --result-dir PATH Output RPM/SRPM directory (default: $result_dir) --log-dir PATH Log directory (default: $log_dir) --build-timestamp EPOCH SOURCE_DATE_EPOCH for deterministic builds - --skip-install Skip dnf install + smoke tests USAGE } diff --git a/t/mockbuild-all.t b/t/mockbuild-all.t index 76076c9..0d3fc83 100644 --- a/t/mockbuild-all.t +++ b/t/mockbuild-all.t @@ -12,7 +12,8 @@ use File::Path qw(make_path); use File::Basename qw(basename); use MockBuildUtils qw(required_pkgs version_matches rpm_sigmd5 rpm_version rpm_release rpm_is_signed restamp_release_line cross_copy_genesis finalize_xcat_dep read_manifest - verify_repo_packages verify_repo_signature bump_dep_release_suffix); + verify_repo_packages verify_repo_signature bump_dep_release_suffix + build_mock_uniqueext); # Run a printing sub with STDOUT muted so its progress lines do not pollute TAP. sub quiet(&) { @@ -325,4 +326,33 @@ is(rpm_release(tempdir(CLEANUP => 1), 'nonexistent-pkg'), undef, 'rpm_release is 'verify_repo_signature: mismatch reported as WRONGKEY : signed by , expected '); } +# ---- build_mock_uniqueext: distinct per target so concurrent mock roots never collide --------- +# (PR #62 review) A long (timestamp) run id must not tail-truncate away the leading EL/arch token: +# for the 7-char "ppc64le" arch that dropped the EL digit, so alma+epel-{8,9,10}-ppc64le collapsed to +# one uniqueext -- and goconserver builds all three ELs in the SAME el10 chroot, so the roots raced. +{ + my $seq = 6; my $label = 'goconserver'; + # The reproducing case: the default timestamp run id (long), folded with the per-target prefix. + my @ppc = map { build_mock_uniqueext("alma+epel-$_-ppc64le-20260821-210716", $seq, $label) } (8, 9, 10); + my %seen; $seen{$_}++ for @ppc; + is(scalar(keys %seen), 3, + 'build_mock_uniqueext: el8/el9/el10 ppc64le get DISTINCT uniqueext on a long run id (no collision)'); + like($ppc[0], qr/^mba-06-alma-epel-8-/, 'uniqueext keeps a readable leading EL/arch token'); + + # x86_64 (6-char arch) was never broken -- assert it stays distinct too. + my @x86 = map { build_mock_uniqueext("alma+epel-$_-x86_64-20260821-210716", $seq, $label) } (8, 9, 10); + my %sx; $sx{$_}++ for @x86; + is(scalar(keys %sx), 3, 'build_mock_uniqueext: el8/el9/el10 x86_64 also distinct'); + + # Short run ids (e.g. the CD "$BUILD_NUMBER") are unchanged and already distinct per target. + isnt(build_mock_uniqueext('alma+epel-8-ppc64le-104', $seq, $label), + build_mock_uniqueext('alma+epel-9-ppc64le-104', $seq, $label), + 'build_mock_uniqueext: short (build-number) run ids distinct per target'); + + # Same run id + same step -> stable (deterministic; a re-run reuses/scrubs the same root). + is(build_mock_uniqueext('alma+epel-8-ppc64le-20260821-210716', $seq, $label), + build_mock_uniqueext('alma+epel-8-ppc64le-20260821-210716', $seq, $label), + 'build_mock_uniqueext: deterministic for a given (run, seq, label)'); +} + done_testing; diff --git a/xnba/mockbuild.pl b/xnba/mockbuild.pl index 08c6084..7c1395a 100755 --- a/xnba/mockbuild.pl +++ b/xnba/mockbuild.pl @@ -19,7 +19,6 @@ my $mock_cfg = ''; my $mock_uniqueext = ''; my $result_dir = "$repo_root/build-output/list3/xnba-undi"; my $log_dir = "$repo_root/build-logs/list3/xnba-undi"; -my $skip_install = 0; my $build_timestamp; GetOptions( @@ -28,7 +27,6 @@ GetOptions( 'mock-uniqueext=s' => \$mock_uniqueext, 'result-dir=s' => \$result_dir, 'log-dir=s' => \$log_dir, - 'skip-install!' => \$skip_install, 'build-timestamp=i' => \$build_timestamp, ) or die usage(); @@ -73,7 +71,6 @@ print "work_dir: $work_dir\n"; print "result_dir: $result_dir\n"; print "log_dir: $log_dir\n"; print "mock_cfg: $mock_cfg\n"; -print "skip_install: $skip_install\n"; make_path($result_dir); make_path($log_dir); @@ -176,7 +173,6 @@ Options: --mock-uniqueext STR Mock uniqueext value --result-dir PATH Output directory for RPMs --log-dir PATH Output directory for logs - --skip-install Skip install verification --build-timestamp EPOCH Unix timestamp for reproducible builds USAGE } From cbb9f4ebfac6b965a7fc6a95c93c74536e8e6836 Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Fri, 21 Aug 2026 18:50:45 -0300 Subject: [PATCH 46/55] fix(xcat-dep): finalize cross-arch genesis in both directions PR #62 review #2: finalize_xcat_dep() discovered OS dirs by grepping for an x86_64 subdir under the x86_64 repo, so a ppc64le-only (an rh that built for ppc but has no x86_64 sibling) was never iterated -- finalize never cross-populated that cell's x86_64 genesis and still exited 0. Discover the union of dirs from both arch repos and require both arch peers for every one, so an x86_64-only AND a ppc64le-only cell both die with a named error instead of passing unnoticed. Add a symmetric fixture for the ppc64le-only case (mirrors the existing x86_64-only peer test). Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- MockBuildUtils.pm | 20 ++++++++++++++------ t/mockbuild-all.t | 8 ++++++++ 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/MockBuildUtils.pm b/MockBuildUtils.pm index 45879d7..19f2ffd 100644 --- a/MockBuildUtils.pm +++ b/MockBuildUtils.pm @@ -287,15 +287,23 @@ sub finalize_xcat_dep { print_step('Finalize xcat-dep: cross-arch genesis-base provisioning (issue #7610)'); print "x86_64-repo: $x86_64_repo\n"; print "ppc64le-repo: $ppc64le_repo\n"; - my @osdirs = grep { -d "$_/x86_64" } glob("$x86_64_repo/*"); + # Discover the UNION of OS dirs from BOTH arch repos. Anchoring discovery on x86_64 alone let a + # ppc64le-only (an rh that built for ppc but not x86_64) slip through unseen -- finalize + # then never cross-populated that cell's x86_64 genesis and still exited 0 (PR #62 review). Both + # arch peers are required for every discovered below, so the check is now symmetric. + my %os; + $os{ basename($_) } = 1 for grep { -d "$_/x86_64" } glob("$x86_64_repo/*"); + $os{ basename($_) } = 1 for grep { -d "$_/ppc64le" } glob("$ppc64le_repo/*"); my $pairs = 0; - for my $p (sort @osdirs) { - my $osdir = basename($p); + for my $osdir (sort keys %os) { my $x86dir = "$x86_64_repo/$osdir/x86_64"; my $ppcdir = "$ppc64le_repo/$osdir/ppc64le"; - # Require the peer repo itself: in the CD both arches build every EL, so a missing - # ppc64le peer for an x86_64 OS means an incomplete input, not something to skip past - # (skipping would leave that OS's x86_64 repo without the ppc64 genesis and still exit 0). + # Both arch peers must exist: in the CD both arches build every EL, so a one-arch is an + # incomplete input, not something to skip past (skipping would leave a cell without the + # foreign-arch genesis and still exit 0). Symmetric -- catches an x86_64-only AND a + # ppc64le-only . + die "FATAL: [finalize] $osdir: no x86_64 peer repo at $x86dir\n" + . " (both arches must build every EL before finalize)\n" if !-d $x86dir; die "FATAL: [finalize] $osdir: no ppc64le peer repo at $ppcdir\n" . " (both arches must build every EL before finalize)\n" if !-d $ppcdir; # Require the expected inputs: each arch's build must have produced its OWN genesis rpm diff --git a/t/mockbuild-all.t b/t/mockbuild-all.t index 0d3fc83..d2201f3 100644 --- a/t/mockbuild-all.t +++ b/t/mockbuild-all.t @@ -176,6 +176,14 @@ SPEC my $ok3 = eval { quiet { finalize_xcat_dep("$tmp3/x", "$tmp3/p") }; 1 }; ok(!$ok3, 'finalize dies when an x86_64 OS has no ppc64le peer repo (no silent skip)'); like($@, qr/no ppc64le peer repo/, 'finalize error names the missing peer'); + + # Symmetric (PR #62 review #2): a ppc64le-ONLY (no x86_64 sibling) must ALSO be caught -- + # the old x86_64-anchored discovery skipped it entirely and exited 0. + my $tmp4 = tempdir(CLEANUP => 1); + make_path("$tmp4/p/rh9/ppc64le"); # ppc64le OS present, but NO x86_64 peer dir at all + my $ok4 = eval { quiet { finalize_xcat_dep("$tmp4/x", "$tmp4/p") }; 1 }; + ok(!$ok4, 'finalize dies when a ppc64le OS has no x86_64 peer repo (was silently skipped)'); + like($@, qr/no x86_64 peer repo/, 'finalize error names the missing x86_64 peer'); } # ---- restamp_release_line: CD --build-number Release stamping (PR #62 review point 1) ---------- From ef966fd95b2e33270547b2340d1bd8a9a6897fd0 Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Fri, 21 Aug 2026 19:01:01 -0300 Subject: [PATCH 47/55] feat(xcat-dep): arch-array cross-arch finalize; verify every rpm signature Two review follow-ups: - Cross-arch genesis matrix is now driven by a single @GENESIS_ARCHES array (arch -> xCAT tarch) in MockBuildUtils. finalize_xcat_dep discovers, gates, and N-way cross-copies over that list, so adding an arch later (aarch64, riscv64) is one entry there plus wiring its repo root. No behaviour change for the current x86_64/ppc64le pair (tests unchanged + a contract assertion on the array). - PR #62 review #4: the repo completeness gate verified only the repomd.xml signature. It now ALSO verifies every binary rpm's header signature is by the configured signing key -- resolve the key's accepted id set (primary + subkey ids) and check each rpm's RSAHEADER/DSAHEADER pgpsig key id, failing on any unsigned or foreign-signed rpm. Runs per-target in deploy_target and per-cell after --finalize-xcat-dep. A signed repomd over an unsigned rpm otherwise passed the gate yet DNF rejects the package at install. Added pure fixtures. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- MockBuildUtils.pm | 117 +++++++++++++++++++++++++++++++--------------- mockbuild-all.pl | 49 ++++++++++++++++++- t/mockbuild-all.t | 27 ++++++++++- 3 files changed, 152 insertions(+), 41 deletions(-) diff --git a/MockBuildUtils.pm b/MockBuildUtils.pm index 19f2ffd..7755450 100644 --- a/MockBuildUtils.pm +++ b/MockBuildUtils.pm @@ -15,7 +15,7 @@ use Digest::MD5 qw(md5_hex); our @EXPORT_OK = qw( sh_quote print_step version_matches required_pkgs have_rpm read_manifest - verify_repo_packages verify_repo_signature + verify_repo_packages verify_repo_signature verify_rpm_signatures rpm_version rpm_release rpm_sigmd5 rpm_is_signed restamp_release_line cross_copy_genesis finalize_xcat_dep bump_dep_release_suffix build_mock_uniqueext @@ -110,6 +110,26 @@ sub verify_repo_signature { return @problems; } +# verify_rpm_signatures: pure decision for the per-rpm signature gate. $rpm_sigs is an arrayref of +# [rpm_basename, observed_keyid|undef] (rpm reports the signing SUBKEY id); $accept is a hashref set +# of acceptable key ids (the signing key's primary + subkey ids, lowercased). Returns one problem per +# rpm that is unsigned or signed by a key not in the set. A signed repomd over unsigned/foreign-signed +# rpms still makes DNF reject the install, so the packages must be checked, not just the metadata. +sub verify_rpm_signatures { + my ($rpm_sigs, $accept) = @_; + my @problems; + for my $rs (@$rpm_sigs) { + my ($name, $kid) = @$rs; + if (!defined($kid) || $kid eq '') { + push @problems, "UNSIGNED rpm $name"; + } elsif (!$accept->{ lc $kid }) { + push @problems, "WRONGKEY rpm $name: signed by $kid, expected one of " + . join('/', sort keys %$accept); + } + } + return @problems; +} + # have_rpm: is there a non-src rpm named -... under $dir? sub have_rpm { my ($dir, $name) = @_; @@ -280,6 +300,17 @@ sub cross_copy_genesis { # a repo whose rpm set changed (or undef). Both injected so this stays free of gpg/createrepo # state and is unit-testable. Requires each arch's own genesis rpm to be present (a pair with no # genesis is a hard error, never a silent no-op) and fails if no repo pair is found at all. +# Architectures whose xCAT-genesis-base is cross-provisioned into every peer repo, so a management +# node can netboot nodes of any arch (issue #7610). Each entry maps the repo/subdir arch name to the +# genesis rpm's xCAT "tarch" (xCAT collapses ppc/ppc64le into tarch ppc64; x86_64 stays x86_64). This +# is the SINGLE SOURCE OF TRUTH for the cross-arch matrix -- to add an arch later (e.g. aarch64, +# riscv64) add an entry here AND wire its repo root into finalize_xcat_dep's %repo (the caller passes +# it). Discovery, the per-arch input gate, and the N-way cross-copy all iterate this list. +our @GENESIS_ARCHES = ( + { arch => 'x86_64', tarch => 'x86_64' }, + { arch => 'ppc64le', tarch => 'ppc64' }, +); + sub finalize_xcat_dep { my ($x86_64_repo, $ppc64le_repo, %opt) = @_; my $sign = $opt{sign}; @@ -287,47 +318,59 @@ sub finalize_xcat_dep { print_step('Finalize xcat-dep: cross-arch genesis-base provisioning (issue #7610)'); print "x86_64-repo: $x86_64_repo\n"; print "ppc64le-repo: $ppc64le_repo\n"; - # Discover the UNION of OS dirs from BOTH arch repos. Anchoring discovery on x86_64 alone let a - # ppc64le-only (an rh that built for ppc but not x86_64) slip through unseen -- finalize - # then never cross-populated that cell's x86_64 genesis and still exited 0 (PR #62 review). Both - # arch peers are required for every discovered below, so the check is now symmetric. + + # Per-arch repo root, keyed by the @GENESIS_ARCHES arch name. A new arch added to that list must + # also get its root wired here (today both roots are the same CD tree); a missing one fails loudly + # below rather than silently skipping. + my %repo = ( x86_64 => $x86_64_repo, ppc64le => $ppc64le_repo ); + + # Discover the UNION of OS dirs across ALL arch repos. Anchoring discovery on one arch let an + # built for only the OTHER arch slip through unseen -- finalize then never cross-populated + # that cell and still exited 0 (PR #62 review). Every discovered must carry every arch below. my %os; - $os{ basename($_) } = 1 for grep { -d "$_/x86_64" } glob("$x86_64_repo/*"); - $os{ basename($_) } = 1 for grep { -d "$_/ppc64le" } glob("$ppc64le_repo/*"); + for my $a (@GENESIS_ARCHES) { + my $root = $repo{ $a->{arch} } + // die "FATAL: [finalize] no repo root configured for arch '$a->{arch}' (wire it in %repo)\n"; + $os{ basename($_) } = 1 for grep { -d "$_/$a->{arch}" } glob("$root/*"); + } + my $pairs = 0; for my $osdir (sort keys %os) { - my $x86dir = "$x86_64_repo/$osdir/x86_64"; - my $ppcdir = "$ppc64le_repo/$osdir/ppc64le"; - # Both arch peers must exist: in the CD both arches build every EL, so a one-arch is an - # incomplete input, not something to skip past (skipping would leave a cell without the - # foreign-arch genesis and still exit 0). Symmetric -- catches an x86_64-only AND a - # ppc64le-only . - die "FATAL: [finalize] $osdir: no x86_64 peer repo at $x86dir\n" - . " (both arches must build every EL before finalize)\n" if !-d $x86dir; - die "FATAL: [finalize] $osdir: no ppc64le peer repo at $ppcdir\n" - . " (both arches must build every EL before finalize)\n" if !-d $ppcdir; - # Require the expected inputs: each arch's build must have produced its OWN genesis rpm - # before finalize cross-populates them. Without this, a pair whose builds produced no - # genesis rpms would make finalize a silent no-op that still exits 0 (the bug this guards). - die "FATAL: [finalize] $osdir: no x86_64 xCAT-genesis-base rpm in $x86dir\n" - if !grep { !/\.src\.rpm$/ } glob("$x86dir/xCAT-genesis-base-x86_64-*.rpm"); - die "FATAL: [finalize] $osdir: no ppc64 xCAT-genesis-base rpm in $ppcdir\n" - if !grep { !/\.src\.rpm$/ } glob("$ppcdir/xCAT-genesis-base-ppc64-*.rpm"); - # xCAT collapses ppc/ppc64/ppc64le into tarch=ppc64, so the ppc genesis rpm is - # named xCAT-genesis-base-ppc64-*. Cross-copy both directions. - my $to_x86 = cross_copy_genesis($ppcdir, $x86dir, 'ppc64', $sign); - my $to_ppc = cross_copy_genesis($x86dir, $ppcdir, 'x86_64', $sign); - # Re-index+sign BOTH repos of the pair every finalize, not only when an rpm was copied this - # run. A crash after a prior run's copy+sign but before its createrepo leaves the genesis rpm - # on disk (so cross_copy_genesis now returns 0) yet ABSENT from repomd.xml -- which no - # signature gate catches. Re-indexing is cheap (these are tiny repos) and idempotent, and it - # heals that partial state; skipped only when no signer/indexer was injected. - if ($reindex) { $reindex->($x86dir); $reindex->($ppcdir); } - printf "[finalize] %s: %d ppc64 genesis -> x86_64, %d x86_64 genesis -> ppc64le\n", - $osdir, $to_x86, $to_ppc; + my %adir = map { $_->{arch} => "$repo{$_->{arch}}/$osdir/$_->{arch}" } @GENESIS_ARCHES; + # Pass 1 -- every arch peer repo dir must exist: a one-arch is an incomplete input, not + # something to skip past (skipping would leave a cell without a foreign-arch genesis and still + # exit 0). Checked before the rpm pass so a missing peer is reported as such. Symmetric across + # all arches (catches an x86_64-only AND a ppc64le-only ). + for my $a (@GENESIS_ARCHES) { + die "FATAL: [finalize] $osdir: no $a->{arch} peer repo at $adir{$a->{arch}}\n" + . " (every arch must build every EL before finalize)\n" if !-d $adir{ $a->{arch} }; + } + # Pass 2 -- every arch must have produced its OWN genesis rpm before finalize cross-populates + # them; otherwise a pair with no genesis rpms would make finalize a silent no-op that still + # exits 0. xCAT collapses ppc/ppc64le into tarch=ppc64, so match on each arch's tarch. + for my $a (@GENESIS_ARCHES) { + die "FATAL: [finalize] $osdir: no $a->{arch} xCAT-genesis-base rpm (tarch $a->{tarch}) in $adir{$a->{arch}}\n" + if !grep { !/\.src\.rpm$/ } glob("$adir{$a->{arch}}/xCAT-genesis-base-$a->{tarch}-*.rpm"); + } + # N-way cross-copy: put each arch's genesis into EVERY other arch's repo dir. + my @summary; + for my $src (@GENESIS_ARCHES) { + for my $dst (@GENESIS_ARCHES) { + next if $src->{arch} eq $dst->{arch}; + my $n = cross_copy_genesis($adir{$src->{arch}}, $adir{$dst->{arch}}, $src->{tarch}, $sign); + push @summary, "$n $src->{tarch} -> $dst->{arch}"; + } + } + # Re-index+sign EVERY arch repo of this each finalize, not only when an rpm was copied + # this run: a crash after a prior run's copy+sign but before its createrepo leaves the genesis + # rpm on disk (so cross_copy_genesis now returns 0) yet ABSENT from repomd.xml -- which no + # signature gate catches. Re-indexing is cheap (tiny repos) and idempotent, and heals that + # partial state; skipped only when no signer/indexer was injected. + if ($reindex) { $reindex->($adir{$_->{arch}}) for @GENESIS_ARCHES; } + print "[finalize] $osdir: " . join(', ', @summary) . "\n"; $pairs++; } - die "FATAL: --finalize-xcat-dep found no /x86_64 + /ppc64le repo pair under\n" + die "FATAL: --finalize-xcat-dep found no repo dir under any arch root\n" . " --x86_64-repo '$x86_64_repo'\n --ppc64le-repo '$ppc64le_repo'\n" if $pairs == 0; print_step('Finalize complete'); } diff --git a/mockbuild-all.pl b/mockbuild-all.pl index 0d3fd52..b091812 100755 --- a/mockbuild-all.pl +++ b/mockbuild-all.pl @@ -14,7 +14,7 @@ use POSIX qw(strftime); use FindBin qw($RealBin); use lib $RealBin; use MockBuildUtils qw(sh_quote print_step version_matches required_pkgs - read_manifest verify_repo_packages verify_repo_signature + read_manifest verify_repo_packages verify_repo_signature verify_rpm_signatures rpm_version rpm_release rpm_sigmd5 restamp_release_line cross_copy_genesis finalize_xcat_dep bump_dep_release_suffix build_mock_uniqueext); @@ -1196,6 +1196,37 @@ sub gpg_key_fingerprint { return $fprs[0]; } +# gpg_key_ids: all acceptable key ids (lowercased) for a signing key NAME -- the primary key id AND +# every subkey id, in both 16-hex (long) and 8-hex (short) forms. rpm header signatures report the +# signing SUBKEY id, so the per-rpm gate accepts any id belonging to the key rather than one exact +# fingerprint. Returns a hashref set (empty if the key can't be listed). +sub gpg_key_ids { + my ($keyname, $home) = @_; + my $h = ($home ne '') ? ' --homedir ' . sh_quote($home) : ''; + my $out = `gpg$h --with-colons --list-keys ${\ sh_quote($keyname)} 2>/dev/null` // ''; + my %ids; + for my $line (split /\n/, $out) { + my @f = split /:/, $line; + next unless ($f[0] // '') =~ /^(?:pub|sub)$/ && defined $f[4] && $f[4] ne ''; + my $id = $f[4]; + $ids{ lc $id } = 1; + $ids{ lc substr($id, -16) } = 1 if length($id) > 16; + $ids{ lc substr($id, -8) } = 1 if length($id) > 8; + } + return \%ids; +} + +# rpm_signer_keyid: the signing key id (lowercased hex) of a built rpm's header signature, or undef +# when the rpm is not signed. Reads the RSA (or DSA) header pgpsig and pulls the "Key ID " field. +sub rpm_signer_keyid { + my ($rpm) = @_; + for my $tag (qw(RSAHEADER DSAHEADER)) { + my $out = `rpm -qp --qf '%{$tag:pgpsig}' ${\ sh_quote($rpm)} 2>/dev/null` // ''; + return lc($1) if $out =~ /Key ID\s+([0-9A-Fa-f]+)/i; + } + return undef; +} + # repomd_observed_signer: run gpg --verify on the detached repomd signature and extract the identity # of the key that actually signed it, as a primary-key fingerprint (the last field of the VALIDSIG # status line). Returns '' when the .asc is absent or verification fails (both read as "unsigned"). @@ -1255,6 +1286,20 @@ sub verify_target_repo { my %exp_sig = ('repomd' => $exp_fpr); my %obs_sig = ('repomd' => repomd_observed_signer($asc, $repomd, $gpg_home)); push @problems, verify_repo_signature(\%exp_sig, \%obs_sig); + + # Per-rpm signature gate: a signed repomd over an unsigned or foreign-signed rpm still + # makes DNF reject that package at install time, so verify EVERY binary rpm -- not just the + # metadata -- is signed by this key (rpm reports the signing subkey id; accept any id of + # the key). Closes the "approves a repo DNF later rejects" gap (PR #62 review #4). + require_command('rpm'); + my $accept = gpg_key_ids($gpg_key_name, $gpg_home); + if (!%$accept) { + push @problems, "SIGKEY: cannot list key ids for '$gpg_key_name' to verify per-rpm signatures"; + } else { + my @rpm_sigs = map { [ basename($_), rpm_signer_keyid($_) ] } + grep { !/\.src\.rpm$/ } glob("$dir/*.rpm"); + push @problems, verify_rpm_signatures(\@rpm_sigs, $accept); + } } } elsif ($sig_required) { # Standalone --verify-repo advertises a signature check; with no keyring we cannot resolve the @@ -1269,7 +1314,7 @@ sub verify_target_repo { die "FATAL: repo INCOMPLETE for $tgt at $dir (" . scalar(@problems) . " problem(s))\n"; } print "[verify-repo] $tgt complete: " . scalar(@names) - . " required packages present + version-pinned in $dir\n"; + . " required packages present + version-pinned, every rpm signed, in $dir\n"; return 1; } diff --git a/t/mockbuild-all.t b/t/mockbuild-all.t index d2201f3..5b038de 100644 --- a/t/mockbuild-all.t +++ b/t/mockbuild-all.t @@ -12,8 +12,8 @@ use File::Path qw(make_path); use File::Basename qw(basename); use MockBuildUtils qw(required_pkgs version_matches rpm_sigmd5 rpm_version rpm_release rpm_is_signed restamp_release_line cross_copy_genesis finalize_xcat_dep read_manifest - verify_repo_packages verify_repo_signature bump_dep_release_suffix - build_mock_uniqueext); + verify_repo_packages verify_repo_signature verify_rpm_signatures + bump_dep_release_suffix build_mock_uniqueext); # Run a printing sub with STDOUT muted so its progress lines do not pollute TAP. sub quiet(&) { @@ -184,6 +184,11 @@ SPEC my $ok4 = eval { quiet { finalize_xcat_dep("$tmp4/x", "$tmp4/p") }; 1 }; ok(!$ok4, 'finalize dies when a ppc64le OS has no x86_64 peer repo (was silently skipped)'); like($@, qr/no x86_64 peer repo/, 'finalize error names the missing x86_64 peer'); + + # @GENESIS_ARCHES is the single source of truth for the cross-arch matrix (add arches there). + my %tarch = map { $_->{arch} => $_->{tarch} } @MockBuildUtils::GENESIS_ARCHES; + is($tarch{x86_64}, 'x86_64', 'GENESIS_ARCHES: x86_64 maps to tarch x86_64'); + is($tarch{ppc64le}, 'ppc64', 'GENESIS_ARCHES: ppc64le maps to xCAT tarch ppc64'); } # ---- restamp_release_line: CD --build-number Release stamping (PR #62 review point 1) ---------- @@ -334,6 +339,24 @@ is(rpm_release(tempdir(CLEANUP => 1), 'nonexistent-pkg'), undef, 'rpm_release is 'verify_repo_signature: mismatch reported as WRONGKEY : signed by , expected '); } +# ---- verify_rpm_signatures: EVERY rpm must be signed by an accepted key (PR #62 review #4) ----- +{ + my %accept = ( '4123c420cb60ad43' => 1, 'cb60ad43' => 1 ); # signing key's long + short id + + my @ok = verify_rpm_signatures( + [ ['a-1.0.rpm', 'cb60ad43'], ['b-2.0.rpm', '4123C420CB60AD43'] ], \%accept); + is_deeply(\@ok, [], 'verify_rpm_signatures: all rpms signed by an accepted key -> no problems (case-insensitive)'); + + my @uns = verify_rpm_signatures([ ['c-3.0.rpm', undef], ['d-4.0.rpm', ''] ], \%accept); + is(scalar(@uns), 2, 'verify_rpm_signatures: undef and empty key id both flagged'); + like($uns[0], qr/^UNSIGNED rpm c-3\.0\.rpm$/, 'verify_rpm_signatures: unsigned rpm reported by name'); + + my @wrong = verify_rpm_signatures([ ['e-5.0.rpm', 'deadbeef'] ], \%accept); + is(scalar(@wrong), 1, 'verify_rpm_signatures: a foreign-signed rpm yields one problem'); + like($wrong[0], qr/^WRONGKEY rpm e-5\.0\.rpm: signed by deadbeef, expected one of\b/, + 'verify_rpm_signatures: wrong key reported as WRONGKEY rpm : signed by , expected one of ...'); +} + # ---- build_mock_uniqueext: distinct per target so concurrent mock roots never collide --------- # (PR #62 review) A long (timestamp) run id must not tail-truncate away the leading EL/arch token: # for the 7-char "ppc64le" arch that dropped the EL digit, so alma+epel-{8,9,10}-ppc64le collapsed to From a447caccea7576ae0a272263192cea9fc7e655da Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Fri, 21 Aug 2026 19:04:34 -0300 Subject: [PATCH 48/55] fix(xcat-dep): deploy each repo cell atomically, self-cleaning PR #62 review #3: deploy_target copied the built rpms ADDITIVELY into an existing rh/ cell (make_path + copy, no wipe), so stale snap-NVR rpms from an earlier run accumulated -- only the pipeline's pre-build `rm -rf rh*/` prevented a version pile-up, and a standalone run accumulated. It also mutated the live cell before the completeness/signature gate ran. Stage the cell in a sibling temp dir, sign+index+verify it THERE, then atomically swap it into place (move the old cell aside, rename the stage in, drop the old). The cell is rebuilt from scratch each run (self-cleaning, no reliance on the pipeline rm), a failed sign/index/verify leaves the previously-published cell untouched, and no reader sees a half-written cell. The staged metadata is path-independent (fixed xcat.org baseurl; mklocalrepo.sh resolves the path at runtime), so the swap is byte-for-byte what a direct write produced. The cross-arch genesis (--finalize-xcat-dep) runs later and re-populates the foreign-arch genesis, so rebuilding this single-arch cell from $src is correct. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- mockbuild-all.pl | 62 ++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 50 insertions(+), 12 deletions(-) diff --git a/mockbuild-all.pl b/mockbuild-all.pl index b091812..3bb85d3 100755 --- a/mockbuild-all.pl +++ b/mockbuild-all.pl @@ -801,19 +801,57 @@ sub deploy_target { my $dest = "$repo_dep/rh$rel/$arch"; print_step("Deploy $tgt -> $dest"); return if $dry_run; - make_path($dest); - for my $rpm (glob("$src/*.rpm")) { - next if $rpm =~ /\.src\.rpm$/; - copy($rpm, "$dest/" . basename($rpm)) - or die "Failed to copy $rpm -> $dest: $!\n"; + + # Stage the cell in a sibling temp dir, sign+index+verify it THERE, then atomically swap it into + # place. This makes the deploy self-cleaning and atomic (PR #62 review #3): + # - self-cleaning: the cell is rebuilt from scratch each run, so stale snap-NVR rpms from an + # earlier build never accumulate. Previously deploy copied ADDITIVELY into an existing $dest + # and relied on the pipeline pre-wiping rh/ -- a standalone run accumulated versions. + # - atomic + verify-before-replace: a failed sign/index/verify leaves the previously-published + # cell untouched, and no reader ever sees a half-written cell. + # The staging dir is a sibling of $dest, so the rename is a same-filesystem (atomic) move. The + # cross-arch genesis (--finalize-xcat-dep) runs later and re-populates the foreign-arch genesis, + # so rebuilding this single-arch cell from $src is correct. + make_path(dirname($dest)); + my $stage = "$dest.stage.$$"; + remove_tree($stage) if -d $stage; + make_path($stage); + my $ok = eval { + for my $rpm (glob("$src/*.rpm")) { + next if $rpm =~ /\.src\.rpm$/; + copy($rpm, "$stage/" . basename($rpm)) + or die "Failed to copy $rpm -> $stage: $!\n"; + } + sign_and_index_repo($stage); + write_dep_repo_metadata($stage, $rel); + # Automatic completeness + signature gate on the freshly signed cell -- the single + # consolidated gate (verify_target_repo, the same one --verify-repo runs). Asserts every + # manifest-required package is present at its pinned version, the repomd signature verifies, + # AND every rpm is signed by the key. Runs on the STAGE so a failure never lands in $dest. + # Suppressible with --no-verify-repo for iteration/debug. + verify_target_repo($stage, $tgt) unless $no_verify_repo; + 1; + }; + if (!$ok) { + my $err = $@; + remove_tree($stage); # leave the previously-published cell exactly as it was + die $err; } - sign_and_index_repo($dest); - write_dep_repo_metadata($dest, $rel); - # Automatic completeness + signature gate on the finalized, signed repo -- the single - # consolidated gate (verify_target_repo, the same one --verify-repo runs). Asserts every - # manifest-required package is present at its pinned version AND that the repomd signature - # verifies. Suppressible with --no-verify-repo for iteration/debug. - verify_target_repo($dest, $tgt) unless $no_verify_repo; + + # Atomic replace: rename cannot overwrite a populated dir, so move the old cell aside, swap the + # staged cell in, then drop the old one. On a failed final rename, restore the old cell. + my $old = "$dest.old.$$"; + remove_tree($old) if -d $old; + if (-d $dest) { + rename($dest, $old) or die "Failed to move old cell $dest aside: $!\n"; + } + unless (rename($stage, $dest)) { + my $err = $!; + rename($old, $dest) if -d $old && !-d $dest; # best-effort restore + die "Failed to swap staged cell into $dest: $err\n"; + } + remove_tree($old) if -d $old; + my $n = scalar(grep { !/\.src\.rpm$/ } glob("$dest/*.rpm")); print "Deployed rh$rel/$arch: $n rpms\n"; } From 9480dfe780cc8c9bf9596ef97e165a4788ccd98a Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Fri, 21 Aug 2026 20:53:08 -0300 Subject: [PATCH 49/55] feat(xcat-dep): EVR-constraint gate + rpmkeys checksig + repo_gpgcheck Three review follow-ups on the repo validation gate: 1. Full EPOCH:VERSION-RELEASE validation. The gate compared only %{VERSION}, so xCAT-genesis-base=2.* accepted a pre-2.18 (2.17.x) genesis even though xCAT-genesis-scripts Requires >= 2:2.18.0, and it could not enforce release floors like perl-IO-Stty >= 0.04-5. Manifest pins now also accept an EVR constraint (>=, >, <=, <, = followed by [epoch:]version[-release]); the built rpm's full EVR is compared with rpm's own algorithm (rpm.vercmp via the lua binding, injected into the pure evr_cmp, which composes epoch/version/release). genesis-base is pinned >= 2:2.18.0 and perl-IO-Stty >= 0.04-5. rpm_evr also catches release-level stale-artifact accumulation that rpm_version (VERSION dedup) missed. 2. RPM-native crypto verification. The per-rpm gate extracted the header signer id but did not verify digests/signatures. It now also runs `rpmkeys --checksig` against an isolated keyring holding only the signing key (exported from the gpg home), so every rpm's header/payload digests AND the signature-by-this-key are cryptographically verified; the signer-id origin check is kept alongside. 3. repo_gpgcheck. The generated xcat-dep.repo set only gpgcheck=1; add repo_gpgcheck=1 (mirroring gpgcheck) so clients enforce the detached repomd.xml.asc signature that sign_and_index_repo already produces. Validated: unit tests for parse_evr/evr_constraint_ok (rpm's real vercmp; the reviewer's 2.17.9-rejected, 0.04-4-rejected, epoch-enforced cases) + checksig verdict; and `--verify-repo` over a real signed rh8/x86_64 cell passes (12 packages EVR-satisfied, every rpm checksig-verified). Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- MockBuildUtils.pm | 94 +++++++++++++++++++++++++++++++++++++++--- mockbuild-all.pl | 91 ++++++++++++++++++++++++++++++++++++++-- packages-manifest.conf | 59 ++++++++++++++------------ t/mockbuild-all.t | 59 ++++++++++++++++++++++++++ 4 files changed, 268 insertions(+), 35 deletions(-) diff --git a/MockBuildUtils.pm b/MockBuildUtils.pm index 7755450..07d0876 100644 --- a/MockBuildUtils.pm +++ b/MockBuildUtils.pm @@ -16,6 +16,7 @@ our @EXPORT_OK = qw( sh_quote print_step version_matches required_pkgs have_rpm read_manifest verify_repo_packages verify_repo_signature verify_rpm_signatures + parse_evr evr_cmp evr_constraint_ok parse_pin rpmkeys_checksig_problem rpm_version rpm_release rpm_sigmd5 rpm_is_signed restamp_release_line cross_copy_genesis finalize_xcat_dep bump_dep_release_suffix build_mock_uniqueext @@ -72,21 +73,104 @@ sub required_pkgs { # Uses version_matches (same semantics as the in-line manifest pin loop), so a '*' or glob pin is # accepted exactly as there. No file/manifest I/O here -- the disk layer builds %present and passes # both hashes in, keeping this unit-testable in isolation. +# parse_evr($s): split an EVR string "[epoch:]version[-release]" into ($epoch, $version, $release). +# epoch defaults to '0' when absent or '(none)'; release is undef when the string carries none (so a +# release-less constraint compares version-only). Neither version nor release may contain '-', so the +# single '-' cleanly separates them. +sub parse_evr { + my ($s) = @_; + $s = '' unless defined $s; + my ($epoch, $rest); + if ($s =~ /^\s*(\d+):(.*)$/) { ($epoch, $rest) = ($1, $2); } + else { ($epoch, $rest) = ('0', $s); } + $epoch = '0' if !defined $epoch || $epoch eq '' || lc($epoch) eq '(none)'; + my ($ver, $rel) = split /-/, $rest, 2; + return ($epoch, $ver, $rel); # $rel undef when no release given +} + +# evr_cmp($got, $want, $vercmp): compare two EVRs with rpm's labelCompare semantics -- epoch first +# (numeric), then version, then release -- returning -1/0/1 (got vs want). $vercmp->($a,$b) is an +# injected rpm-native segment comparator (rpm's rpmvercmp) returning -1/0/1, so this stays pure and +# unit-testable. Release is compared only when the CONSTRAINT specifies one (rpm's EVR semantics: a +# version-only requirement ignores the built release). +sub evr_cmp { + my ($got, $want, $vercmp) = @_; + my ($ge, $gv, $gr) = parse_evr($got); + my ($we, $wv, $wr) = parse_evr($want); + return (($ge <=> $we) <=> 0) if ($ge <=> $we) != 0; # epoch: numeric + my $c = $vercmp->($gv, $wv); + return $c if $c; + return 0 unless defined $wr && $wr ne ''; # constraint release-agnostic + $gr = '' unless defined $gr; + return $vercmp->($gr, $wr); +} + +# evr_constraint_ok($got, $op, $want, $vercmp): does the observed EVR satisfy " "? +sub evr_constraint_ok { + my ($got, $op, $want, $vercmp) = @_; + my $c = evr_cmp($got, $want, $vercmp); + return $c >= 0 if $op eq '>='; + return $c > 0 if $op eq '>'; + return $c <= 0 if $op eq '<='; + return $c < 0 if $op eq '<'; + return $c == 0 if $op eq '=' || $op eq '=='; + return undef; # unknown operator +} + +# parse_pin($pin): classify a manifest version pin. +# '*' -> ('any') +# ' ' (>=,>,<=,<,=) -> ('evr', $op, $evr) full EPOCH:VERSION-RELEASE constraint +# glob or exact version -> ('version') %{VERSION}-only match (version_matches) +sub parse_pin { + my ($pin) = @_; + return ('any') if !defined($pin) || $pin eq '*'; + return ('evr', $1, $2) if $pin =~ /^\s*(>=|<=|==|=|>|<)\s*(\S+)\s*$/; + return ('version'); +} + sub verify_repo_packages { - my ($expected, $present) = @_; + my ($expected, $present_ver, $present_evr, $vercmp) = @_; + $present_evr //= $present_ver; my @problems; for my $pkg (sort keys %$expected) { my $pin = $expected->{$pkg}; - my $got = $present->{$pkg}; - if (!defined $got) { + my $got_ver = $present_ver->{$pkg}; + if (!defined $got_ver) { push @problems, "MISSING $pkg (manifest requires " . (defined($pin) ? $pin : '*') . ")"; - } elsif (!version_matches($got, $pin)) { - push @problems, "VERSION $pkg: repo has $got, manifest pins $pin"; + next; } + my ($kind, $op, $want) = parse_pin($pin); + if ($kind eq 'evr') { + my $got_evr = $present_evr->{$pkg} // $got_ver; + if (!$vercmp) { + push @problems, "EVR $pkg: no EVR comparator available to check '$op $want'"; + } elsif (!evr_constraint_ok($got_evr, $op, $want, $vercmp)) { + push @problems, "EVR $pkg: repo has $got_evr, manifest requires $op $want"; + } + } elsif ($kind eq 'version') { # VERSION glob/exact (unchanged) + push @problems, "VERSION $pkg: repo has $got_ver, manifest pins $pin" + if !version_matches($got_ver, $pin); + } + # 'any' -> accept } return @problems; } +# rpmkeys_checksig_problem($name, $rc, $out): pure verdict for one `rpmkeys --checksig -v` run against +# an isolated keyring holding only the signing key. A clean rpm exits 0 and every digest/signature +# line reads OK; a tampered digest reads NOT OK; an rpm signed by another key (or unsigned) reads +# NOKEY. Return a problem string (or empty list) so the gate is testable without rpm. +sub rpmkeys_checksig_problem { + my ($name, $rc, $out) = @_; + $out = '' unless defined $out; + return () if ($rc // 0) == 0 && $out !~ /NOT OK|NOKEY|MISSING KEYS/i; + my $why = $out =~ /NOT OK/i ? 'digest/signature NOT OK' + : $out =~ /NOKEY/i ? 'NOKEY (unsigned or signed by an unaccepted key)' + : $out =~ /MISSING KEYS/i ? 'MISSING KEYS' + : "rpmkeys --checksig failed (rc=" . ($rc // '?') . ")"; + return "BADSIG rpm $name: $why"; +} + # verify_repo_signature: the PURE signature-decision layer of the repo gate. Given %expected # { unit => expected signing-key identity } and %observed { unit => key that ACTUALLY signed (a # string the script extracts from gpg), or undef/'' when unsigned / verification failed }, return a diff --git a/mockbuild-all.pl b/mockbuild-all.pl index 3bb85d3..e544ef6 100755 --- a/mockbuild-all.pl +++ b/mockbuild-all.pl @@ -8,6 +8,7 @@ use File::Basename qw(dirname basename); use File::Copy qw(copy); use File::Find qw(find); use File::Path qw(make_path remove_tree); +use File::Temp qw(tempdir); use Getopt::Long qw(GetOptions); use Parallel::ForkManager; use POSIX qw(strftime); @@ -17,7 +18,7 @@ use MockBuildUtils qw(sh_quote print_step version_matches required_pkgs read_manifest verify_repo_packages verify_repo_signature verify_rpm_signatures rpm_version rpm_release rpm_sigmd5 restamp_release_line cross_copy_genesis finalize_xcat_dep bump_dep_release_suffix - build_mock_uniqueext); + build_mock_uniqueext rpmkeys_checksig_problem); # --- Mount-namespace isolation: guard the host cgroup against mock teardown propagation ---------- # mock mounts /sys/fs/cgroup into every build chroot. On these systemd build hosts every mount is @@ -889,6 +890,9 @@ sub write_dep_repo_metadata { my $baseurl = "https://xcat.org/files/xcat/repos/yum/devel/xcat-dep/rh$rel/$arch"; my $gpgcheck = $gpg_sign ? 1 : 0; my $gpgkey_line = $gpg_sign ? "gpgkey=$baseurl/repodata/repomd.xml.key" : "# gpgkey="; + # repo_gpgcheck=1 makes clients verify the DETACHED repomd.xml signature (repomd.xml.asc) against + # gpgkey before trusting the metadata -- sign_and_index_repo produces both, so enforce it. Mirrors + # gpgcheck: off when the repo is unsigned. open my $r, '>', "$dir/xcat-dep.repo" or die "Cannot write $dir/xcat-dep.repo: $!\n"; print {$r} <<"EOF"; [xcat-dep] @@ -896,6 +900,7 @@ name=xCAT 2 dependencies (rh$rel $arch) baseurl=$baseurl enabled=1 gpgcheck=$gpgcheck +repo_gpgcheck=$gpgcheck $gpgkey_line EOF close $r; @@ -1265,6 +1270,74 @@ sub rpm_signer_keyid { return undef; } +# rpm_evr: the single distinct EPOCH:VERSION-RELEASE of package $name's binary rpm(s) in $dir (epoch +# defaults to 0 when the header carries none), or undef if none match. Mirrors rpm_version's dedup: +# more than one distinct EVR means a stale artifact was not cleaned before the build (a version pin +# could then pass against the wrong rpm). genesis's x86_64 + ppc64 rpms share one EVR, so a normal +# pair is a single entry. +sub rpm_evr { + my ($dir, $name) = @_; + my $glob = ($name eq 'xCAT-genesis-base') + ? "$dir/xCAT-genesis-base-*.rpm" + : "$dir/${name}-*.rpm"; + my %evrs; + for my $f (sort glob($glob)) { + next if $f =~ /\.src\.rpm$/ || $f =~ /-debug(?:info|source)-/; + my $n = `rpm -qp --qf '%{name}' ${\ sh_quote($f)} 2>/dev/null`; + my $match = ($name eq 'xCAT-genesis-base') + ? ($n =~ /^xCAT-genesis-base-/) : ($n eq $name); + next unless $match; + my $evr = `rpm -qp --qf '%{epochnum}:%{version}-%{release}' ${\ sh_quote($f)} 2>/dev/null`; + chomp $evr; + $evrs{$evr} = 1 if $evr ne ''; + } + return undef unless %evrs; + die "Multiple EVRs of $name present in $dir: " . join(', ', sort keys %evrs) + . " (stale artifact not cleaned before the build)\n" if keys(%evrs) > 1; + my ($evr) = keys %evrs; + return $evr; +} + +# rpm_vercmp_segment: ONE rpmvercmp segment comparison via rpm's own lua binding, returning -1/0/1. +# Used as the injected comparator for MockBuildUtils::evr_cmp so the EVR gate uses rpm's canonical +# version algorithm (epoch/release composition is done in evr_cmp). Long-bracket the args so any +# version char (. _ ~ ^ +) passes through literally; rpm versions never contain the ]==] sequence. +sub rpm_vercmp_segment { + my ($a, $b) = @_; + $a = '' unless defined $a; + $b = '' unless defined $b; + my $out = `rpm --eval '%{lua:print(rpm.vercmp([==[$a]==],[==[$b]==]))}' 2>/dev/null`; + chomp $out; + die "FATAL: rpm.vercmp gave no result for '$a' vs '$b'\n" unless $out =~ /^-?\d+$/; + return $out <=> 0; +} + +# verify_rpms_checksig: cryptographically verify EVERY binary rpm in $dir with `rpmkeys --checksig` +# against an ISOLATED keyring holding only the signing key. This is the RPM-native integrity + origin +# check: it verifies each rpm's header/payload digests AND that the signature is by this key (NOKEY / +# NOT OK => a real failure, since the key IS imported). Returns @problems. +sub verify_rpms_checksig { + my ($dir, $keyname, $home) = @_; + my @rpms = grep { !/\.src\.rpm$/ } glob("$dir/*.rpm"); + return () unless @rpms; + require_command('rpmkeys'); + require_command('gpg'); + my $tmpdb = tempdir('rpmkeys-XXXXXXXX', TMPDIR => 1, CLEANUP => 1); + my $h = ($home ne '') ? ' --homedir ' . sh_quote($home) : ''; + my $keyfile = "$tmpdb/pubkey.asc"; + system("gpg$h --batch --yes -a --export " . sh_quote($keyname) . ' > ' . sh_quote($keyfile) . ' 2>/dev/null'); + return ("SIGKEY: cannot export public key '$keyname' for rpmkeys --checksig") if !-s $keyfile; + my $dbopt = '--dbpath ' . sh_quote($tmpdb); + system("rpmkeys $dbopt --import " . sh_quote($keyfile) . ' >/dev/null 2>&1') == 0 + or return ("SIGKEY: rpmkeys --import of '$keyname' into the temp keyring failed"); + my @problems; + for my $rpm (@rpms) { + my $out = `rpmkeys $dbopt --checksig -v ${\ sh_quote($rpm)} 2>&1`; + push @problems, rpmkeys_checksig_problem(basename($rpm), $? >> 8, $out); + } + return @problems; +} + # repomd_observed_signer: run gpg --verify on the detached repomd signature and extract the identity # of the key that actually signed it, as a primary-key fingerprint (the last field of the VALIDSIG # status line). Returns '' when the .asc is absent or verification fails (both read as "unsigned"). @@ -1302,9 +1375,14 @@ sub verify_target_repo { die "FATAL: no manifest section for target '$tgt' in $manifest\n" if !%req; # Skip flags default 0 -> the full required set. A package whose builder was skipped is not required. my @names = required_pkgs([sort keys %req], $skip_genesis, $skip_perl, $skip_xcat_dep); - my %present = repo_present_versions($dir, \@names); + my %present = repo_present_versions($dir, \@names); + # Full EPOCH:VERSION-RELEASE per package, so a manifest EVR constraint (e.g. genesis-base + # '>= 2:2.18.0', which %{VERSION}-only matching cannot enforce -- 2.* would accept a pre-2.18 + # genesis) is checked with rpm's own version algorithm (PR #62 review). rpm_vercmp_segment is + # rpm's rpmvercmp; evr_cmp composes epoch/version/release around it. + my %present_evr = map { $_ => rpm_evr($dir, $_) } @names; my %expected = map { $_ => $req{$_} } @names; - my @problems = verify_repo_packages(\%expected, \%present); + my @problems = verify_repo_packages(\%expected, \%present, \%present_evr, \&rpm_vercmp_segment); # Signature gate: the IO (gpg) lives here; the decision is the pure verify_repo_signature. The # pipeline always signs, so a signed repo's repomd MUST be signed by --gpg-key-name. We resolve @@ -1330,6 +1408,11 @@ sub verify_target_repo { # metadata -- is signed by this key (rpm reports the signing subkey id; accept any id of # the key). Closes the "approves a repo DNF later rejects" gap (PR #62 review #4). require_command('rpm'); + # (a) RPM-native crypto verification: rpmkeys --checksig against an isolated keyring + # holding only this key verifies every rpm's digests AND that the signature is by the key. + push @problems, verify_rpms_checksig($dir, $gpg_key_name, $gpg_home); + # (b) Explicit signer-id origin check kept alongside: assert each rpm's header signature + # key id is one of this key's ids (primary/subkey). my $accept = gpg_key_ids($gpg_key_name, $gpg_home); if (!%$accept) { push @problems, "SIGKEY: cannot list key ids for '$gpg_key_name' to verify per-rpm signatures"; @@ -1352,7 +1435,7 @@ sub verify_target_repo { die "FATAL: repo INCOMPLETE for $tgt at $dir (" . scalar(@problems) . " problem(s))\n"; } print "[verify-repo] $tgt complete: " . scalar(@names) - . " required packages present + version-pinned, every rpm signed, in $dir\n"; + . " required packages present + EVR-satisfied, repomd + every rpm checksig-verified, in $dir\n"; return 1; } diff --git a/packages-manifest.conf b/packages-manifest.conf index 1669a87..955813b 100644 --- a/packages-manifest.conf +++ b/packages-manifest.conf @@ -1,22 +1,29 @@ # Per-target required xcat-dep package manifest. # # One [section] per mockbuild-all target (matches --target). Each entry is -# = +# = # where is the builder/package name (the dep builder name, the perl -# package name, or xCAT-genesis-base) and is one of: -# - an exact Version (e.g. 1.8.18) -- the build must produce exactly it; -# - a shell-style glob (e.g. 2.*) -- the built Version must match it (* and ?); -# - '*' -- any version is accepted. -# Only the Version is matched, never the Release (which carries the per-EL dist -# tag elN and the genesis snap). Bump an exact pin here when the -# corresponding in-tree source Version is bumped. +# package name, or xCAT-genesis-base) and is one of: +# - an exact Version (e.g. 1.8.18) -- the built %{VERSION} must equal it; +# - a shell-style glob (e.g. 2.*) -- the built %{VERSION} must match it (* and ?); +# - '*' -- any version is accepted; +# - an EVR constraint (e.g. >= 2:2.18.0, >= 0.04-5, = 1.8.18) -- an operator +# (>=, >, <=, <, =) followed by an [epoch:]version[-release]. The built rpm's +# full EPOCH:VERSION-RELEASE is compared with rpm's own version algorithm +# (rpm.vercmp; epoch numeric, then version, then release; release ignored when +# the constraint omits it). Use this to enforce a minimum that a %{VERSION} +# glob cannot -- e.g. a release floor (perl-IO-Stty >= 0.04-5) or an Epoch. +# A bare version pin matches %{VERSION} only (Release carries the per-EL dist tag +# elN and the snap, so it is not pinned there). Bump a pin here when the +# corresponding in-tree source is bumped. # -# xCAT-genesis-base is pinned as 2.* (not an exact version) on purpose: its -# Version is NOT owned by xcat-dep -- it is whatever xcat-core the genesis build -# compiles against (XCAT_CORE_REF), so it walks with the paired core (2.18.x, -# 2.19.x, ...). 2.* asserts "a 2.x genesis" without coupling the manifest to one -# core release. (xCAT-genesis-scripts Requires xCAT-genesis-base >= 2:2.18.0 -- a -# minimum with Epoch 2 -- so any 2.x genesis-base installs against a 2.18+ core.) +# xCAT-genesis-base is pinned as '>= 2:2.18.0' (an EVR floor, not an exact version): +# its Version is NOT owned by xcat-dep -- it is whatever xcat-core the genesis build +# compiles against (XCAT_CORE_REF), so it walks with the paired core (2.18.x, 2.19.x, +# ...). The floor still walks (accepts any 2.18+ genesis) but, unlike the old '2.*' +# glob, REJECTS a pre-2.18 genesis -- xCAT-genesis-scripts Requires xCAT-genesis-base +# >= 2:2.18.0 (Epoch 2), which a 2.17.x genesis would violate. genesis-base carries +# Epoch 2, so the epoch in the constraint is enforced too. # # mockbuild-all.pl reads this file and, per target, builds ONLY the listed # packages -- a package not listed for a target is not built for it. Any listed @@ -45,9 +52,9 @@ syslinux-xcat=6.03 xnba-undi=1.21.1 perl-HTML-Form=6.07 perl-HTTP-Async=0.30 -perl-IO-Stty=0.04 +perl-IO-Stty=>= 0.04-5 perl-Net-HTTPS-NB=0.14 -xCAT-genesis-base=2.* +xCAT-genesis-base=>= 2:2.18.0 [alma+epel-8-ppc64le] conserver-xcat=8.2.1 @@ -59,9 +66,9 @@ syslinux-xcat=6.03 xnba-undi=1.21.1 perl-HTML-Form=6.07 perl-HTTP-Async=0.30 -perl-IO-Stty=0.04 +perl-IO-Stty=>= 0.04-5 perl-Net-HTTPS-NB=0.14 -xCAT-genesis-base=2.* +xCAT-genesis-base=>= 2:2.18.0 [alma+epel-9-x86_64] conserver-xcat=8.2.1 @@ -72,10 +79,10 @@ ipmitool-xcat=1.8.18 syslinux-xcat=6.03 xnba-undi=1.21.1 perl-HTTP-Async=0.30 -perl-IO-Stty=0.04 +perl-IO-Stty=>= 0.04-5 perl-Net-HTTPS-NB=0.14 perl-Sys-Virt=11.10.0 -xCAT-genesis-base=2.* +xCAT-genesis-base=>= 2:2.18.0 [alma+epel-9-ppc64le] conserver-xcat=8.2.1 @@ -86,10 +93,10 @@ ipmitool-xcat=1.8.18 syslinux-xcat=6.03 xnba-undi=1.21.1 perl-HTTP-Async=0.30 -perl-IO-Stty=0.04 +perl-IO-Stty=>= 0.04-5 perl-Net-HTTPS-NB=0.14 perl-Sys-Virt=11.10.0 -xCAT-genesis-base=2.* +xCAT-genesis-base=>= 2:2.18.0 [alma+epel-10-x86_64] conserver-xcat=8.2.1 @@ -101,11 +108,11 @@ syslinux-xcat=6.03 xnba-undi=1.21.1 perl-Crypt-SSLeay=0.72 perl-HTTP-Async=0.30 -perl-IO-Stty=0.04 +perl-IO-Stty=>= 0.04-5 perl-Net-HTTPS-NB=0.14 perl-Net-Telnet=3.04 perl-Sys-Virt=11.10.0 -xCAT-genesis-base=2.* +xCAT-genesis-base=>= 2:2.18.0 [alma+epel-10-ppc64le] conserver-xcat=8.2.1 @@ -117,8 +124,8 @@ syslinux-xcat=6.03 xnba-undi=1.21.1 perl-Crypt-SSLeay=0.72 perl-HTTP-Async=0.30 -perl-IO-Stty=0.04 +perl-IO-Stty=>= 0.04-5 perl-Net-HTTPS-NB=0.14 perl-Net-Telnet=3.04 perl-Sys-Virt=11.10.0 -xCAT-genesis-base=2.* +xCAT-genesis-base=>= 2:2.18.0 diff --git a/t/mockbuild-all.t b/t/mockbuild-all.t index 5b038de..382cf2a 100644 --- a/t/mockbuild-all.t +++ b/t/mockbuild-all.t @@ -13,6 +13,7 @@ use File::Basename qw(basename); use MockBuildUtils qw(required_pkgs version_matches rpm_sigmd5 rpm_version rpm_release rpm_is_signed restamp_release_line cross_copy_genesis finalize_xcat_dep read_manifest verify_repo_packages verify_repo_signature verify_rpm_signatures + parse_evr evr_constraint_ok parse_pin rpmkeys_checksig_problem bump_dep_release_suffix build_mock_uniqueext); # Run a printing sub with STDOUT muted so its progress lines do not pollute TAP. @@ -357,6 +358,64 @@ is(rpm_release(tempdir(CLEANUP => 1), 'nonexistent-pkg'), undef, 'rpm_release is 'verify_rpm_signatures: wrong key reported as WRONGKEY rpm : signed by , expected one of ...'); } +# ---- EVR constraints: full EPOCH:VERSION-RELEASE validation (PR #62 review) ------------------- +# rpm's own version algorithm, via its lua rpm.vercmp binding, is the injected segment comparator -- +# the same primitive mockbuild-all passes in production, so these assert the real rpm semantics. +my $vercmp = sub { + my ($a, $b) = @_; + my $o = `rpm --eval '%{lua:print(rpm.vercmp([==[$a]==],[==[$b]==]))}' 2>/dev/null`; + chomp $o; return $o <=> 0; +}; +{ + is_deeply([parse_evr('2:2.18.0-5')], ['2','2.18.0','5'], 'parse_evr: epoch:version-release'); + is_deeply([parse_evr('2.18.0')], ['0','2.18.0',undef], 'parse_evr: bare version -> epoch 0, no release'); + is_deeply([parse_evr('0.04-5.el8')], ['0','0.04','5.el8'], 'parse_evr: release kept whole'); + + is_deeply([parse_pin('>= 2:2.18.0')], ['evr','>=','2:2.18.0'], 'parse_pin: EVR operator constraint'); + is_deeply([parse_pin('2.*')], ['version'], 'parse_pin: glob stays a version pin'); + is_deeply([parse_pin('*')], ['any'], 'parse_pin: * is any'); + + # the reviewer's cases: genesis-base >= 2:2.18.0 rejects a pre-2.18 (Epoch 2) genesis... + ok( evr_constraint_ok('2:2.19.0-snap202607211907', '>=', '2:2.18.0', $vercmp), + 'EVR: 2:2.19.0 satisfies >= 2:2.18.0'); + ok(!evr_constraint_ok('2:2.17.9-snap', '>=', '2:2.18.0', $vercmp), + 'EVR: 2:2.17.9 REJECTED by >= 2:2.18.0 (2.* would have wrongly accepted it)'); + ok(!evr_constraint_ok('0:2.18.0-1', '>=', '2:2.18.0', $vercmp), + 'EVR: epoch enforced -- 0:2.18.0 rejected by >= 2:2.18.0'); + # ...and a release floor perl-IO-Stty >= 0.04-5. + ok( evr_constraint_ok('0.04-5.el8.snap202607221225.13', '>=', '0.04-5', $vercmp), + 'EVR: 0.04-5.el8.snap... satisfies release floor >= 0.04-5'); + ok(!evr_constraint_ok('0.04-4.el8', '>=', '0.04-5', $vercmp), + 'EVR: 0.04-4 REJECTED by release floor >= 0.04-5 (VERSION-only match would have passed)'); + + # end-to-end through the gate: EVR pin honored, with the got EVR supplied separately from %{VERSION}. + my @okp = verify_repo_packages( + { 'xCAT-genesis-base' => '>= 2:2.18.0' }, + { 'xCAT-genesis-base' => '2.19.0' }, + { 'xCAT-genesis-base' => '2:2.19.0-snap202607211907' }, $vercmp); + is_deeply(\@okp, [], 'gate: EVR-satisfying genesis passes'); + my @badp = verify_repo_packages( + { 'xCAT-genesis-base' => '>= 2:2.18.0' }, + { 'xCAT-genesis-base' => '2.17.0' }, + { 'xCAT-genesis-base' => '2:2.17.0-snap' }, $vercmp); + is(scalar(@badp), 1, 'gate: pre-2.18 genesis yields exactly one problem'); + like($badp[0], qr/^EVR xCAT-genesis-base: repo has 2:2\.17\.0-snap, manifest requires >= 2:2\.18\.0$/, + 'gate: EVR failure names the observed EVR and the requirement'); +} + +# ---- rpmkeys --checksig verdict (pure) ------------------------------------------------------- +{ + is_deeply([rpmkeys_checksig_problem('a.rpm', 0, + "Header V4 RSA/SHA256 Signature, key ID cb60ad43: OK\nPayload SHA256 digest: OK\n")], [], + 'checksig: all-OK rpm -> no problem'); + my @nok = rpmkeys_checksig_problem('b.rpm', 1, "Header SHA256 digest: NOT OK\n"); + like($nok[0], qr/^BADSIG rpm b\.rpm: digest\/signature NOT OK$/, 'checksig: NOT OK flagged'); + my @nokey = rpmkeys_checksig_problem('c.rpm', 1, "Header V4 RSA/SHA256 Signature, key ID deadbeef: NOKEY\n"); + like($nokey[0], qr/^BADSIG rpm c\.rpm: NOKEY/, 'checksig: NOKEY (unaccepted/unsigned) flagged'); + my @rc = rpmkeys_checksig_problem('d.rpm', 2, ""); + like($rc[0], qr/rc=2/, 'checksig: non-zero exit with no marker still flagged'); +} + # ---- build_mock_uniqueext: distinct per target so concurrent mock roots never collide --------- # (PR #62 review) A long (timestamp) run id must not tail-truncate away the leading EL/arch token: # for the 7-char "ppc64le" arch that dropped the EL digit, so alma+epel-{8,9,10}-ppc64le collapsed to From b3103ec690b4f603ff98154c88772f618351ce53 Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Mon, 24 Aug 2026 18:19:12 -0300 Subject: [PATCH 50/55] fix(xcat-dep): satisfy the perlcritic gate master now runs over mockbuild-all.pl PR #64 added a perlcritic step to the package-tests workflow, and mockbuild-all.pl is on its file list. This branch's code had never been run through it, so the first CI run after merging master failed on six severity-5 findings. Two of them cannot be fixed the way the policy's headline suggests: rpm_evr and rpm_signer_keyid are both called in LIST context -- `map { $_ => rpm_evr(...) }` builds a hash, and `map { [ basename($_), rpm_signer_keyid($_) ] }` builds pairs -- where a bare `return` yields NO element instead of one undef, silently shifting every following hash pair and truncating the signature tuples. All five "return undef" sites therefore return an undefined variable, which keeps the one-element list, and the "return sort" is assigned first. No behaviour change: 238 tests still pass, and perlcritic is now clean over the whole gated file list. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- mockbuild-all.pl | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/mockbuild-all.pl b/mockbuild-all.pl index e38c730..884c3da 100755 --- a/mockbuild-all.pl +++ b/mockbuild-all.pl @@ -1298,7 +1298,8 @@ sub run_build_steps_parallel { } assert_build_progress(scalar(@{$steps}), scalar(keys %failed)); - return sort keys %failed; + my @failed_ids = sort keys %failed; + return @failed_ids; } # The caller enforces zero tolerance per required package (verify_target_repo). ALL steps @@ -1347,8 +1348,9 @@ sub gpg_key_fingerprint { elsif ($line =~ /^sub:/) { $want = 0; } elsif ($want && $line =~ /^fpr:+([0-9A-Fa-f]+):/) { push @fprs, $1; $want = 0; } } - return undef if @fprs != 1; - return $fprs[0]; + my $fingerprint; + $fingerprint = $fprs[0] if @fprs == 1; + return $fingerprint; } # gpg_key_ids: all acceptable key ids (lowercased) for a signing key NAME -- the primary key id AND @@ -1375,11 +1377,12 @@ sub gpg_key_ids { # when the rpm is not signed. Reads the RSA (or DSA) header pgpsig and pulls the "Key ID " field. sub rpm_signer_keyid { my ($rpm) = @_; + my $keyid; for my $tag (qw(RSAHEADER DSAHEADER)) { my $out = `rpm -qp --qf '%{$tag:pgpsig}' ${\ sh_quote($rpm)} 2>/dev/null` // ''; - return lc($1) if $out =~ /Key ID\s+([0-9A-Fa-f]+)/i; + if ($out =~ /Key ID\s+([0-9A-Fa-f]+)/i) { $keyid = lc($1); last; } } - return undef; + return $keyid; } # rpm_evr: the single distinct EPOCH:VERSION-RELEASE of package $name's binary rpm(s) in $dir (epoch @@ -1403,10 +1406,11 @@ sub rpm_evr { chomp $evr; $evrs{$evr} = 1 if $evr ne ''; } - return undef unless %evrs; + my $evr; + return $evr unless %evrs; die "Multiple EVRs of $name present in $dir: " . join(', ', sort keys %evrs) . " (stale artifact not cleaned before the build)\n" if keys(%evrs) > 1; - my ($evr) = keys %evrs; + ($evr) = keys %evrs; return $evr; } @@ -1556,9 +1560,10 @@ sub verify_target_repo { # so the standalone --verify-repo mode can require an explicit --target instead. sub derive_target_from_repo_path { my ($dir) = @_; - return undef unless defined $dir; - return "alma+epel-$1-$2" if $dir =~ m{/rh(\d+)/([^/]+)/*$}; - return undef; + my $tgt; + return $tgt unless defined $dir; + $tgt = "alma+epel-$1-$2" if $dir =~ m{/rh(\d+)/([^/]+)/*$}; + return $tgt; } sub reset_staging_repo { From 632379e2d8d33a6fd042aad6d6c5c4021f1cc8cc Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Wed, 26 Aug 2026 17:15:08 -0300 Subject: [PATCH 51/55] test(xcat-dep): capture --skip-* weakening the per-target repo gate verify_target_repo filters the manifest through required_pkgs() with the invocation's --skip-genesis / --skip-perl / --skip-xcat-dep, so the flags that say what THIS run built also decide what the verified repository is allowed to be missing. A repo with no xCAT-genesis-base passes when the verifying run was given --skip-genesis (PR #62 review). Drives the real `mockbuild-all.pl --verify-repo` over fixture repos built from two minimal rpms, so the gate reads real header names. The assertions are on the reported problems rather than the exit code: a standalone --verify-repo demands a repomd signature by contract and these fixtures are unsigned, so it exits non-zero either way -- what separates a working gate from a broken one is whether the missing package is NAMED. The complete-repo baseline is asserted too, so the test cannot pass by the gate simply always complaining. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- t/verify-repo-el.t | 116 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 t/verify-repo-el.t diff --git a/t/verify-repo-el.t b/t/verify-repo-el.t new file mode 100644 index 0000000..3ae9bc4 --- /dev/null +++ b/t/verify-repo-el.t @@ -0,0 +1,116 @@ +#!/usr/bin/perl +# Focused end-to-end test for the PER-TARGET REPO GATE, driving the real +# `mockbuild-all.pl --verify-repo` against a hand-built fixture repo. +# +# What is tested here is the wiring the PR #62 review found: the gate filtered the manifest through +# required_pkgs() with THIS invocation's --skip-* flags, so the flags that say what an invocation +# BUILT also decided what the published repository was allowed to be missing. A repo carrying no +# xCAT-genesis-base therefore passed when the run that verified it had been given --skip-genesis. +# +# The assertions are on the REPORTED PROBLEMS, not on the exit code: a standalone --verify-repo also +# demands a repomd signature by contract, and these fixtures are unsigned, so it exits non-zero +# either way. What distinguishes a fixed gate from a broken one is whether the missing package is +# NAMED. +use strict; +use warnings; +use Test::More; +use FindBin qw($RealBin); +use File::Temp qw(tempdir); +use File::Path qw(make_path); + +my $SCRIPT = "$RealBin/../mockbuild-all.pl"; +plan skip_all => "mockbuild-all.pl not found" unless -f $SCRIPT; +plan skip_all => "rpm tooling required" + unless $^O eq 'linux' + && !system('sh', '-c', 'command -v rpmbuild >/dev/null 2>&1') + && !system('sh', '-c', 'command -v rpm >/dev/null 2>&1') + && !system('sh', '-c', 'command -v createrepo_c >/dev/null 2>&1'); + +my $tmp = tempdir(CLEANUP => 1); +my $target = 'alma+epel-10-x86_64'; + +# build_rpm($name, $version): a minimal noarch rpm, so the gate reads a REAL header name. +sub build_rpm { + my ($name, $version) = @_; + my $top = "$tmp/rpmbuild"; + make_path("$top/SPECS"); + my $spec = "$top/SPECS/$name.spec"; + open my $fh, '>', $spec or die $!; + print $fh <<"SPEC"; +Name: $name +Version: $version +Release: 1 +Summary: fixture +License: EPL +BuildArch: noarch +%description +fixture package for the repository gate test +%install +mkdir -p %{buildroot}/usr/share/$name +%files +/usr/share/$name +SPEC + close $fh; + my $rc = system('rpmbuild', '--quiet', '-bb', '--define', "_topdir $top", $spec); + die "cannot build fixture rpm $name\n" if $rc != 0; + my ($built) = glob("$top/RPMS/noarch/$name-$version-1.noarch.rpm"); + die "fixture rpm $name not produced\n" unless $built && -f $built; + return $built; +} + +# make_repo(%opt): an indexed repo carrying ipmitool-xcat, and xCAT-genesis-base unless with_genesis +# is turned off. +sub make_repo { + my (%o) = @_; + my $dir = "$tmp/repo" . ($o{tag} // ''); + make_path($dir); + system('cp', build_rpm('ipmitool-xcat', '1.8.18'), $dir) == 0 or die $!; + system('cp', build_rpm('xCAT-genesis-base-x86_64', '2.18.0'), $dir) == 0 or die $! + if $o{with_genesis}; + system('createrepo_c', '--quiet', $dir) == 0 or die "createrepo_c failed\n"; + return $dir; +} + +# The manifest is the source of truth for what the target must carry. +my $root = "$tmp/repo-root"; +make_path($root); +open my $m, '>', "$root/packages-manifest.conf" or die $!; +print $m "[$target]\nipmitool-xcat=1.8.18\nxCAT-genesis-base=2.*\n"; +close $m; + +sub run_gate { + my ($repo, @extra) = @_; + my $cmd = join(' ', map { my $x = $_; $x =~ s/'/'"'"'/g; "'$x'" } + ($^X, $SCRIPT, '--verify-repo', $repo, '--target', $target, '--repo-root', $root, @extra)) + . ' 2>&1'; + my $out = `$cmd`; + return ($? >> 8, defined $out ? $out : ''); +} + +# ---- baseline: a complete repo reports no MISSING package --------------------------------------- +{ + my $repo = make_repo(tag => '-full', with_genesis => 1); + my (undef, $out) = run_gate($repo); + unlike($out, qr/MISSING/, 'a complete repo reports no missing package') or diag($out); +} + +# ---- --skip-genesis must not excuse a repo that lacks Genesis ------------------------------------ +{ + my $repo = make_repo(tag => '-nogenesis'); + my (undef, $out) = run_gate($repo, '--skip-genesis'); + like($out, qr/xCAT-genesis-base/, + 'a repo missing Genesis is reported even when the run passed --skip-genesis') or diag($out); +} + +# ---- the same for the compiled deps --------------------------------------------------------------- +{ + my $repo = make_repo(tag => '-nodeps'); + system('rm', '-f', glob("$repo/ipmitool-xcat-*.rpm")) == 0 or die $!; + system('createrepo_c', '--quiet', '--update', $repo) == 0 or die $!; + my (undef, $out) = run_gate($repo, '--skip-xcat-dep'); + like($out, qr/ipmitool-xcat/, + 'a repo missing a compiled dep is reported even when the run passed --skip-xcat-dep') + or diag($out); +} + +done_testing(); From 36f9e38a2688f116c3b1db72c983bc86fcbcc824 Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Wed, 26 Aug 2026 17:18:20 -0300 Subject: [PATCH 52/55] fix(xcat-dep): gate each target on the whole manifest, whatever this run built verify_target_repo filtered the manifest through required_pkgs() with the invocation's --skip-genesis / --skip-perl / --skip-xcat-dep, so the flags that describe what a run BUILT also decided what the verified repository was allowed to lack: a repo with no xCAT-genesis-base passed whenever the verifying run carried --skip-genesis (PR #62 review). Those flags mean "this invocation did not build it", never "the repository may ship without it" -- a package an earlier run produced is still expected to be present. The gate now takes the manifest whole. No change for the CD pipeline, which passes no package-selection skips; it closes the hole for the documented skip-mode and finalize invocations. The Genesis-release consumer fixtures now pass --no-verify-repo. Their dependency packages are copies of a single rpm, so no manifest describes them the way a real one describes a real build -- with the gate honest, a fixture manifest could only be satisfied by lying about what the cell contains. The gate is covered instead against purpose-built rpms in t/verify-repo-el.t, and those runs still need a manifest SECTION to exist, which is all they ever needed. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- mockbuild-all.pl | 7 +++++-- t/genesis_openembedded_consumer.t | 26 ++++++++++++++++++++------ 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/mockbuild-all.pl b/mockbuild-all.pl index 129ec87..6bdbc84 100755 --- a/mockbuild-all.pl +++ b/mockbuild-all.pl @@ -1562,8 +1562,11 @@ sub verify_target_repo { my %MAN = read_manifest($manifest); my %req = %{ $MAN{$tgt} // {} }; die "FATAL: no manifest section for target '$tgt' in $manifest\n" if !%req; - # Skip flags default 0 -> the full required set. A package whose builder was skipped is not required. - my @names = required_pkgs([sort keys %req], $skip_genesis, $skip_perl, $skip_xcat_dep); + # The WHOLE manifest, deliberately -- the --skip-* flags are NOT applied here. They say what + # this INVOCATION built; they never say what the verified repository may be missing. Honouring + # them let a repo with no xCAT-genesis-base pass whenever the verifying run happened to carry + # --skip-genesis (PR #62 review). A package an earlier run built is still expected to be here. + my @names = sort keys %req; my %present = repo_present_versions($dir, \@names); # Full EPOCH:VERSION-RELEASE per package, so a manifest EVR constraint (e.g. genesis-base # '>= 2:2.18.0', which %{VERSION}-only matching cannot enforce -- 2.* would accept a pre-2.18 diff --git a/t/genesis_openembedded_consumer.t b/t/genesis_openembedded_consumer.t index 409e88d..65c4542 100644 --- a/t/genesis_openembedded_consumer.t +++ b/t/genesis_openembedded_consumer.t @@ -87,12 +87,15 @@ SKIP: { done_testing(); # mockbuild-all.pl builds and gates each target against /packages-manifest.conf, and a -# target with no section there is fatal. These fixtures use a synthetic target, and their dependency -# packages are copies of one rpm -- so the section names the header name those copies actually carry. -# The per-package manifest gate itself is covered by t/mockbuild-all.t; what these runs exercise is -# the Genesis release path. The runs pass --skip-genesis rather than the removed --skip-xcat: this -# script no longer builds the xCAT core, so xcat-core's buildrpms.pl is required only for the -# per-EL xCAT-genesis-base build. +# target with no section there is fatal -- so these runs, which use a synthetic target, need a +# section to exist at all. Its CONTENT is deliberately not meaningful: the dependency packages here +# are copies of one rpm, so no set of names describes them the way a real manifest describes a real +# build. The runs therefore pass --no-verify-repo and the completeness gate is covered where it can +# be tested honestly, against purpose-built rpms, in t/verify-repo-el.t. What these runs exercise is +# the Genesis release path. +# +# They pass --skip-genesis rather than the removed --skip-xcat: this script no longer builds the +# xCAT core, so xcat-core's buildrpms.pl is required only for the per-EL xCAT-genesis-base build. sub write_target_manifest { my ($root, $target) = @_; make_path($root); @@ -168,6 +171,7 @@ sub test_rpm_consumer { '--run-id', 'consumer', '--build-timestamp', $epoch, '--skip-build', '--skip-genesis', '--skip-xcat-dep', '--skip-perl', + '--no-verify-repo', '--skip-createrepo', '--skip-tarball', '--genesis-release', $release_root, '--collect-dir', $dependencies, @@ -272,6 +276,7 @@ SH '--run-id', 'publication-failure', '--build-timestamp', $epoch, '--skip-build', '--skip-genesis', '--skip-xcat-dep', '--skip-perl', + '--no-verify-repo', '--skip-createrepo', '--skip-tarball', '--genesis-release', $release_root, '--collect-dir', $dependencies, @@ -615,6 +620,7 @@ sub test_signed_common_rpm_repository { '--run-id', 'signed-consumer', '--build-timestamp', $epoch, '--skip-build', '--skip-genesis', '--skip-xcat-dep', '--skip-perl', + '--no-verify-repo', '--skip-createrepo', '--skip-tarball', '--collect-dir', $dependencies, '--genesis-release', $release_root, @@ -673,6 +679,7 @@ sub test_legacy_rpm_consumer { '--target', $target, '--run-id', 'legacy', '--skip-build', '--skip-genesis', '--skip-xcat-dep', '--skip-perl', + '--no-verify-repo', '--skip-createrepo', '--skip-tarball', '--collect-dir', $dependencies, ); @@ -736,6 +743,7 @@ sub test_partial_rpm_release { '--run-id', 'partial', '--build-timestamp', $epoch, '--skip-build', '--skip-genesis', '--skip-xcat-dep', '--skip-perl', + '--no-verify-repo', '--skip-createrepo', '--skip-tarball', '--genesis-release', $release_root, ); @@ -795,6 +803,7 @@ sub test_failed_build_release { '--run-id', 'empty', '--build-timestamp', $epoch, '--skip-build', '--skip-genesis', '--skip-xcat-dep', '--skip-perl', + '--no-verify-repo', '--skip-createrepo', '--skip-tarball', '--genesis-release', $release_root, '--collect-dir', $collected, @@ -837,6 +846,7 @@ sub test_skip_build_collects_results { '--run-id', 'kept', '--build-timestamp', $epoch, '--skip-build', '--skip-genesis', '--skip-xcat-dep', '--skip-perl', + '--no-verify-repo', '--skip-createrepo', '--skip-tarball', ); @@ -866,6 +876,7 @@ sub test_dry_run_release { '--run-id', 'dry', '--build-timestamp', $epoch, '--skip-build', '--skip-genesis', '--skip-xcat-dep', '--skip-perl', + '--no-verify-repo', '--skip-createrepo', '--skip-tarball', '--genesis-release', $release_root, '--collect-dir', $dependencies, @@ -907,6 +918,7 @@ sub test_rpm_repository_lock { '--repo-dep', $repository, '--target', $target, '--skip-build', '--skip-genesis', '--skip-xcat-dep', '--skip-perl', + '--no-verify-repo', '--skip-createrepo', '--skip-tarball', '--dry-run', ); isnt($status, 0, 'a shared RPM repository cannot have two publishers'); @@ -926,6 +938,7 @@ sub test_rpm_repository_lock { '--repo-dep', $repository, '--target', $target, '--skip-build', '--skip-genesis', '--skip-xcat-dep', '--skip-perl', + '--no-verify-repo', '--skip-createrepo', '--skip-tarball', '--dry-run', '--force-unlock', ); is($forced_status, 0, '--force-unlock recovers an interrupted RPM publication'); @@ -963,6 +976,7 @@ sub test_rpm_signal_cleanup { '--repo-dep', $repository, '--target', 'test+epel-10-x86_64', '--skip-build', '--skip-genesis', '--skip-xcat-dep', '--skip-perl', + '--no-verify-repo', '--skip-createrepo', '--skip-tarball', '--dry-run', ); exit 127; From bbb711431f9ed583c5eda743ccdf430dad005468 Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Wed, 26 Aug 2026 17:23:42 -0300 Subject: [PATCH 53/55] fix(xcat-dep): pin the release-sensitive deps to xCAT's own EVR floors The manifest pinned most packages by %{VERSION} alone, so the gate accepted an rpm with the right Version and a Release older than xCAT will install against -- xCAT states several of these as ">= version-release" Requires (PR #62 review). Converted, in every section, to the floors taken verbatim from xcat-core's specs: goconserver >= 0.3.3-snap202011021058, xnba-undi >= 1.21.1-1, syslinux-xcat >= 6.03-1, ipmitool-xcat >= 1.8.18-4 (xCAT.spec / xCATsn.spec), perl-HTTP-Async >= 0.30-3, perl-Net-HTTPS-NB >= 0.14-3 (xCAT-server.spec). Where two specs disagree the stronger floor is used. Each was checked against the EVR this repository actually builds, using the gate's own comparator, so none of them reds a build that is in fact correct. grub2-xcat is deliberately left on its Version pin. xCAT-server asks for '>= 2.02-0.76.el7.1.snap201905160255', but the grub2-xcat built here -- and shipped by both published channels today -- is 1.0-2, which cannot satisfy it. Encoding that Requires would fail every build over a discrepancy that lives in xcat-core, so it is documented in the manifest header and reported upstream instead. t/mockbuild-all.t now guards the shipped manifest: every release-sensitive package keeps an EVR floor in every section, and grub2-xcat stays the documented exception. Verified the guard fails when a floor is regressed to a bare version. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- packages-manifest.conf | 84 ++++++++++++++++++++++++------------------ t/mockbuild-all.t | 32 ++++++++++++++++ 2 files changed, 80 insertions(+), 36 deletions(-) diff --git a/packages-manifest.conf b/packages-manifest.conf index 955813b..d95ead4 100644 --- a/packages-manifest.conf +++ b/packages-manifest.conf @@ -25,6 +25,18 @@ # >= 2:2.18.0 (Epoch 2), which a 2.17.x genesis would violate. genesis-base carries # Epoch 2, so the epoch in the constraint is enforced too. # +# The pins that xCAT states as release-sensitive Requires are EVR floors taken VERBATIM from +# xcat-core's own specs, so the gate rejects a build whose Version is right but whose Release is +# older than xCAT accepts: goconserver (xCAT.spec), xnba-undi, syslinux-xcat, ipmitool-xcat +# (xCAT.spec / xCATsn.spec), perl-HTTP-Async, perl-Net-HTTPS-NB and perl-IO-Stty +# (xCAT-server.spec). Where two specs disagree the STRONGER floor is used. +# +# grub2-xcat is deliberately NOT converted. xCAT-server.spec asks for +# '>= 2.02-0.76.el7.1.snap201905160255', but the grub2-xcat this repository builds -- and that both +# published channels ship -- is 1.0-2, which cannot satisfy it (1 < 2). Encoding that Requires here +# would fail every build on a discrepancy that lives in xcat-core, not in this manifest, so the pin +# stays on the Version actually produced and the discrepancy is reported upstream instead. +# # mockbuild-all.pl reads this file and, per target, builds ONLY the listed # packages -- a package not listed for a target is not built for it. Any listed # package that fails to build (or builds a mismatched version) fails the whole @@ -45,71 +57,71 @@ [alma+epel-8-x86_64] conserver-xcat=8.2.1 elilo-xcat=3.14 -goconserver=0.3.3 +goconserver=>= 0.3.3-snap202011021058 grub2-xcat=1.0 -ipmitool-xcat=1.8.18 -syslinux-xcat=6.03 -xnba-undi=1.21.1 +ipmitool-xcat=>= 1.8.18-4 +syslinux-xcat=>= 6.03-1 +xnba-undi=>= 1.21.1-1 perl-HTML-Form=6.07 -perl-HTTP-Async=0.30 +perl-HTTP-Async=>= 0.30-3 perl-IO-Stty=>= 0.04-5 -perl-Net-HTTPS-NB=0.14 +perl-Net-HTTPS-NB=>= 0.14-3 xCAT-genesis-base=>= 2:2.18.0 [alma+epel-8-ppc64le] conserver-xcat=8.2.1 elilo-xcat=3.14 -goconserver=0.3.3 +goconserver=>= 0.3.3-snap202011021058 grub2-xcat=1.0 -ipmitool-xcat=1.8.18 -syslinux-xcat=6.03 -xnba-undi=1.21.1 +ipmitool-xcat=>= 1.8.18-4 +syslinux-xcat=>= 6.03-1 +xnba-undi=>= 1.21.1-1 perl-HTML-Form=6.07 -perl-HTTP-Async=0.30 +perl-HTTP-Async=>= 0.30-3 perl-IO-Stty=>= 0.04-5 -perl-Net-HTTPS-NB=0.14 +perl-Net-HTTPS-NB=>= 0.14-3 xCAT-genesis-base=>= 2:2.18.0 [alma+epel-9-x86_64] conserver-xcat=8.2.1 elilo-xcat=3.14 -goconserver=0.3.3 +goconserver=>= 0.3.3-snap202011021058 grub2-xcat=1.0 -ipmitool-xcat=1.8.18 -syslinux-xcat=6.03 -xnba-undi=1.21.1 -perl-HTTP-Async=0.30 +ipmitool-xcat=>= 1.8.18-4 +syslinux-xcat=>= 6.03-1 +xnba-undi=>= 1.21.1-1 +perl-HTTP-Async=>= 0.30-3 perl-IO-Stty=>= 0.04-5 -perl-Net-HTTPS-NB=0.14 +perl-Net-HTTPS-NB=>= 0.14-3 perl-Sys-Virt=11.10.0 xCAT-genesis-base=>= 2:2.18.0 [alma+epel-9-ppc64le] conserver-xcat=8.2.1 elilo-xcat=3.14 -goconserver=0.3.3 +goconserver=>= 0.3.3-snap202011021058 grub2-xcat=1.0 -ipmitool-xcat=1.8.18 -syslinux-xcat=6.03 -xnba-undi=1.21.1 -perl-HTTP-Async=0.30 +ipmitool-xcat=>= 1.8.18-4 +syslinux-xcat=>= 6.03-1 +xnba-undi=>= 1.21.1-1 +perl-HTTP-Async=>= 0.30-3 perl-IO-Stty=>= 0.04-5 -perl-Net-HTTPS-NB=0.14 +perl-Net-HTTPS-NB=>= 0.14-3 perl-Sys-Virt=11.10.0 xCAT-genesis-base=>= 2:2.18.0 [alma+epel-10-x86_64] conserver-xcat=8.2.1 elilo-xcat=3.14 -goconserver=0.3.3 +goconserver=>= 0.3.3-snap202011021058 grub2-xcat=1.0 -ipmitool-xcat=1.8.18 -syslinux-xcat=6.03 -xnba-undi=1.21.1 +ipmitool-xcat=>= 1.8.18-4 +syslinux-xcat=>= 6.03-1 +xnba-undi=>= 1.21.1-1 perl-Crypt-SSLeay=0.72 -perl-HTTP-Async=0.30 +perl-HTTP-Async=>= 0.30-3 perl-IO-Stty=>= 0.04-5 -perl-Net-HTTPS-NB=0.14 +perl-Net-HTTPS-NB=>= 0.14-3 perl-Net-Telnet=3.04 perl-Sys-Virt=11.10.0 xCAT-genesis-base=>= 2:2.18.0 @@ -117,15 +129,15 @@ xCAT-genesis-base=>= 2:2.18.0 [alma+epel-10-ppc64le] conserver-xcat=8.2.1 elilo-xcat=3.14 -goconserver=0.3.3 +goconserver=>= 0.3.3-snap202011021058 grub2-xcat=1.0 -ipmitool-xcat=1.8.18 -syslinux-xcat=6.03 -xnba-undi=1.21.1 +ipmitool-xcat=>= 1.8.18-4 +syslinux-xcat=>= 6.03-1 +xnba-undi=>= 1.21.1-1 perl-Crypt-SSLeay=0.72 -perl-HTTP-Async=0.30 +perl-HTTP-Async=>= 0.30-3 perl-IO-Stty=>= 0.04-5 -perl-Net-HTTPS-NB=0.14 +perl-Net-HTTPS-NB=>= 0.14-3 perl-Net-Telnet=3.04 perl-Sys-Virt=11.10.0 xCAT-genesis-base=>= 2:2.18.0 diff --git a/t/mockbuild-all.t b/t/mockbuild-all.t index 382cf2a..1c5ffdc 100644 --- a/t/mockbuild-all.t +++ b/t/mockbuild-all.t @@ -445,4 +445,36 @@ my $vercmp = sub { 'build_mock_uniqueext: deterministic for a given (run, seq, label)'); } +# ---- the shipped manifest keeps xCAT's release-sensitive floors (PR #62 review) ----------------- +# xCAT states these as ">= version-release" Requires, so a VERSION-only pin lets the gate accept an +# rpm with the right Version and a Release older than xCAT will install against. Guard the shipped +# manifest itself, in every section, so a later edit cannot quietly drop a floor back to a bare +# version. +{ + my $shipped = "$RealBin/../packages-manifest.conf"; + SKIP: { + skip 'packages-manifest.conf not found', 2 unless -f $shipped; + my %m = read_manifest($shipped); + my @evr_pinned = qw(goconserver xnba-undi syslinux-xcat ipmitool-xcat + perl-HTTP-Async perl-Net-HTTPS-NB perl-IO-Stty xCAT-genesis-base); + my @bare; + for my $tgt (sort keys %m) { + for my $pkg (@evr_pinned) { + my $pin = $m{$tgt}{$pkg}; + next unless defined $pin; # not every target lists every package + push @bare, "[$tgt] $pkg=$pin" unless $pin =~ /^\s*(?:>=|>|<=|<|=)\s*\S/; + } + } + is_deeply(\@bare, [], 'every release-sensitive package keeps an EVR floor in every section') + or diag("VERSION-only pin(s):\n " . join("\n ", @bare)); + + # grub2-xcat is the deliberate exception: xCAT-server asks for >= 2.02-0.76.el7.1.snap..., + # which the grub2-xcat this repo builds (1.0-2) cannot satisfy. Encoding that Requires would + # fail every build, so the pin tracks what is actually produced -- see the file's header. + my @grub = grep { defined } map { $m{$_}{'grub2-xcat'} } sort keys %m; + is_deeply([ grep { /^\s*>=/ } @grub ], [], + 'grub2-xcat stays a version pin (its xCAT Requires cannot be met by what is built)'); + } +} + done_testing; From b280fb8b4823874dee704f45b6c58c716a4021eb Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Wed, 26 Aug 2026 18:24:19 -0300 Subject: [PATCH 54/55] feat(xcat-dep): gate the shared Genesis repository on a manifest section of its own Every package in the per-EL cells is gated against packages-manifest.conf, but the OpenEmbedded Genesis release is published into xcat-dep/common, which sits BESIDE those cells and is described by no [] section. Nothing asserted the published shared repository was complete: its packages were checked only as they were copied, against the release checksums, so a repository that lost one afterwards would publish quietly. [common] describes that repository -- all seven architectures, floored at the paired xcat-core version (>= 2.18.0; these carry no Epoch, unlike xCAT-genesis-base). verify_common_repo runs on the STAGE, before the atomic swap, so an incomplete shared repo is never published. Completeness only: the release checksums cover the bytes and the deploy asserts every signature. [common] is not a build target, so the manifest now has two kinds of section. No code iterates sections blindly, but t/mockbuild-all.t did, and asserted conserver-xcat in every one; it now selects target-named sections and asserts the shared-repo section is NOT treated as a target. t/common-repo-gate.t drives the real publish path and asserts on the repository left behind. Verified it fails without the gate, and that dropping an architecture from [common] is caught. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- mockbuild-all.pl | 42 +++++++++++++ packages-manifest.conf | 19 ++++++ t/common-repo-gate.t | 131 +++++++++++++++++++++++++++++++++++++++++ t/mockbuild-all.t | 6 +- 4 files changed, 197 insertions(+), 1 deletion(-) create mode 100644 t/common-repo-gate.t diff --git a/mockbuild-all.pl b/mockbuild-all.pl index 6bdbc84..85a481c 100755 --- a/mockbuild-all.pl +++ b/mockbuild-all.pl @@ -952,12 +952,54 @@ sub publish_genesis_common_repo { verify_genesis_release_packages('rpm', $COMMON_STAGE); sign_and_index_repo($COMMON_STAGE); write_common_repo_metadata($COMMON_STAGE); + # Gate the STAGE, so an incomplete shared repo is never swapped into place. Completeness only: + # the packages were verified against the release checksums as they were copied, and the deploy + # asserts every rpm's signature, but until now nothing checked that the repository being + # published actually carries the whole architecture set the manifest says it must. + verify_common_repo($COMMON_STAGE) unless $no_verify_repo; chmod(0755, $COMMON_STAGE) or die "Cannot make $COMMON_STAGE traversable: $!\n"; replace_common_repository($COMMON_STAGE, $dest); print "Published common Genesis repository: $published rpms\n"; } +#-------------------------------------------------------------------------------- + +=head3 verify_common_repo + + Assert the shared OpenEmbedded Genesis repository carries every package the manifest's [common] + section requires, at a version satisfying its pin. [common] is not a build target: it describes + the one repository published beside the per-EL cells, which no [] section covers. + + Arguments: + $dir - the repository to check (the staging directory, before it is swapped into place) + Returns: + 1, or dies listing every problem + +=cut + +#-------------------------------------------------------------------------------- +sub verify_common_repo { + my ($dir) = @_; + my $manifest = "$repo_root/packages-manifest.conf"; + my %MAN = read_manifest($manifest); + my %req = %{ $MAN{common} // {} }; + die "FATAL: no [common] section in $manifest -- cannot verify the shared Genesis repository\n" + if !%req; + + my @names = sort keys %req; + my %present = repo_present_versions($dir, \@names); + my %present_evr = map { $_ => rpm_evr($dir, $_) } @names; + my @problems = verify_repo_packages(\%req, \%present, \%present_evr, \&rpm_vercmp_segment); + if (@problems) { + print " - $_\n" for @problems; + die "FATAL: shared Genesis repo INCOMPLETE at $dir (" . scalar(@problems) . " problem(s))\n"; + } + print "[verify-repo] common complete: " . scalar(@names) + . " packages present + EVR-satisfied in $dir\n"; + return 1; +} + sub replace_common_repository { my ($staged, $destination) = @_; my $backup = "$repo_dep/.common.previous.$$"; diff --git a/packages-manifest.conf b/packages-manifest.conf index d95ead4..28dabb0 100644 --- a/packages-manifest.conf +++ b/packages-manifest.conf @@ -141,3 +141,22 @@ perl-Net-HTTPS-NB=>= 0.14-3 perl-Net-Telnet=3.04 perl-Sys-Virt=11.10.0 xCAT-genesis-base=>= 2:2.18.0 + +# [common] is NOT a build target. It describes the SHARED repository the OpenEmbedded Genesis +# release is published into (/common), which lives beside the per-EL cells and is +# therefore invisible to every [] section above. Without it nothing asserted the published +# shared repo was COMPLETE: its packages were checked only as they were copied, against the release +# checksums, so a repository that lost one afterwards would publish quietly. +# +# The floor tracks the paired xcat-core, like xCAT-genesis-base: these are built FROM xcat-core, so +# the version walks with it (2.18.x, 2.19.x, ...). Epoch 0 -- unlike genesis-base, these packages +# carry no Epoch. Listing every architecture is the point: the release is only useful if a +# management node can serve an image for any of them. +[common] +xCAT-genesis-openembedded-x86=>= 2.18.0 +xCAT-genesis-openembedded-x86_64=>= 2.18.0 +xCAT-genesis-openembedded-ppc64=>= 2.18.0 +xCAT-genesis-openembedded-ppc64le=>= 2.18.0 +xCAT-genesis-openembedded-armv7hf=>= 2.18.0 +xCAT-genesis-openembedded-aarch64=>= 2.18.0 +xCAT-genesis-openembedded-riscv64=>= 2.18.0 diff --git a/t/common-repo-gate.t b/t/common-repo-gate.t new file mode 100644 index 0000000..9658e99 --- /dev/null +++ b/t/common-repo-gate.t @@ -0,0 +1,131 @@ +#!/usr/bin/perl +# The shared OpenEmbedded Genesis repository (xcat-dep/common) is published outside the per-target +# cells, so the per-target manifest sections never described it and nothing asserted it was COMPLETE +# once published. Its packages were only checked as they were copied, against the release checksums. +# +# This drives the real mockbuild-all.pl publish path and asserts on the repository it leaves behind: +# a complete release publishes and is gated against the manifest's [common] section, and a release +# missing one architecture is refused rather than published. +use strict; +use warnings; +use Test::More; +use FindBin qw($RealBin); +use File::Temp qw(tempdir); +use File::Path qw(make_path); +use File::Copy qw(copy); +use lib "$RealBin/.."; +use MockBuildUtils qw(read_manifest); + +my $SCRIPT = "$RealBin/../mockbuild-all.pl"; +my $RELEASE = '/opt/xcat-ci-shared/builds/genesis-openembedded-initial-20260825/release'; +plan skip_all => 'mockbuild-all.pl not found' unless -f $SCRIPT; +plan skip_all => 'no Genesis release fixture' unless -d "$RELEASE/rpm"; +# root, like every other test that drives mockbuild-all.pl: the script refuses to run otherwise, +# and the CI builder (XCAT_GENESIS_CI) is root. +plan skip_all => 'rpm tooling and a root Linux builder required' + unless $^O eq 'linux' + && $> == 0 + && !system('sh', '-c', 'command -v rpm >/dev/null 2>&1') + && !system('sh', '-c', 'command -v createrepo_c >/dev/null 2>&1') + && !system('sh', '-c', 'command -v rpmbuild >/dev/null 2>&1'); + +my $tmp = tempdir(CLEANUP => 1); +my $target = 'alma+epel-10-' . do { my $m = `uname -m`; chomp $m; $m }; + +# The shipped manifest must describe the shared repo, else nothing can gate it. +{ + my %m = read_manifest("$RealBin/../packages-manifest.conf"); + ok($m{common} && %{ $m{common} }, 'the shipped manifest has a [common] section'); + is(scalar(keys %{ $m{common} // {} }), 7, + '... naming every architecture the release must carry'); +} + +# fixture_rpm: a minimal noarch rpm, built once, standing in for a compiled dep. +my $FIXTURE; +sub fixture_rpm { + return $FIXTURE if $FIXTURE; + my $top = "$tmp/rpmbuild"; + make_path("$top/SPECS"); + open my $fh, '>', "$top/SPECS/fixture.spec" or die $!; + print $fh <<'SPEC'; +Name: ipmitool-xcat +Version: 1.8.18 +Release: 4 +Summary: fixture +License: EPL +BuildArch: noarch +%description +fixture package standing in for a compiled dependency +%install +mkdir -p %{buildroot}/usr/share/ipmitool-xcat +%files +/usr/share/ipmitool-xcat +SPEC + close $fh; + system('rpmbuild', '--quiet', '-bb', '--define', "_topdir $top", "$top/SPECS/fixture.spec") == 0 + or die "cannot build the fixture rpm\n"; + ($FIXTURE) = glob("$top/RPMS/noarch/ipmitool-xcat-1.8.18-4.noarch.rpm"); + die "fixture rpm not produced\n" unless $FIXTURE && -f $FIXTURE; + return $FIXTURE; +} + +# run_publish($release_dir) -> ($exit, $output, $common_dir) +sub run_publish { + my ($release, $tag) = @_; + my $out = "$tmp/$tag"; + make_path("$out/root", "$out/collect"); + # Something to collect, so the run gets past the "built nothing" guard. It must NOT be an + # OpenEmbedded package: collect_rpms drops those when --genesis-release is given (they come from + # the release, not from the build), so collecting one would leave the run with nothing. + copy(fixture_rpm(), "$out/collect/") or die $!; + open my $fh, '>', "$out/root/packages-manifest.conf" or die $!; + # the cell carries exactly the fixture dep, so the per-target gate runs for real too + print $fh "[$target]\nipmitool-xcat=1.8.18\n"; + # the shared repo's own section, copied from the shipped manifest so the test uses the real one + my %m = read_manifest("$RealBin/../packages-manifest.conf"); + print $fh "\n[common]\n"; + print $fh "$_=$m{common}{$_}\n" for sort keys %{ $m{common} // {} }; + close $fh; + my $cmd = join(' ', map { my $x = $_; $x =~ s/'/'"'"'/g; "'$x'" } + ($^X, $SCRIPT, '--repo-root', "$out/root", '--output', "$out/build", + '--repo-dep', "$out/repo", '--target', $target, '--run-id', $tag, + '--build-timestamp', '1787672536', + '--skip-build', '--skip-genesis', '--skip-xcat-dep', '--skip-perl', + '--skip-tarball', + '--collect-dir', "$out/collect", '--genesis-release', $release)) . ' 2>&1'; + my $log = `$cmd`; + return ($? >> 8, $log, "$out/repo/common"); +} + +# ---- a complete release publishes, and says it was gated ----------------------------------------- +{ + my ($rc, $out, $common) = run_publish($RELEASE, 'full'); + is($rc, 0, 'a complete release publishes') or diag($out); + is(scalar(grep { !/\.src\.rpm$/ } glob("$common/*.rpm")), 7, + 'the published shared repo carries every architecture'); + like($out, qr/\[verify-repo\] common complete/, 'the shared repo is gated against [common]'); +} + +# ---- an incomplete release is refused, and publishes nothing -------------------------------------- +{ + my $partial = "$tmp/partial-release"; + make_path("$partial/rpm", "$partial/srpm"); + for my $f (glob("$RELEASE/rpm/*.rpm"), glob("$RELEASE/srpm/*.rpm")) { + next if $f =~ /riscv64/; # drop one architecture + my ($sub) = $f =~ m{/(rpm|srpm)/[^/]+$}; + copy($f, "$partial/$sub/") or die $!; + } + copy("$RELEASE/release.manifest", $partial) or die $!; + # SHA256SUMS without the dropped arch, so the release itself still self-describes consistently + open my $in, '<', "$RELEASE/SHA256SUMS" or die $!; + open my $o, '>', "$partial/SHA256SUMS" or die $!; + while (<$in>) { print {$o} $_ unless /riscv64/ } + close $in; close $o; + + my ($rc, $out, $common) = run_publish($partial, 'partial'); + isnt($rc, 0, 'a release missing an architecture is refused'); + ok(!-d $common || !glob("$common/*.rpm"), + '... and nothing is published into the shared repository'); +} + +done_testing(); diff --git a/t/mockbuild-all.t b/t/mockbuild-all.t index 1c5ffdc..e7579aa 100644 --- a/t/mockbuild-all.t +++ b/t/mockbuild-all.t @@ -237,8 +237,12 @@ is(rpm_release(tempdir(CLEANUP => 1), 'nonexistent-pkg'), undef, 'rpm_release is # doc and the manifest can never silently drift apart again. { my %m = read_manifest("$RealBin/../packages-manifest.conf"); - my @targets = sort keys %m; + # Not every section is a build target: [common] describes the SHARED repository the + # OpenEmbedded Genesis release is published into, which no builder produces. Target sections are + # the ones named after a mock config (+epel--, opensuse-leap--). + my @targets = grep { /^[a-z0-9.+-]+-\d+(?:\.\d+)?-[a-z0-9_]+$/ } sort keys %m; cmp_ok(scalar(@targets), '>=', 1, 'packages-manifest.conf has at least one target section'); + ok(!grep({ $_ eq 'common' } @targets), 'the shared-repo section is not treated as a build target'); my @missing = grep { !exists $m{$_}{'conserver-xcat'} } @targets; is_deeply(\@missing, [], 'conserver-xcat is present in every manifest target section') or diag("missing conserver-xcat in: @missing"); From 1e9ec56bdc7f17a25619bcbeec32d4bdf7a97bed Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Thu, 27 Aug 2026 15:29:58 -0300 Subject: [PATCH 55/55] feat(xcat-dep): --install-deps makes a build host able to run the script Two CD runs died at compile time inside XCAT::BuildUtils because a builder was missing a Perl module the script loads: perl-File-Slurper on xcat-master-ub and perl-IPC-Cmd on xcat-master-ppc. Both surfaced as "Can't locate ... in @INC" in the middle of a build, and both were fixed by hand -- so the next unprovisioned host fails the same way, and BUILD.md's install line can drift from what the code actually requires. --install-deps installs this host's prerequisites and exits: the toolchain plus the modules, through dnf or zypper as the host's ID dictates. It then LOADS each module and fails naming any that is still missing, rather than trusting the package manager's exit code -- a package that installs cleanly but leaves the module unusable is exactly the failure this exists to prevent. The list and the command are pure functions in MockBuildUtils, so the decision is unit-tested (package sets per family, the right installer non-interactively, and the probe reporting only what genuinely cannot be loaded); the side effect stays in the caller. Run on xcat-master and xcat-master-ppc: both report every module present. xcat-master-suse is unreachable and still needs it. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- BUILD.md | 12 +++++++++++- MockBuildUtils.pm | 42 ++++++++++++++++++++++++++++++++++++++++++ mockbuild-all.pl | 30 +++++++++++++++++++++++++++++- t/mockbuild-all.t | 32 +++++++++++++++++++++++++++++++- 4 files changed, 113 insertions(+), 3 deletions(-) diff --git a/BUILD.md b/BUILD.md index 2c21ad4..1598239 100644 --- a/BUILD.md +++ b/BUILD.md @@ -142,7 +142,17 @@ Use these flags to skip specific operations: - `mockbuild-all.pl` and package sources present under ``. - xCAT sources present under ``. -Install baseline tooling: +Install baseline tooling — let the script do it, so the list cannot drift from what it loads: + +```bash +./mockbuild-all.pl --install-deps # as root, once per build host +``` + +It installs the toolchain and the Perl modules for this host's package manager (dnf on EL, zypper +on SUSE), then **loads** each module and fails if one is still missing. That last step is the point: +a missing module surfaces otherwise as a compile-time abort inside `XCAT::BuildUtils`, in the middle +of a CD run, which is how `perl-File-Slurper` and `perl-IPC-Cmd` each took a pipeline down. The +equivalent by hand: ```bash dnf -y install perl perl-File-Slurper perl-IPC-Cmd \ diff --git a/MockBuildUtils.pm b/MockBuildUtils.pm index 07d0876..aa9065b 100644 --- a/MockBuildUtils.pm +++ b/MockBuildUtils.pm @@ -13,6 +13,7 @@ use Sys::Hostname; use Digest::MD5 qw(md5_hex); our @EXPORT_OK = qw( + install_deps_packages install_deps_command missing_perl_modules sh_quote print_step version_matches required_pkgs have_rpm read_manifest verify_repo_packages verify_repo_signature verify_rpm_signatures @@ -22,6 +23,47 @@ our @EXPORT_OK = qw( build_mock_uniqueext ); +# install_deps_packages($os_id): the host packages mockbuild-all.pl needs to run at all, for the +# given /etc/os-release ID. Kept as data, beside the code that needs them, because the failure mode +# is a build host provisioned by hand: xcat-master-ub had no perl-File-Slurper and xcat-master-ppc no +# perl-IPC-Cmd, and each surfaced as a compile-time abort in the middle of a CD run. +sub install_deps_packages { + my ($os_id) = @_; + $os_id = '' unless defined $os_id; + # The perl modules are what actually break a run; the rest is the toolchain the script drives. + return qw(perl perl-File-Slurper perl-IPC-Cmd perl-Parallel-ForkManager perl-Digest-SHA + mock createrepo_c tar findutils rpm rpm-build rpm-sign rpmdevtools gnupg2 wget git) + if $os_id =~ /^(?:opensuse|sles|sled)/; + return qw(perl perl-File-Slurper perl-IPC-Cmd perl-Parallel-ForkManager perl-Digest-SHA + mock createrepo_c tar findutils rpm rpm-build rpm-sign rpmdevtools + dnf-plugins-core gnupg2 wget git); +} + +# install_deps_command($os_id): the argv that installs them, non-interactively. +sub install_deps_command { + my ($os_id) = @_; + $os_id = '' unless defined $os_id; + my @pkgs = install_deps_packages($os_id); + return ('zypper', '--non-interactive', 'install', '--no-recommends', @pkgs) + if $os_id =~ /^(?:opensuse|sles|sled)/; + return ('dnf', '-y', 'install', @pkgs); +} + +# missing_perl_modules(@modules): those that cannot be loaded, in order. The point of --install-deps +# is that the run AFTER it cannot die on a missing module, so the modules are proven by loading +# them, not by trusting the package manager's exit code. +sub missing_perl_modules { + my (@modules) = @_; + my @missing; + for my $m (@modules) { + my $file = $m; + $file =~ s{::}{/}g; + $file .= '.pm'; + eval { require $file; 1 } or push @missing, $m; + } + return @missing; +} + # sh_quote: single-quote a string for safe use in a shell command. sub sh_quote { my ($s) = @_; diff --git a/mockbuild-all.pl b/mockbuild-all.pl index 85a481c..306175d 100755 --- a/mockbuild-all.pl +++ b/mockbuild-all.pl @@ -16,6 +16,7 @@ use POSIX qw(strftime); use FindBin qw($RealBin); use lib $RealBin, "$RealBin/lib"; use MockBuildUtils qw(sh_quote print_step version_matches required_pkgs + install_deps_packages install_deps_command missing_perl_modules read_manifest verify_repo_packages verify_repo_signature verify_rpm_signatures rpm_version rpm_release rpm_sigmd5 restamp_release_line cross_copy_genesis finalize_xcat_dep bump_dep_release_suffix @@ -51,7 +52,7 @@ use XCAT::GenesisRelease qw( # otherwise warn loudly and continue unisolated. MOCKBUILD_ALL_MOUNTNS guards against a re-exec loop. # Build-free modes (--verify-repo, --finalize-xcat-dep) run no mock and are documented no-root, so they # skip the re-exec entirely -- no cgroup exposure, and no spurious non-root warning. -my $mountns_build_free = grep { /^--(?:verify-repo|finalize-xcat-dep)(?:=|$)/ } @ARGV; +my $mountns_build_free = grep { /^--(?:verify-repo|finalize-xcat-dep|install-deps)(?:=|$)/ } @ARGV; unless ($ENV{MOCKBUILD_ALL_MOUNTNS} || $mountns_build_free) { if ($> != 0) { warn "WARN: not root -- skipping mount-namespace isolation (host-cgroup propagation guard); " @@ -101,6 +102,7 @@ my $GOCONSERVER_REF = '6166fe5ec1c5b3c20475e322a9f0e8e93c87e45f'; my $skip_build = 0; my $skip_xcat_dep = 0; my $skip_perl = 0; +my $install_deps = 0; my $skip_genesis = 0; my $skip_createrepo = 0; my $skip_tarball = 0; @@ -161,6 +163,7 @@ GetOptions( 'skip-build!' => \$skip_build, 'skip-xcat-dep!' => \$skip_xcat_dep, 'skip-perl!' => \$skip_perl, + 'install-deps!' => \$install_deps, 'skip-genesis!' => \$skip_genesis, 'skip-createrepo!' => \$skip_createrepo, 'skip-tarball!' => \$skip_tarball, @@ -321,6 +324,28 @@ die "Could not resolve ID from /etc/os-release\n" if $os_id eq ''; die "Could not resolve major release from VERSION_ID='$version_id' in /etc/os-release\n" if !defined($rel) || $rel eq ''; +# --install-deps: make THIS host able to run the script, then exit. It has to come before the +# require_command checks below -- those are the very things it installs, and a host that lacks them +# would die here with no way to fix itself. Run once per build host, as root. +# +# The perl modules are re-checked by LOADING them afterwards rather than trusting the package +# manager: a module that is still missing is exactly the failure this mode exists to prevent, and it +# aborted CD runs mid-build twice (perl-File-Slurper on xcat-master-ub, perl-IPC-Cmd on +# xcat-master-ppc), each time as a compile-time error inside XCAT::BuildUtils. +if ($install_deps) { + die "--install-deps must run as root (uid=$>)\n" if $> != 0; + my @cmd = install_deps_command($os_id); + print_step("Install build prerequisites ($os_id)"); + print " " . join(' ', @cmd) . "\n"; + run_command(@cmd); + my @modules = qw(File::Slurper IPC::Cmd Parallel::ForkManager Digest::SHA); + my @missing = missing_perl_modules(@modules); + die "FATAL: still missing after install: " . join(', ', @missing) . "\n" if @missing; + print " perl modules present: " . join(', ', @modules) . "\n"; + print " host is ready\n"; + exit 0; +} + for my $bin (qw(perl uname createrepo_c tar find rpm)) { require_command($bin); } @@ -1244,6 +1269,9 @@ Options: --skip-build Skip all build steps and only collect/create repo/tarballs --skip-xcat-dep Skip xcat-dep mockbuild.pl package steps --skip-perl Skip perl package build step + --install-deps Install this host's build prerequisites (package manager + the perl + modules the script loads), verify each module now loads, then exit. + Run once per build host, as root. Use alone. --skip-genesis Skip the existing per-EL Genesis image build --skip-createrepo Skip createrepo --skip-tarball Skip binary/SRPM tarball creation diff --git a/t/mockbuild-all.t b/t/mockbuild-all.t index e7579aa..7511ca9 100644 --- a/t/mockbuild-all.t +++ b/t/mockbuild-all.t @@ -10,7 +10,8 @@ use lib "$RealBin/.."; use File::Temp qw(tempdir); use File::Path qw(make_path); use File::Basename qw(basename); -use MockBuildUtils qw(required_pkgs version_matches rpm_sigmd5 rpm_version rpm_release rpm_is_signed +use MockBuildUtils qw(install_deps_packages install_deps_command missing_perl_modules + required_pkgs version_matches rpm_sigmd5 rpm_version rpm_release rpm_is_signed restamp_release_line cross_copy_genesis finalize_xcat_dep read_manifest verify_repo_packages verify_repo_signature verify_rpm_signatures parse_evr evr_constraint_ok parse_pin rpmkeys_checksig_problem @@ -481,4 +482,33 @@ my $vercmp = sub { } } +# ---- --install-deps: the host prerequisites (the modules are what actually break a run) ---------- +# Two CD runs died at compile time inside XCAT::BuildUtils because a builder lacked a module +# (perl-File-Slurper on one host, perl-IPC-Cmd on another), so the list must carry every module the +# script loads, and the mode must PROVE them by loading rather than trusting the package manager. +{ + my @el = install_deps_packages('almalinux'); + for my $need (qw(perl-File-Slurper perl-IPC-Cmd perl-Parallel-ForkManager mock createrepo_c)) { + ok(scalar(grep { $_ eq $need } @el), "EL prerequisites include $need"); + } + my @cmd = install_deps_command('almalinux'); + is($cmd[0], 'dnf', 'EL installs with dnf'); + ok(scalar(grep { $_ eq '-y' } @cmd), '... non-interactively'); + + my @suse = install_deps_command('opensuse-leap'); + is($suse[0], 'zypper', 'SUSE installs with zypper'); + ok(scalar(grep { $_ eq '--non-interactive' } @suse), '... non-interactively'); + is_deeply([ grep { /^perl-/ } install_deps_packages('opensuse-leap') ], + [ grep { /^perl-/ } @el ], + 'both families install the same perl modules'); + + # the probe reports what cannot be loaded, and nothing else + is_deeply([ missing_perl_modules('Digest::SHA') ], [], + 'missing_perl_modules: a loadable module is not reported'); + is_deeply([ missing_perl_modules('No::Such::Module::Here') ], ['No::Such::Module::Here'], + 'missing_perl_modules: an absent module is reported'); + is_deeply([ missing_perl_modules('Digest::SHA', 'No::Such::Module::Here') ], + ['No::Such::Module::Here'], '... and only the absent one, from a mixed list'); +} + done_testing;