diff --git a/BuildUtils.pm b/BuildUtils.pm index 85293b4..3f753bb 100644 --- a/BuildUtils.pm +++ b/BuildUtils.pm @@ -26,7 +26,7 @@ our @EXPORT_OK = qw( version_matches required_pkgs read_manifest standard_options codename_to_version version_to_codename known_codenames chroot_name chroot_sources_list - deb_snap_version rewrite_changelog_top control_field genesis_deb_control + control_field genesis_deb_control deb_field deb_version deb_upstream_version deb_hash cross_copy_genesis_deb build_deb_in_chroot ); @@ -164,46 +164,6 @@ sub chroot_sources_list { ); } -# --------------------------------------------------------------------------------------------------- -# Debian version / changelog helpers (out-of-tree stamping: NEVER edit a tracked debian/changelog). -# --------------------------------------------------------------------------------------------------- - -# deb_snap_version: derive the CD snapshot Debian version from a base upstream version, a build -# timestamp (YYYYMMDDHHMM), and an optional monotonic build number, e.g. -# deb_snap_version('0.3.3', '202608101400', 57) -> '0.3.3-snap202608101400.57' -# deb_snap_version('0.3.3', '202608101400') -> '0.3.3-snap202608101400' -# Idempotent / self-healing: a $base that ALREADY carries one or more '-snap[.n]' stamps (from a -# reused build tree or an earlier corrupted run) has them stripped before the fresh stamp is applied, -# so a re-run REPLACES the stamp instead of stacking a second one. Mirrors the intent of the EL -# MockBuildUtils::restamp_release_line, in Debian version grammar (the stamp is the debian_revision). -sub deb_snap_version { - my ($base, $ts, $build_number) = @_; - die "deb_snap_version: base version required\n" if !defined $base || $base eq ''; - die "deb_snap_version: timestamp required\n" if !defined $ts || $ts !~ /^\d{12}$/; - (my $clean = $base) =~ s/-snap\d{12}(?:\.\d+)?(?:-snap\d{12}(?:\.\d+)?)*\z//; - my $rev = "snap$ts"; - $rev .= ".$build_number" if defined $build_number && $build_number ne ''; - return "$clean-$rev"; -} - -# rewrite_changelog_top: return $changelog_text with the TOP (newest) entry rewritten to $new_version -# and, when $maint/$date are given, its trailer line set to "-- $maint $date". Pure string function -# (no file I/O) so it is unit-testable and so the caller can apply it to a COPY of debian/changelog in -# an out-of-tree build dir — the tracked changelog in the checkout is never touched. A Debian -# changelog's first line is "pkg (version) dist; urgency=..."; only the first "(version)" and the -# first "-- ... date" trailer are changed. -sub rewrite_changelog_top { - my ($text, $new_version, $maint, $date) = @_; - return $text unless defined $text && defined $new_version; - # Rewrite the version in the first header line only. - $text =~ s/^(\S[^\n]*?\()[^)]*(\)[^\n]*)/$1$new_version$2/; - # Rewrite the first trailer line if maintainer + date supplied. - if (defined $maint && defined $date) { - $text =~ s/^ --[^\n]*/ -- $maint $date/m; - } - return $text; -} - # --------------------------------------------------------------------------------------------------- # Debian control-metadata helpers (concern #2: preserve the maintained packaging's semantics). # --------------------------------------------------------------------------------------------------- diff --git a/build-debs-all b/build-debs-all deleted file mode 100755 index afaffe6..0000000 --- a/build-debs-all +++ /dev/null @@ -1,101 +0,0 @@ -#!/bin/bash -########### -# -# This script call makedeb and create the deb packages -# for xCAT -# -# Author: Arif Ali -# -# Original Work taken from the xcat-core repo done by Leonardo -# -# Input: -# -# $1 is the xCAT_genesis_build directory path -# $2 is the xCAT_genesis_base rpm path -# -# the build target path is ../../debs, this path can not changed, because the build-ubuntu script will use this fix path -# -############ -function pingusage() -{ - printf "Usage: [BUILDPKGS=\"pkgname1 pkgname2 ...\"] %s ] \n" $(basename $0) >&2 -} - -function makedeb { - SRC_ROOT=$1 - - # - # Make DEBs - # - # build perl-xCAT - deps are libsoap-lite-perl, libdigest-sha1-perl, libdbi-perl - # - - find $SRC_ROOT -maxdepth 2 -name make_deb.sh -type f | while read DEBIAN_SCRIPT - do - DIR=`echo ${DEBIAN_SCRIPT} | sed -e 's/[/]make_deb.sh$//'` - cd ${DIR} - chmod a+x make_deb.sh - ./make_deb.sh - cd - - RC=$? - if [ ${RC} -gt 0 ] - then - echo "Warning: ${DIR} failed exit code ${RC}" - fi - done -} - -#check parameters -if [ $# -lt 2 ];then - pingusage - exit 2 -fi - -old_pwd=`pwd` -curdir=`dirname $0` -#did some prepare for the xcat-genesis -echo "prepare building gensis_base_amd64 as a special case" - -#copy the debian_dir and rpm for xcat-genesia-base to the tem directory -mkdir -p $curdir/genesis_tmp/genesis-base-amd64 -cp -rL ${1}/debian $curdir/genesis_tmp/genesis-base-amd64 -cp -L ${1}/LICENSE.html $curdir/genesis_tmp/genesis-base-amd64 -cp ${2} $curdir/genesis_tmp/genesis-base-amd64 -cd $curdir -cat << __EOF__ > genesis_tmp/make_deb.sh - cd genesis-base-amd64/ - rpm2cpio *.rpm | cpio -id - rm -f *.rpm - dpkg-buildpackage -uc -us - cd - -__EOF__ - -#update to the loatest code -# svn --quiet update -packages="" -if [ $BUILDPKGS ];then - packages=$BUILDPKGS -else - packages=`find . -maxdepth 2 -name make_deb.sh | cut -d/ -f 2 | xargs` -fi -#fix 'all warnings being treated as errors' -export NO_WERROR=1 - -# build all debian packages -#for file in `echo $packages` -for file in `echo $packages` -do - makedeb $file $PKG_LOCATION "$BUILD_STRING" $VERSION -done - -if [ ! -d ../../debs ]; then - mkdir -p ../../debs/ -fi - -for file in `echo $packages` -do - mv $file/*.deb ../../debs/ -done - -rm -rf genesis_tmp -exit 0 diff --git a/build.sh b/build.sh deleted file mode 100755 index de6927f..0000000 --- a/build.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -pkgname=$1 -cur_path=$(dirname "$0") -if [ "$pkgname" ]; then - $cur_path/$pkgname/build.sh - exit $? -else - # TODO: if not specify, build all packages for xcat-dep - echo "Please specify package want to build" - exit 1 -fi diff --git a/conserver/sbuild.pl b/conserver/sbuild.pl index d94bc8d..83e60fb 100755 --- a/conserver/sbuild.pl +++ b/conserver/sbuild.pl @@ -18,6 +18,9 @@ my $pkg_dir = abs_path($RealBin); my $pkg = basename($pkg_dir); my ($codename, $arch, $chroot, $result_dir, $log_dir) = ('', '', '', '', ''); my ($build_timestamp, $build_number, $skip_install) = (undef, undef, 0); +# --log-dir / --build-number / --skip-install are accepted for CLI-compat with sbuild-all.pl (which +# passes them uniformly to every per-package builder) but are intentionally UNUSED here: sbuild-all +# does its own per-package logging and there is no deb install-smoke. They are parsed and ignored. GetOptions( 'codename=s' => \$codename, 'arch=s' => \$arch, 'chroot=s' => \$chroot, 'result-dir=s' => \$result_dir, 'log-dir=s' => \$log_dir, diff --git a/elilo/README b/elilo/README index 69c8bba..9489d90 100644 --- a/elilo/README +++ b/elilo/README @@ -15,10 +15,13 @@ To build elilo-xcat RPM: To build elilo-xcat DEB: ------------------------ -1. Login to Ubuntu x86 node -2. git clone https://github.com/xcat2/xcat-dep -3. cd xcat-dep/elilo -4. apt install quilt -5. apt install ./gnu-efi_3.0v-5_amd64.deb -6. ./make_deb.sh -7. Verify contents with dpkg -c +The Ubuntu/Debian build is now driven by the top-level sbuild-all.pl orchestrator, which builds +each package inside the matching --sbuild chroot from its maintained debian/ +packaging (build-deps such as quilt are installed inside the chroot automatically). The former +./make_deb.sh has been removed. + +1. Login to an Ubuntu x86 build host +2. git clone https://github.com/xcat2/xcat-dep; cd xcat-dep +3. Build every package for a codename: ./sbuild-all.pl --arch amd64 --dists noble + or this package alone: ./elilo/sbuild.pl --codename noble --arch amd64 --result-dir +4. Verify contents with dpkg -c diff --git a/elilo/sbuild.pl b/elilo/sbuild.pl index a3ebe1b..e19c558 100755 --- a/elilo/sbuild.pl +++ b/elilo/sbuild.pl @@ -18,6 +18,9 @@ my $pkg_dir = abs_path($RealBin); my $pkg = basename($pkg_dir); my ($codename, $arch, $chroot, $result_dir, $log_dir) = ('', '', '', '', ''); my ($build_timestamp, $build_number, $skip_install) = (undef, undef, 0); +# --log-dir / --build-number / --skip-install are accepted for CLI-compat with sbuild-all.pl (which +# passes them uniformly to every per-package builder) but are intentionally UNUSED here: sbuild-all +# does its own per-package logging and there is no deb install-smoke. They are parsed and ignored. GetOptions( 'codename=s' => \$codename, 'arch=s' => \$arch, 'chroot=s' => \$chroot, 'result-dir=s' => \$result_dir, 'log-dir=s' => \$log_dir, diff --git a/goconserver/sbuild.pl b/goconserver/sbuild.pl index e2c0b9b..66c8e25 100755 --- a/goconserver/sbuild.pl +++ b/goconserver/sbuild.pl @@ -18,6 +18,9 @@ my $pkg_dir = abs_path($RealBin); my $pkg = basename($pkg_dir); my ($codename, $arch, $chroot, $result_dir, $log_dir) = ('', '', '', '', ''); my ($build_timestamp, $build_number, $skip_install) = (undef, undef, 0); +# --log-dir / --build-number / --skip-install are accepted for CLI-compat with sbuild-all.pl (which +# passes them uniformly to every per-package builder) but are intentionally UNUSED here: sbuild-all +# does its own per-package logging and there is no deb install-smoke. They are parsed and ignored. GetOptions( 'codename=s' => \$codename, 'arch=s' => \$arch, 'chroot=s' => \$chroot, 'result-dir=s' => \$result_dir, 'log-dir=s' => \$log_dir, @@ -77,10 +80,17 @@ cp -rL debian "$gc/debian" cd "$gc" export GOPATH="$PWD/.gopath" GOCACHE="$PWD/.gocache" GOMODCACHE="$PWD/.gomodcache" CGO_ENABLED=0 -go mod init github.com/xcat2/goconserver +# Guard `go mod init`: the pinned upstream tree may already carry a go.mod at this SHA, and re-running +# `go mod init` on an existing module aborts (fatal under `set -e`). Only initialize when absent. +[ -f go.mod ] || go mod init github.com/xcat2/goconserver # kr/pty is abandoned and its pty.Start sets Ctty in a way Go >=1.15 rejects; creack/pty is the # maintained, API-compatible fork that fixes it. go mod edit -replace github.com/kr/pty=github.com/creack/pty@v1.1.21 +# TODO(reproducibility): commit go.sum for the pinned SHA (6166fe5ec1c5b3c20475e322a9f0e8e93c87e45f) +# and build with `go mod download`/module verification instead of `go mod tidy`. The EL branch pins +# deps via a committed go.sum (built from a goconserver/gomod/ tree); this Ubuntu branch has no such +# committed go.mod/go.sum, and producing one soundly requires a Go build with network access, so +# `go mod tidy` (which resolves module versions from the network at build time) stays for now. go mod tidy # stamp the maintained debian/ to the snapshot version, OUT-OF-TREE (this is the cloned copy) diff --git a/grub2-xcat/sbuild.pl b/grub2-xcat/sbuild.pl index 9cfddcb..d38bdcc 100755 --- a/grub2-xcat/sbuild.pl +++ b/grub2-xcat/sbuild.pl @@ -18,6 +18,9 @@ my $pkg_dir = abs_path($RealBin); my $pkg = basename($pkg_dir); my ($codename, $arch, $chroot, $result_dir, $log_dir) = ('', '', '', '', ''); my ($build_timestamp, $build_number, $skip_install) = (undef, undef, 0); +# --log-dir / --build-number / --skip-install are accepted for CLI-compat with sbuild-all.pl (which +# passes them uniformly to every per-package builder) but are intentionally UNUSED here: sbuild-all +# does its own per-package logging and there is no deb install-smoke. They are parsed and ignored. GetOptions( 'codename=s' => \$codename, 'arch=s' => \$arch, 'chroot=s' => \$chroot, 'result-dir=s' => \$result_dir, 'log-dir=s' => \$log_dir, diff --git a/ipmitool/Build-notes b/ipmitool/Build-notes index 2cd69be..f0914ab 100755 --- a/ipmitool/Build-notes +++ b/ipmitool/Build-notes @@ -24,12 +24,15 @@ RPM Option #2 Use the manual steps listed below: DEB Option #1 -1) git clone https://github.com/xcat2/xcat-dep.git -2) apt install dpkg-dev debhelper libssl-dev quilt libreadline-dev -3) cd xcat-dep/ipmitool -4) ./make_deb.sh -5) Will generate file ipmitool-xcat__.deb file in current directory -6) Use debian/changelog file to update release version +The Ubuntu/Debian build is now driven by the top-level sbuild-all.pl orchestrator, which builds +each package inside the matching --sbuild chroot (build-deps installed in the +chroot automatically). The former ./make_deb.sh has been removed. + +1) git clone https://github.com/xcat2/xcat-dep.git; cd xcat-dep +2) Build every package for a codename: ./sbuild-all.pl --arch amd64 --dists noble + or this package alone: ./ipmitool/sbuild.pl --codename noble --arch amd64 --result-dir +3) Generates ipmitool-xcat__.deb into --result-dir +4) Use debian/changelog file to update release version NOTE #1 HOW TO CREATE THESE PATCHES diff --git a/ipmitool/build.sh b/ipmitool/build.sh deleted file mode 100755 index 572e74e..0000000 --- a/ipmitool/build.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/bin/bash - -function check_linux_distro() -{ - local distro="$(source /etc/os-release >/dev/null 2>&1 && echo "${ID}")" - [[ -z "${distro}" && -f /etc/redhat-release ]] && distro="rhel" - [[ -z "${distro}" && -f /etc/SuSE-release ]] && distro="sles" - echo "${distro}" -} - -pkgname="ipmitool" - -build_dir=${DEST:-/${pkgname}_build} - -XCAT_BUILD_DISTRO="$(check_linux_distro)" -echo "[INFO] Start to build $pkgname on $XCAT_BUILD_DISTRO" - -cur_path=$(dirname "$0") -cd $cur_path - -XCAT_BUILD_DISTRO="$(check_linux_distro)" -case "${XCAT_BUILD_DISTRO}" in -"centos"|"fedora"|"rhel"|"sles") - buildcmd="./bldipmi.pl" - dftpath="/tmp/build/" - pkgtype="rpm" - ;; -"ubuntu") - buildcmd="./make_deb.sh" - dftpath=$cur_path - pkgtype="deb" - ;; -*) - echo "${XCAT_BUILD_DISTRO}: unsupported Linux distribution to build goconserver" - exit 1 - ;; -esac - -$buildcmd |& tee /tmp/build.log -if [ $? != 0 ]; then - echo "[ERROR] Failed to build $pkgname by command $buildcmd" - exit 1 -fi - -buildpath=`find $dftpath -name ${pkgname}*.$pkgtype | xargs ls -t | head -n 1` -if [ -z "$buildpath" ]; then - echo "[ERROR] Could not find build ${pkgname}*.$pkgtype" - exit 1 -fi - -filepath=$(dirname $buildpath) -pathpre=${filepath:${#dftpath}} -build_dir=$build_dir/$pathpre -mkdir -p $build_dir - -cp -f $buildpath $build_dir -if [ $? != 0 ]; then - echo "[ERROR] Failed to copy $buildpath to $build_dir" - exit 1 -fi -cp -f /tmp/build.log $build_dir - -buildname=$(basename $buildpath) -echo "[INFO] Package path is $build_dir/$buildname" - -exit 0 diff --git a/ipmitool/sbuild.pl b/ipmitool/sbuild.pl index c86e3b0..cf26a4a 100755 --- a/ipmitool/sbuild.pl +++ b/ipmitool/sbuild.pl @@ -18,6 +18,9 @@ my $pkg_dir = abs_path($RealBin); my $pkg = basename($pkg_dir); my ($codename, $arch, $chroot, $result_dir, $log_dir) = ('', '', '', '', ''); my ($build_timestamp, $build_number, $skip_install) = (undef, undef, 0); +# --log-dir / --build-number / --skip-install are accepted for CLI-compat with sbuild-all.pl (which +# passes them uniformly to every per-package builder) but are intentionally UNUSED here: sbuild-all +# does its own per-package logging and there is no deb install-smoke. They are parsed and ignored. GetOptions( 'codename=s' => \$codename, 'arch=s' => \$arch, 'chroot=s' => \$chroot, 'result-dir=s' => \$result_dir, 'log-dir=s' => \$log_dir, diff --git a/sbuild-all.pl b/sbuild-all.pl index 603c29a..23c2216 100755 --- a/sbuild-all.pl +++ b/sbuild-all.pl @@ -32,11 +32,12 @@ use File::Temp qw(tempdir); use Getopt::Long qw(GetOptions); use Pod::Usage qw(pod2usage); use POSIX qw(strftime); +use Fcntl qw(:flock); use FindBin qw($RealBin); use lib $RealBin; use BuildUtils qw(sh_quote print_step version_matches required_pkgs read_manifest standard_options codename_to_version known_codenames chroot_name chroot_sources_list - deb_snap_version rewrite_changelog_top control_field genesis_deb_control + control_field genesis_deb_control deb_field deb_version deb_hash cross_copy_genesis_deb); my $script_dir = abs_path(dirname(__FILE__)); @@ -67,6 +68,9 @@ my @genesis_debs; # native xcat-genesis-base- deb(s): p my $genesis_rpm = ''; # fallback: native-arch genesis rpm to convert my $genesis_rpm_ppc = ''; # fallback: cross-arch ppc genesis rpm to convert (amd64 host) my $require_ppc_genesis = 0; +# File-scoped exclusive run-lock handle. MUST be file-scoped (not a lexical inside a block) so the +# flock lives for the WHOLE process -- a lexical would close the FH and release the lock early. +my $RUN_LOCK_FH; # Builder map: manifest binary-package name -> the in-tree package dir that carries /sbuild.pl # and the maintained debian/. (goconserver's dir == its binary name.) @@ -185,6 +189,21 @@ for my $cn (@dist_list) { my $staging = "$output_root/staging"; unless ($dry_run) { make_path($staging); } +# Fail-fast exclusive run lock. $staging and $apt_dir are STABLE, SHARED paths (not per-run), so two +# overlapping sbuild-all runs against the same --output-root corrupt each other -- the observed +# "remove_tree .../staging//: Directory not empty" is an NFS silly-rename from a concurrent +# run holding files open. Hold an exclusive flock on /.sbuild-all.lock for the whole +# process (LOCK_NB -> fail fast rather than block), mirroring how mockbuild-all.pl locks its output +# base. Not taken under --dry-run (no side effects to protect). +unless ($dry_run) { + make_path($output_root); + my $lockfile = "$output_root/.sbuild-all.lock"; + open($RUN_LOCK_FH, '>', $lockfile) or die "FATAL: cannot open run lock $lockfile: $!\n"; + unless (flock($RUN_LOCK_FH, LOCK_EX | LOCK_NB)) { + die "FATAL: another sbuild-all is running (lock held): $lockfile\n"; + } +} + print_step('Configuration'); print " repo-root: $repo_root\n"; print " xcat-source: $xcat_src\n"; @@ -210,6 +229,27 @@ sub run { return $ec; } +# wipe_tree: remove_tree that FAILS LOUD. A bare remove_tree() carps-and-ignores an ENOTEMPTY (e.g. an +# NFS silly-rename from a concurrent run); make_path then no-ops on the surviving dir and stale debs +# persist. Capturing {error} and dying makes the corruption fatal instead of silent. +sub wipe_tree { + my (@dirs) = @_; + remove_tree(@dirs, { safe => 1, error => \my $err }); + if ($err && @$err) { + my @msgs = map { my ($f, $m) = %$_; ($f eq '') ? $m : "$f: $m" } @$err; + die "FATAL: failed to remove @dirs: " . join('; ', @msgs) . "\n"; + } +} + +# deb_ver_gt: is Debian version $a strictly greater than $b? Uses dpkg's version comparison (the only +# correct arbiter of Debian version ordering). A missing/empty $b makes any $a "greater". +sub deb_ver_gt { + my ($a, $b) = @_; + return 1 if !defined $b || $b eq ''; + return 0 if !defined $a || $a eq ''; + return system('dpkg', '--compare-versions', $a, 'gt', $b) == 0 ? 1 : 0; +} + # --------------------------------------------------------------------------------------------------- # Phase: ensure chroots (absorbed mk-dep-chroots.sh; first-run auto-init, idempotent) # --------------------------------------------------------------------------------------------------- @@ -258,7 +298,7 @@ sub ensure_chroots { sub build_one_codename { my ($cn) = @_; my $tgt = "$cn-$arch"; - my $out = "$staging/$cn/$arch"; remove_tree($out) if -d $out; make_path($out); + my $out = "$staging/$cn/$arch"; wipe_tree($out) if -d $out; make_path($out); my @pkgs = grep { $_ ne 'xcat-genesis-base' } required_pkgs([sort keys %{$MANIFEST{$tgt}}], $skip_genesis, $skip_xcat_dep); print "== [$cn] building: @pkgs -> $out ==\n"; @@ -372,7 +412,7 @@ sub convert_genesis_rpm { } sub build_genesis { print_step('Genesis-base deb (maintained packaging preserved)'); - my $gen = "$output_root/$run_id/genesis"; remove_tree($gen) if -d $gen; make_path($gen); + my $gen = "$output_root/$run_id/genesis"; wipe_tree($gen) if -d $gen; make_path($gen); my $native_arch_pkg = "xcat-genesis-base-$arch"; my $produced_native = 0; # 1) prefer an ingested native deb (full metadata + maintainer scripts, no conversion loss) @@ -410,11 +450,23 @@ sub build_genesis { } # stage the arch:all genesis deb(s) into every codename (this host's arch subdir; the cross-arch # ppc genesis produced on the amd64 host rides in the amd64 subdir and is picked up by assemble). + # Use BuildUtils::cross_copy_genesis_deb -- the tested, hash-based, stale-dropping copier -- once + # per genesis package-arch present in $gen (the native-arch one, plus the cross-converted ppc64el + # one on the amd64 host). It refreshes a stale same-name deb by content and is idempotent. + my %gen_arches; + for my $d (glob("$gen/*.deb")) { + $gen_arches{$1}++ if basename($d) =~ /^xcat-genesis-base-([a-z0-9]+)_/; + } for my $cn (@dist_list) { - make_path("$staging/$cn/$arch") unless $dry_run; - for my $d (glob("$gen/*.deb")) { - copy($d, "$staging/$cn/$arch/" . basename($d)) unless $dry_run; - print " staged " . basename($d) . " -> $cn/$arch\n"; + my $dst = "$staging/$cn/$arch"; + if ($dry_run) { + print " [dry-run] would stage genesis (" . join(',', sort keys %gen_arches) . ") -> $cn/$arch\n"; + next; + } + make_path($dst); + for my $ga (sort keys %gen_arches) { + my $n = cross_copy_genesis_deb($gen, $dst, $ga, undef); + print " staged xcat-genesis-base-$ga -> $cn/$arch ($n newly copied)\n"; } } } @@ -454,11 +506,36 @@ sub assemble_apt { # wipe ONLY this codename's published pool+dists, then repopulate from validated staging # (both arches: staging//{amd64,ppc64el}/*.deb). Wiping first is what removes stale debs # from a prior run so the published repo never carries a mixture (concern #1). - remove_tree($pool, "$apt_dir/dists/$cn", "$apt_dir/$ver") unless $dry_run; + wipe_tree($pool, "$apt_dir/dists/$cn", "$apt_dir/$ver") unless $dry_run; make_path($pool, "$apt_dir/$ver") unless $dry_run; - for my $deb (glob("$staging/$cn/*/*.deb")) { - my $b = basename($deb); - unless ($dry_run) { link($deb, "$pool/$b") or copy($deb, "$pool/$b"); copy($deb, "$apt_dir/$ver/$b"); } + unless ($dry_run) { + # Collect this codename's staged debs across both arches, deduping on binary package + # NAME+ARCH: if two files resolve to the same package+arch (e.g. a native ppc genesis and + # an amd64-host cross-converted one both claiming xcat-genesis-base-ppc64el/all) only ONE + # may reach the pool. Keep the highest version and warn naming both -- a safety net that + # holds regardless of the --skip-genesis single-producer contract (concern #4). + my %best; # "name|arch" => { file => path, ver => version } + for my $deb (glob("$staging/$cn/*/*.deb")) { + my $name = deb_field($deb, 'Package'); + my $darch = deb_field($deb, 'Architecture'); + my $dver = deb_field($deb, 'Version'); + my $key = "$name|$darch"; + if (my $cur = $best{$key}) { + my $new_wins = deb_ver_gt($dver, $cur->{ver}); + my ($win, $lose) = $new_wins ? ($deb, $cur->{file}) : ($cur->{file}, $deb); + warn "WARN: duplicate binary $name/$darch in staging for $cn -- keeping " + . basename($win) . ", dropping " . basename($lose) . "\n"; + $best{$key} = { file => $deb, ver => $dver } if $new_wins; + next; + } + $best{$key} = { file => $deb, ver => $dver }; + } + for my $key (sort keys %best) { + my $deb = $best{$key}{file}; + my $b = basename($deb); + link($deb, "$pool/$b") or copy($deb, "$pool/$b"); + copy($deb, "$apt_dir/$ver/$b"); + } } # Packages index per binary-: an arch's index carries that arch's debs + all Architecture:all. for my $a (qw(amd64 ppc64el)) { @@ -476,13 +553,18 @@ sub assemble_apt { run("gzip -9 -kf -n " . sh_quote("$bindir/Packages")); } next if $dry_run; + # Advertise ONLY the arches actually staged for this codename: an arch counts iff its + # binary-/Packages is non-empty. A single-arch run must not claim a missing arch in + # Release (apt would then error on the absent index). + my @staged_arches = grep { -s "$apt_dir/dists/$cn/main/binary-$_/Packages" } qw(amd64 ppc64el); + @staged_arches = ('amd64') unless @staged_arches; # never emit an empty Architectures line # Release + sign my @rel = ('apt-ftparchive', '-o', 'APT::FTPArchive::Release::Origin=xCAT', '-o', 'APT::FTPArchive::Release::Label=xcat-dep', '-o', "APT::FTPArchive::Release::Suite=$cn", '-o', "APT::FTPArchive::Release::Codename=$cn", - '-o', 'APT::FTPArchive::Release::Architectures=amd64 ppc64el', + '-o', 'APT::FTPArchive::Release::Architectures=' . join(' ', @staged_arches), '-o', 'APT::FTPArchive::Release::Components=main', '-o', "APT::FTPArchive::Release::Description=xCAT dependency packages for $ver", 'release', "$apt_dir/dists/$cn/"); diff --git a/syslinux/sbuild.pl b/syslinux/sbuild.pl index 5616620..a584856 100755 --- a/syslinux/sbuild.pl +++ b/syslinux/sbuild.pl @@ -18,6 +18,9 @@ my $pkg_dir = abs_path($RealBin); my $pkg = basename($pkg_dir); my ($codename, $arch, $chroot, $result_dir, $log_dir) = ('', '', '', '', ''); my ($build_timestamp, $build_number, $skip_install) = (undef, undef, 0); +# --log-dir / --build-number / --skip-install are accepted for CLI-compat with sbuild-all.pl (which +# passes them uniformly to every per-package builder) but are intentionally UNUSED here: sbuild-all +# does its own per-package logging and there is no deb install-smoke. They are parsed and ignored. GetOptions( 'codename=s' => \$codename, 'arch=s' => \$arch, 'chroot=s' => \$chroot, 'result-dir=s' => \$result_dir, 'log-dir=s' => \$log_dir, diff --git a/t/sbuild-all.t b/t/sbuild-all.t index b619afe..8b54e31 100644 --- a/t/sbuild-all.t +++ b/t/sbuild-all.t @@ -15,7 +15,7 @@ use File::Basename qw(basename); use BuildUtils qw(required_pkgs version_matches read_manifest standard_options codename_to_version version_to_codename known_codenames chroot_name chroot_sources_list - deb_snap_version rewrite_changelog_top control_field genesis_deb_control + control_field genesis_deb_control deb_field deb_version deb_upstream_version deb_hash cross_copy_genesis_deb); # Run a printing sub with STDOUT muted so its progress lines do not pollute TAP. @@ -73,36 +73,6 @@ is(chroot_name('noble', 'amd64'), 'noble-amd64-sbuild', 'chroot_name shape'); 'mirror override honored'); } -# ---- deb_snap_version: out-of-tree CD stamping (idempotent / self-healing) ---------------------- -is(deb_snap_version('0.3.3', '202608101400', 57), '0.3.3-snap202608101400.57', - 'snap stamp with build number'); -is(deb_snap_version('0.3.3', '202608101400'), '0.3.3-snap202608101400', - 'snap stamp without build number'); -is(deb_snap_version('0.3.3-snap202601010000.1', '202608101400', 58), '0.3.3-snap202608101400.58', - 'a prior snap stamp is REPLACED, not stacked'); -unlike(deb_snap_version('0.3.3-snap202601010000.1', '202608101400', 58), qr/snap.*snap/, - 'never leaves two stacked snap stamps'); -{ - my $died = !eval { deb_snap_version('0.3.3', 'notatimestamp'); 1 }; - ok($died, 'deb_snap_version dies on a non-YYYYMMDDHHMM timestamp'); -} - -# ---- rewrite_changelog_top: edit a COPY of debian/changelog (never the tracked one) ------------- -{ - my $cl = "goconserver (0.3.3-1) unstable; urgency=medium\n\n * something\n\n" - . " -- xCAT Mon, 01 Jan 2024 00:00:00 +0000\n\n" - . "goconserver (0.3.2-1) unstable; urgency=low\n\n * older\n\n" - . " -- xCAT Sun, 01 Jan 2023 00:00:00 +0000\n"; - my $out = rewrite_changelog_top($cl, '0.3.3-snap202608101400.57', - 'xCAT Build ', 'Mon, 10 Aug 2026 14:00:00 +0000'); - like($out, qr/^goconserver \(0\.3\.3-snap202608101400\.57\) unstable/, - 'top entry version rewritten'); - like($out, qr/ -- xCAT Build Mon, 10 Aug 2026/, - 'top trailer rewritten to the build identity + date'); - like($out, qr/goconserver \(0\.3\.2-1\)/, 'older entry left untouched'); - is(scalar(() = $out =~ /urgency=/g), 2, 'still two entries (nothing duplicated/dropped)'); -} - # ---- control_field: parse a control paragraph, folding continuations ---------------------------- { my $ctrl = "Package: foo\nDepends: libc6 (>= 2.15),\n bar,\n baz\nBreaks: old-foo\n"; diff --git a/xnba/README b/xnba/README index bac5037..1b9b6c4 100644 --- a/xnba/README +++ b/xnba/README @@ -46,7 +46,10 @@ Build xNBA deb for xcat-dep --------------------------- 1. On x86 Ubuntu machine, install required packages: git, liblzma-dev, dpkg-dev, debhelper, and quilt 2. Checkout the branch created above: git clone https://github.com/xcat2/xcat-dep.git; cd xcat-dep; git checkout -3. DEB build: cd xcat-dep/xnba; ./make_deb.sh: build the package from the binaries under "binary" directory +3. DEB build: the Ubuntu/Debian build is now driven by the top-level sbuild-all.pl orchestrator + (builds inside the matching --sbuild chroot). Build this package with + ./sbuild-all.pl --arch amd64 --dists noble (or ./xnba/sbuild.pl --codename noble --arch amd64 + --result-dir for this package alone). The former ./make_deb.sh has been removed. 4. ./rebuild.sh : build the package from source code, this should only run on amd64/x86_64 platform, it will update the files under "binary" directory after compilation 5. Copy generated deb file xcat-dep/xnba-undi__all.deb into /gsa/pokgsa/projects/x/xcat/build/ubuntu/xcat-dep/xcat-dep/pool/main/x/xnba-undi/ and /gsa/pokgsa/projects/x/xcat/build/ubuntu/xcat-dep/debs diff --git a/xnba/sbuild.pl b/xnba/sbuild.pl index 9ccef0c..b5e993d 100755 --- a/xnba/sbuild.pl +++ b/xnba/sbuild.pl @@ -18,6 +18,9 @@ my $pkg_dir = abs_path($RealBin); my $pkg = basename($pkg_dir); my ($codename, $arch, $chroot, $result_dir, $log_dir) = ('', '', '', '', ''); my ($build_timestamp, $build_number, $skip_install) = (undef, undef, 0); +# --log-dir / --build-number / --skip-install are accepted for CLI-compat with sbuild-all.pl (which +# passes them uniformly to every per-package builder) but are intentionally UNUSED here: sbuild-all +# does its own per-package logging and there is no deb install-smoke. They are parsed and ignored. GetOptions( 'codename=s' => \$codename, 'arch=s' => \$arch, 'chroot=s' => \$chroot, 'result-dir=s' => \$result_dir, 'log-dir=s' => \$log_dir,