From b8510e1be3e5bc64c445785bcf16323eb3cb3085 Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Mon, 31 Aug 2026 23:26:51 -0300 Subject: [PATCH] refactor(build): replace build-ubunturepo with builddebs.pl and BuildUtils.pm build-ubunturepo was 710 lines of shell doing the Debian half of what buildrpms.pl does for rpms, with no code in common and a different CLI. It also carried paths that are dead: GSA uploads, the PROMOTE/PREGA release flows, and a -d mode that built an xcat-dep repository from a different project's packages. builddebs.pl replaces it and mirrors buildrpms.pl -- Getopt::Long options, one package list, build then index then sign -- so the two builders read the same way and share BuildUtils.pm. The design rests on one fact: xcat-core debs are Perl. They are byte-identical for every Ubuntu release, so they are built ONCE and the same files are published into every codename. Only xCAT, xCATsn and xCAT-genesis-scripts carry an architecture, and there the difference is packaging metadata, not compiled output. That is why this needs no sbuild and no per-codename chroot -- unlike xcat-dep, whose packages are compiled and genuinely differ per release. BuildUtils.pm holds what both builders need and what was worth making testable: the Version-Release derivation from the commit time, the xCAT-probe helper staging, the deb arch and dist tables, the debian/control version pinning, the changelog rewrite, the reprepro conf generation, and the build lock. Every function is pure or takes its side effect as an argument, so build_utils.t (45 assertions) drives each one rather than grepping a builder for evidence that it is called. Verified by mutation: shrinking the arch table reddens 1, dropping the /g from the control pin reddens 2. The env-var CLI maps to options: BUILDALL=1 -> --force, GPGSIGN=1 -> --gpg-sign, GPG_HOME -> --gpg-home, DEST -> --dest, DISTS -> --dist (repeatable). UP=0 has no equivalent because uploading is gone -- the CD pipeline's deploy step publishes. Callers updated: github_action_xcat_test.pl and travis.pl. The comment in github_action_xcat_test.pl explaining why CI copies the tree before building is corrected -- build-ubunturepo rm -rf'd $curdir/../../xcat-core, which under GitHub's work// layout is the checkout's own parent; builddebs.pl writes under dist/debs inside the checkout and restores every file it edits, so the copy is now only isolating the tests from build residue. Two tests moved with it. build_ubunturepo_lock.t extracted the lock out of the shell with a regex and ran that; the lock is now a function, so builddebs_lock.t calls it -- and asserts what actually matters, that two builds of one checkout fail fast while two builds of different checkouts run concurrently. ubuntu_2604_pkglist.t asserted that resolute appeared in a shell fragment of build-ubunturepo's source; it now asks BuildUtils for the release list and checks a resolute stanza reaches conf/distributions. That assertion would have passed on any file containing the fragment and broken on a reflow that changed nothing. Verified: prove -r xCAT-test/unit fails on 6 files here against 7 on upstream/master, the difference being apache_config_sources.t, fixed by the preceding commit. The remaining 6 are missing DB modules on the machine that ran it and are identical on both. NOT done here, and required before this can merge: the Ubuntu core CD pipelines still invoke ./build-ubunturepo (ci/ubuntu/Jenkinsfile.core-ubuntu-{devel,stable} in VersatusHPC/xcat-core-ci-cd, and the inline script in each live Jenkins job). Those must be switched to builddebs.pl in the same change window. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- BuildUtils.pm | 238 ++++++ build-ubunturepo | 710 ------------------ builddebs.pl | 422 +++++++++++ docs/source/developers/guides/code/builds.rst | 31 +- github_action_xcat_test.pl | 17 +- travis.pl | 2 +- xCAT-test/unit/build_ubunturepo_lock.t | 155 ---- xCAT-test/unit/build_utils.t | 182 +++++ xCAT-test/unit/builddebs_lock.t | 54 ++ xCAT-test/unit/ubuntu_2604_pkglist.t | 18 +- 10 files changed, 946 insertions(+), 883 deletions(-) create mode 100644 BuildUtils.pm delete mode 100755 build-ubunturepo create mode 100644 builddebs.pl delete mode 100644 xCAT-test/unit/build_ubunturepo_lock.t create mode 100644 xCAT-test/unit/build_utils.t create mode 100644 xCAT-test/unit/builddebs_lock.t diff --git a/BuildUtils.pm b/BuildUtils.pm new file mode 100644 index 000000000..306d7f7cb --- /dev/null +++ b/BuildUtils.pm @@ -0,0 +1,238 @@ +package BuildUtils; +# Reusable, unit-testable helpers shared by the xcat-core build tooling: buildrpms.pl +# (rpm/mock) and builddebs.pl (deb/reprepro). Both derive the same Version-Release from +# the same git state, stage the same xCAT-probe helpers, and shell out the same way, so +# that logic lives here once instead of twice. +# +# Everything here is a pure function of its arguments, or a thin wrapper whose side +# effect is the argument. Nothing reaches for an orchestrator global, so +# xCAT-test/unit/build_utils.t drives every function directly rather than grepping the +# builders for evidence that they call it. +# +# It mirrors xcat-dep's BuildUtils.pm in shape and intent; the two repos ship separate +# copies because neither installs the other's tooling. +use strict; +use warnings; +use Exporter 'import'; +use File::Copy qw(copy); +use File::Path qw(make_path); +use POSIX qw(strftime); + +our @EXPORT_OK = qw( + source_date_epoch snap_release deb_version + stage_probe_helpers XCAT_PROBE_HELPERS + deb_package_arches dist_arches default_dists + orig_tarball_name pin_control_version rewrite_changelog_header + reprepro_distributions reprepro_options + lock_id_for take_build_lock + sh_quote +); + +# The xCAT-probe helpers. xcat-probe reuses functions shipped by xCAT; they are COPIED +# rather than symlinked because a symlink does not survive packaging, and rather than +# maintained twice because they would drift. Both builders stage them the same way. +use constant XCAT_PROBE_HELPERS => qw( + GlobalDef.pm + NetworkUtils.pm + ServiceNodeUtils.pm +); + +# Packages whose .deb carries a real architecture. Everything else in xcat-core is +# Perl and ships as Architecture: all -- one binary serving every Ubuntu release and +# every arch, which is why this build never needs a per-codename chroot. +my %ARCH_PACKAGES = map { $_ => 1 } qw(xCAT xCATsn xCAT-genesis-scripts); + +# Ubuntu releases predating ppc64el. Kept as data rather than an `if` in the caller so +# the repo-assembly and the package-selection paths cannot disagree about it. +my %NO_PPC64EL = map { $_ => 1 } qw(saucy); + +my @DEB_ARCHES = qw(amd64 ppc64el); + +# The Ubuntu releases the apt repository serves by default. Single source of truth: +# the builder, the repo assembly and the tests all read it here, so they cannot drift. +my @DEFAULT_DISTS = qw(focal jammy noble resolute); + +sub default_dists { return @DEFAULT_DISTS; } + +# sh_quote: single-quote a string for safe use in a shell command. +sub sh_quote { + my ($s) = @_; + $s = '' if !defined $s; + $s =~ s/'/'"'"'/g; + return "'$s'"; +} + +# source_date_epoch: the commit time the build is reproducible against. +# +# Gitepoch wins when present -- CI writes it so every arch of one release stamps an +# identical epoch even when the arches build minutes apart. Falling back to the local +# clock is last-resort: it makes the build non-reproducible, so the caller is told. +sub source_date_epoch { + my (%args) = @_; + my $read = $args{read_file} || sub { + my ($p) = @_; + return unless -f $p; + open my $fh, '<', $p or return; + my $v = <$fh>; + close $fh; + return $v; + }; + my $git = $args{git_epoch} || sub { return scalar `git log -1 --format=%ct HEAD 2>/dev/null`; }; + + for my $candidate ($read->('Gitepoch'), $git->()) { + next unless defined $candidate; + chomp $candidate; + return $candidate if $candidate =~ /\A\d+\z/; + } + return $args{now} || time(); +} + +# snap_release: the Release string, derived from the commit time so identical sources +# give identical NVRs. UTC, because a build host's timezone must not change the name. +sub snap_release { + my ($epoch) = @_; + return strftime("snap%Y%m%d%H%M", gmtime($epoch)); +} + +# deb_version: the Debian version. Same Version-Release pair the rpms carry, so an +# apt repo and a yum repo built from one commit report the same thing. +sub deb_version { + my ($version, $release) = @_; + return "$version-$release"; +} + +# stage_probe_helpers: copy the shared helpers into xCAT-probe's tree. +# Returns the list of destination paths, so a caller can remove exactly what it added. +sub stage_probe_helpers { + my ($source_dir, $dest_dir) = @_; + make_path($dest_dir) unless -d $dest_dir; + my @staged; + for my $helper (XCAT_PROBE_HELPERS) { + my $from = "$source_dir/$helper"; + my $to = "$dest_dir/$helper"; + copy($from, $to) or die "Unable to stage $from into $dest_dir: $!\n"; + push @staged, $to; + } + return @staged; +} + +# deb_package_arches: the architectures to build a package for. +# 'all' is a single arch-independent build; the three arch packages get one per arch. +sub deb_package_arches { + my ($package) = @_; + return @DEB_ARCHES if $ARCH_PACKAGES{$package // ''}; + return ('all'); +} + +# dist_arches: the architectures a release's apt repo declares. +sub dist_arches { + my ($dist) = @_; + return ('amd64') if $NO_PPC64EL{$dist // ''}; + return @DEB_ARCHES; +} + +# orig_tarball_name: the .orig.tar.gz dpkg-source expects for a 3.0 (quilt) package. +# The name is lower-cased because dpkg requires a lower-case source package name. +sub orig_tarball_name { + my ($package, $version) = @_; + return lc($package) . "_$version.orig.tar.gz"; +} + +# pin_control_version: pin xCAT's inter-package dependencies to this exact build. +# +# debian/control carries the sentinel ">= 2.13-snap000000000000" on every intra-xCAT +# dependency. Left alone, apt would satisfy them with any older xCAT already installed, +# so a partial upgrade could mix versions. Replacing it with "= " makes the set +# install or fail as a unit. +sub pin_control_version { + my ($control, $version) = @_; + return $control unless defined $control; + $control =~ s/>= \Q2.13-snap000000000000\E/= $version/g; + return $control; +} + +# rewrite_changelog_header: set the version and the trailer date of the top stanza. +# +# The date comes from SOURCE_DATE_EPOCH rather than "now" so two builds of one commit +# produce byte-identical packages. Only the first stanza is touched -- the history below +# it is not ours to rewrite. +sub rewrite_changelog_header { + my ($changelog, $version, $date, $maintainer) = @_; + return $changelog unless defined $changelog; + $changelog =~ s/\A(\S+) \([^)]*\)/$1 ($version)/; + $changelog =~ s/^ -- .*$/ -- $maintainer $date/m; + return $changelog; +} + +# reprepro_distributions: the conf/distributions body for the whole repo. +# +# One stanza per release, all listing the same packages: xcat-core debs are Perl and are +# byte-identical across releases, so the build produces them once and every codename +# serves the same files. keyid is undef for an unsigned repo. +sub reprepro_distributions { + my ($dists, $keyid) = @_; + my $out = ''; + for my $dist (@$dists) { + my $arches = join ' ', dist_arches($dist); + $out .= <<"STANZA"; +Origin: xCAT internal repository +Label: xcat-core bazaar repository +Codename: $dist +Architectures: $arches +Components: main +Description: Repository automatically genereted conf +STANZA + $out .= "SignWith: $keyid\n" if defined $keyid && length $keyid; + $out .= "\n"; + } + return $out; +} + +# reprepro_options: the conf/options body. +# +# ask-passphrase is omitted when a GNUPGHOME is supplied, because that key is +# passphrase-less and an unattended build must never stop to prompt. +sub reprepro_options { + my ($gpg_home) = @_; + my $out = "verbose\n"; + $out .= "ask-passphrase\n" unless defined $gpg_home && length $gpg_home; + $out .= "basedir .\n"; + return $out; +} + + +# lock_id_for: a short, stable id for a checkout path. +# +# The build rewrites debian/changelog and debian/control and runs dpkg-buildpackage +# inside the package directories, so what two builds contend for is the CHECKOUT, not +# the host. A host-global lock made the devel and stable CD lanes collide even though +# they share nothing. Keying on the path lets distinct checkouts build in parallel while +# two builds of one checkout still fail fast. +sub lock_id_for { + my ($path) = @_; + require Digest::MD5; + return substr(Digest::MD5::md5_hex(defined $path ? $path : ''), 0, 12); +} + +# lock_path_for: where that checkout's lock lives. +# Local /var/lock deliberately: the checkout itself may be on NFS, where flock is not +# reliable. +sub lock_path_for { + my ($path, $dir) = @_; + $dir = '/var/lock' unless defined $dir; + return "$dir/xcatbld-" . lock_id_for($path) . ".lock"; +} + +# take_build_lock: take the checkout's lock, or die. +# Returns the open handle -- the lock is held for as long as the caller keeps it. +sub take_build_lock { + my ($path, $dir) = @_; + require Fcntl; + my $lockfile = lock_path_for($path, $dir); + open my $fh, '>', $lockfile or die "FATAL: cannot open $lockfile: $!\n"; + flock($fh, Fcntl::LOCK_EX() | Fcntl::LOCK_NB()) + or die "FATAL: another build of $path already holds $lockfile\n"; + return $fh; +} + +1; diff --git a/build-ubunturepo b/build-ubunturepo deleted file mode 100755 index bc92165ee..000000000 --- a/build-ubunturepo +++ /dev/null @@ -1,710 +0,0 @@ -#!/bin/bash -# Update GSA Ubuntu Repositories or create a local repository -# -# Author: Leonardo Tonetto (tonetto@linux.vnet.ibm.com) -# Revisor: Arif Ali (aali@ocf.co.uk) -# -# -# Getting Started: -# - Clone the xcat-core git repository under a directory named "xcat-core/src" -# - make sure reprepro is installed on the build machine -# - Run this script from the local git repository you just created. -# ./build-ubunturepo -c BUILDALL=1 - -# Usage: attr=value attr=value ... ./build-ubunturepo { -c | -d } -# PROMOTE=1 - if the attribute "PROMOTE" is specified, means an official dot release. This does not -# actually build xcat, just uploads the most recent snap build to http://xcat.org/files/xcat/ . -# If not specified, a snap build is assumed, which uploads to https://xcat.org/files/xcat/ -# PREGA=1 - use this option with PROMOTE=1 on a branch that already has a released dot release, but this -# build is a GA candidate build, not to be released yet. This will result in the tarball -# being uploaded to http://xcat.org/files/xcat/repos/apt -# (but the tarball file name will be like a released tarball, not a snap build). -# When you are ready to release this build, use PROMOTE=1 without PREGA -# BUILDALL=1 - build all rpms, whether they changed or not. Should be used for snap builds that are in -# prep for a release. -# GPGSIGN=0 - Do not sign the repo in the end of the build. The repo will be signed by default -# -# LOCAL_KEY=1 Use local keys to sign repo instead of WGET from GSA. By default use GSA. -# -# GPG_HOME= - Use the specified directory as GNUPGHOME for signing (no passphrase assumed). -# Bypasses GSA download and LOCAL_KEY. -# -# SETUP=1 Setup environment for build. By default do not setup environment. -# -# LOG= - provide an LOG file option to redirect some output into log file -# -# DEST= - provide a directory to contains the build result -# -# Running builds in parallel on one host: the build lock is scoped to the source checkout (see -# the "build-lock" block below), so two builds from DIFFERENT -# checkouts -- e.g. the devel and stable Ubuntu CD lanes -- run -# concurrently, while two builds of the SAME checkout still fail-fast -# (they build in-place and would corrupt each other). For parallel -# builds give each a separate checkout and a separate output tree -# (a distinct DEST). GPG_HOME may be SHARED between parallel builds -# -- it is used read-only for signing. -# -# For the dependency packages 1. All the xcat dependency deb packages should be uploaded to -# "pokgsa/projects/x/xcat/build/ubuntu/xcat-dep/debs/" on GSA -# 2. run ./build-ubunturepo -d -# -# 3. the built xcat-dep deb packages tarball can be found in "../../xcat-dep" -# related to the path of this script -############################ -printusage() -{ - printf "Usage: %s {-c | -d} \n" $(basename $0) >&2 - echo " -c : Build the xcat-core packages and create the repo" - echo " -d : Create the xcat-dep repo." -} -# For the purpose of getting the distribution name -if [[ ! -f /etc/lsb-release ]]; then - echo "ERROR: Could not find /etc/lsb-release, is this script executed on a Ubuntu machine?" - exit 1 -fi -. /etc/lsb-release - -export HOME=/root - - -# Process cmd line variable assignments, assigning each attr=val pair to a variable of same name -for i in $*; do - echo $i | grep '=' -q - if [ $? != 0 ];then - continue - fi - # upper case the variable name - varstring=`echo "$i"|cut -d '=' -f 1|tr '[a-z]' '[A-Z]'`=`echo "$i"|cut -d '=' -f 2` - export $varstring -done - -#Setup environment so the xcat-deps can be built on a FVT test machine -if [ "$SETUP" = "1" ];then - #Mount GSA - POKGSA="/gsa/pokgsa" - POKGSA2="/gsa/pokgsa-p2" - POKGSAIBM="pokgsa.ibm.com" - if [ ! -d $POKGSA ];then - mkdir -p $POKGSA - mount ${POKGSAIBM}:${POKGSA} ${POKGSA} - fi - if [ ! -d $POKGSA2 ];then - mkdir -p $POKGSA2 - mount ${POKGSAIBM}:${POKGSA2} ${POKGSA2} - fi - - # Verify needed packages installed - REPREPO="reprepro" - DEVSCRIPTS="devscripts" - DEBHELPER="debhelper" - QUILT="quilt" - - apt-get -y install $REPREPO $DEVSCRIPTS $DEBHELPER $QUILT - - echo "Finished setup for xcat-dep build. Rerun this script with SETUP=0 LOCAL_KEY=1 flags" - exit 1 -fi - -# Check the necessary packages before starting the build -declare -a packages=( "reprepro" "devscripts" "debhelper" "libsoap-lite-perl" "libdbi-perl" "quilt" "git") - -for package in ${packages[@]}; do - RC=`dpkg -l | grep $package >> /dev/null 2>&1; echo $?` - if [[ ${RC} != 0 ]]; then - echo "ERROR: Could not find $package, install using 'apt-get install $package' to continue" - exit 1 - fi -done - -# Supported distributions. Set DISTS="jammy noble resolute" to limit local validation builds. -dists="${DISTS:-saucy trusty utopic xenial bionic focal jammy noble resolute}" - -# GPG key used to sign the apt repo (reprepro SignWith). Defaults to the historic -# name. Override with GPG_KEY_ID= (space-free, since it is passed via -# the attr=value parser above), e.g. GPG_KEY_ID=xcat-build@xcat.org -GPG_KEY_ID="${GPG_KEY_ID:-xCAT Automatic Signing Key}" - -c_flag= # xcat-core (trunk-delvel) path -d_flag= # xcat-dep (trunk) path -r_flag= #genesis base rpm package path - -while getopts 'cdr:' OPTION -do - case $OPTION in - c) c_flag=1 - ;; - d) d_flag=1 - ;; - r) r_flag=1 - genesis_rpm_path="$OPTARG" - ;; - ?) printusage - exit 2 - ;; - esac -done -shift $(($OPTIND - 1)) - -if [ -z "$c_flag" -a -z "$d_flag" ];then - printusage - exit 2 -fi - -if [ "$c_flag" -a "$d_flag" ];then - printusage - exit 2 -fi - -if [ -z "$BUILDALL" ]; then - BUILDALL=1 -fi - -# Find where this script is located to set some build variables -old_pwd=`pwd` -cd `dirname $0` -curdir=`pwd` - -# Scope the build lock to THIS checkout. build-ubunturepo builds the packages in-place -# in its own source tree (it rewrites debian/changelog and debian/control, drops -# *.orig.tar.gz at the checkout root and runs dpkg-buildpackage inside the package -# dirs), so the resource two builds actually contend for is the checkout -- not the -# host. The historic single /var/lock/xcatbld.lock was host-global and fail-fast, so -# two builds from *different* checkouts (e.g. the devel and stable Ubuntu CD lanes on -# one build host) collided and the loser failed the pipeline even though they share -# nothing. Key the lock on the checkout path instead: builds of the SAME checkout -# still fail-fast (they would corrupt each other in-place), while builds of DISTINCT -# checkouts get distinct locks and run in parallel. The lock file stays on the local -# /var/lock (reliable flock; the checkout may live on NFS/virtiofs where flock is not) -# and the source tree is left byte-pristine. -# -# NOTE: the two marked regions below are extracted verbatim and exercised by the unit -# test xCAT-test/unit/build_ubunturepo_lock.t (which runs them with a chosen $curdir) -# -- keep the markers, and keep each region self-contained. -# BEGIN build-lock-id -lock_id_for() { printf '%s' "$1" | md5sum | cut -c1-12; } -LOCKFILE="/var/lock/xcatbld-$(lock_id_for "$curdir").lock" -# END build-lock-id -# BEGIN build-lock-acquire -exec 8>"$LOCKFILE" -if ! flock -n 8; then - echo "ERROR: Can't get lock $LOCKFILE for checkout $curdir. Another build is already using this checkout. Exiting...." - exit 1 -fi -# END build-lock-acquire - -# for the git case, query the current branch and set REL (changing master to devel if necessary) -function setbranch { - # Get the current branch name. safe.directory='*' so this still works when the - # build runs as root against a repo owned by another user (otherwise git errors - # with "dubious ownership", returns empty, and REL collapses to an unstable value). - branch=`git -c safe.directory='*' rev-parse --abbrev-ref HEAD 2>/dev/null` - if [ "$branch" = "master" ]; then - REL="devel" - elif [ "$branch" = "HEAD" ] || [ -z "$branch" ]; then - # Special handling when in a 'detached HEAD' state - branch=`git -c safe.directory='*' describe --abbrev=0 HEAD 2>/dev/null` - [[ -n "$branch" ]] && REL=`echo $branch|cut -d. -f 1,2` - else - REL=$branch - fi -} - -WGET_CMD="wget" -if [ ! -z ${LOG} ]; then - WGET_CMD="wget -o ${LOG}" -fi - -if [ "$GPGSIGN" = "0" ];then - echo "GPGSIGN=$GPGSIGN specified, skip gnupg key downloading" -elif [ -n "$GPG_HOME" ];then - echo "GPG_HOME=$GPG_HOME specified, using provided GNUPGHOME" - export GNUPGHOME="$GPG_HOME" -else - #sync the gpg key to the build machine local - gsa_url=http://pokgsa.ibm.com/projects/x/xcat/build/linux - mkdir -p $HOME/.gnupg - for key_name in pubring.gpg secring.gpg trustdb.gpg; do - if [ "$LOCAL_KEY" = "1" ];then - # Keys are already in the local $HOME/.gnupg directory - chmod 600 $HOME/.gnupg/$key_name - else - # Need to download keys from GSA - if [ ! -f $HOME/.gnupg/$key_name ] || [ `wc -c $HOME/.gnupg/$key_name|cut -f 1 -d' '` == 0 ]; then - rm -f $HOME/.gnupg/$key_name - ${WGET_CMD} -P $HOME/.gnupg $gsa_url/keys/$key_name - chmod 600 $HOME/.gnupg/$key_name - fi - fi - done -fi - -REL=xcat-core -if [ "$c_flag" ] -then - setbranch - # Sanitize REL into a stable, filesystem-safe token: replace any character that - # isn't [A-Za-z0-9._-] (e.g. the '/' in a branch like feat/ubuntu-e2e, which would - # otherwise create nested dirs) with '-', and never let it be empty. - REL=${REL//[^A-Za-z0-9._-]/-} - [ -z "$REL" ] && REL="local" - package_dir_name=debs$REL - - #define the dep source code path, core build target path and dep build target path - if [ -z "$DEST" ]; then - local_core_repo_path="$curdir/../../xcat-core" - PKGDIR="../../$package_dir_name" - else - local_core_repo_path="$DEST/$package_dir_name/xcat-core" - PKGDIR="$DEST/$package_dir_name/$package_dir_name" - fi - if [ ! -d "$PKGDIR" ];then - mkdir -p "$PKGDIR" - fi - - echo "#############################################################" - echo "Building xcat-core on branch ($REL) to $local_core_repo_path" - echo "#############################################################" - if [ "$PROMOTE" != 1 ]; then - code_change=0 - update_log='' - - if [ -z "$GITUP" ];then - update_log=../coregitup - echo "git pull > $update_log" - git pull > $update_log - else - update_log=$GITUP - fi - - if ! grep -q 'Already up-to-date' $update_log; then - code_change=1 - fi - ver=`cat Version` - short_ver=`cat Version|cut -d. -f 1,2` - short_short_ver=`cat Version|cut -d. -f 1` - commit_id_long=`git rev-parse HEAD` - commit_id="${commit_id_long:0:7}" - if [ -f Gitepoch ]; then - source_date_epoch=$(cat Gitepoch) - else - source_date_epoch=$(git log -1 --format=%ct HEAD 2>/dev/null || date +%s) - fi - export SOURCE_DATE_EPOCH="$source_date_epoch" - export DEBEMAIL="xcat-build@xcat.org" - export DEBFULLNAME="xCAT Build" - build_time=$(date -d "@$source_date_epoch" --utc '+%Y-%m-%dT%H:%M:%SZ' 2>/dev/null || date -u) - build_machine=`hostname` - - if [ $code_change == 0 -a "$UP" != 1 -a "$BUILDALL" != 1 ]; then - echo "Nothing new detected. Exiting...." - exit 0 - fi - - echo "###############################" - echo "# Building xcat-core packages #" - echo "###############################" - - #the package type: local | snap | alpha - #the build introduce string - build_string="Snap_Build" - if [ -f Release ]; then - xcat_release=$(cat Release) - else - xcat_release="snap$(date -d "@$source_date_epoch" --utc '+%Y%m%d%H%M')" - fi - pkg_version="${ver}-${xcat_release}" - - packages="xCAT-client xCAT-genesis-scripts perl-xCAT xCAT-server xCAT xCATsn xCAT-test xCAT-buildkit xCAT-vlan xCAT-confluent xCAT-probe" - if [ -n "$PACKAGE" ]; then - match="" - for p in $packages; do - p_low=$(echo "$p" | tr '[A-Z]' '[a-z]') - pkg_low=$(echo "$PACKAGE" | tr '[A-Z]' '[a-z]') - if [ "$p_low" = "$pkg_low" ]; then - match="$p" - break - fi - done - if [ -z "$match" ]; then - echo "ERROR: Package '$PACKAGE' not found. Valid packages: $packages" - exit 1 - fi - packages="$match" - fi - target_archs=(amd64 ppc64el) - for file in $packages - do - file_low=`echo $file | tr '[A-Z]' '[a-z]'` - if [ "$file" = "xCAT" -o "$file" = "xCAT-genesis-scripts" -o "$file" = "xCATsn" ]; then - target_archs="amd64 ppc64el" - else - target_archs="all" - fi - for target_arch in $target_archs - do - tar_orig="${file_low}_${ver}.orig.tar.gz" - if grep -q "3.0 (quilt)" "${file}/debian/source/format" && [ ! -f "$tar_orig" ]; then - tar czf "$tar_orig" --exclude debian -C "$file" . - fi - - if grep -q $file $update_log || [ "$BUILDALL" == 1 -o "$file" = "perl-xCAT" ]; then - rm -f $PKGDIR/${file_low}_*.$target_arch.deb - cd $file - CURDIR=$(pwd) - - find . -name '*.dch' -delete - deterministic_date=$(date -R -d "@$SOURCE_DATE_EPOCH" --utc 2>/dev/null || date -R --utc) - sed -i "1s/(.*)/(${pkg_version})/" debian/changelog - sed -i "s/^ -- .*/ -- $DEBFULLNAME <$DEBEMAIL> $deterministic_date/" debian/changelog - if [ "$target_arch" = "all" ]; then - #xcat probe use some functions shipped by xCAT, for below reasons we need to copy files to xCAT-probe directory - #1 make xcat probe code to be self-contained - #2 don't maintain two files for each script - #3 symbolic link can't work during package - if [ $file_low = "xcat-probe" ]; then - mkdir -p ${CURDIR}/lib/perl/xCAT/ - cp -f ${CURDIR}/../perl-xCAT/xCAT/NetworkUtils.pm ${CURDIR}/lib/perl/xCAT/ - cp -f ${CURDIR}/../perl-xCAT/xCAT/GlobalDef.pm ${CURDIR}/lib/perl/xCAT/ - cp -f ${CURDIR}/../perl-xCAT/xCAT/ServiceNodeUtils.pm ${CURDIR}/lib/perl/xCAT/ - fi - CURDIR=$(pwd) - cp ${CURDIR}/debian/control ${CURDIR}/debian/control.save.998 - # Magic string used here - sed -i -e "s#>= 2.13-snap000000000000#= ${pkg_version}#g" ${CURDIR}/debian/control - dpkg-buildpackage -rfakeroot -uc -us - mv ${CURDIR}/debian/control.save.998 ${CURDIR}/debian/control - else - if [ "$file" = "xCAT-genesis-scripts" ]; then - echo "Rename control file to build pkg: mv ${CURDIR}/debian/control-${target_arch} ${CURDIR}/debian/control" - cp ${CURDIR}/debian/control-${target_arch} ${CURDIR}/debian/control - elif [ "$file" = "xCAT" ]; then - # shipping bmcsetup and getipmi scripts as part of postscripts - files=("bmcsetup" "getipmi") - for f in "${files[@]}"; do - cp ${CURDIR}/../xCAT-genesis-scripts/usr/bin/$f ${CURDIR}/postscripts/$f - sed -i "s/xcat.genesis.$f/$f/g" ${CURDIR}/postscripts/$f - done - fi - CURDIR=$(pwd) - cp ${CURDIR}/debian/control ${CURDIR}/debian/control.save.998 - # Magic string used here - sed -i -e "s#>= 2.13-snap000000000000#= ${pkg_version}#g" ${CURDIR}/debian/control - dpkg-buildpackage -rfakeroot -uc -us -a$target_arch - mv ${CURDIR}/debian/control.save.998 ${CURDIR}/debian/control - if [ "$file" = "xCAT-genesis-scripts" ]; then - echo "Move control file back: mv ${CURDIR}/debian/control ${CURDIR}/debian/control-${target_arch}" - rm ${CURDIR}/debian/control - elif [ "$file" = "xCAT" ]; then - files=("bmcsetup" "getipmi") - for f in "${files[@]}"; do - rm -f ${CURDIR}/postscripts/$f - done - fi - fi - rc=$? - if [ $rc -gt 0 ]; then - echo "Error: $file build package failed exit code $rc" - exit $rc - fi - cd - - find $file -maxdepth 3 -type d -name "${file_low}*" | grep debian | xargs rm -rf - find $file -maxdepth 3 -type f -name "files" | grep debian | xargs rm -rf - mv ${file_low}* $PKGDIR/ - fi - done - done - - find $PKGDIR/* ! -name '*.deb' | xargs rm -f - fi - - if [ "$PROMOTE" = 1 ]; then - upload_dir="xcat-core" - tar_name="xcat-core-$ver.tar.bz2" - else - upload_dir="core-snap" - tar_name="core-debs-snap.tar.bz2" - fi - - echo "#################################" - echo "# Creating xcat-core repository #" - echo "#################################" - - #clean the repo directory - if [ -e $local_core_repo_path ]; then - rm -rf $local_core_repo_path - fi - mkdir -p $local_core_repo_path - cd $local_core_repo_path - mkdir conf - - for dist in $dists; do - # for all releases moving forward, support amd64 and ppc64el - tmp_out_arch="amd64 ppc64el" - if [ "$dist" = "saucy" ]; then - # for older releases of Ubuntu that does not support ppc64el - tmp_out_arch="amd64" - fi - cat << __EOF__ >> conf/distributions -Origin: xCAT internal repository -Label: xcat-core bazaar repository -Codename: $dist -Architectures: $tmp_out_arch -Components: main -Description: Repository automatically genereted conf -__EOF__ - - if [ "$GPGSIGN" = "0" ];then - #echo "GPGSIGN=$GPGSIGN specified, the repo will not be signed" - echo "" >> conf/distributions - else - keyid=$(gpg --list-keys --keyid-format long "$GPG_KEY_ID" | grep '^pub' | sed -e 's/.*\///' -e 's/ .*//') - echo "SignWith: $keyid" >> conf/distributions - echo "" >> conf/distributions - fi - done - - if [ -n "$GPG_HOME" ]; then - cat << __EOF__ > conf/options -verbose -basedir . -__EOF__ - else - cat << __EOF__ > conf/options -verbose -ask-passphrase -basedir . -__EOF__ - fi - - #import the deb packages into the repo - amd_files=`ls ../$package_dir_name/*.deb | grep -v "ppc64el"` - all_files=`ls ../$package_dir_name/*.deb` - for dist in $dists; do - deb_files=$all_files - if [ "$dist" = "saucy" ]; then - # for older releases of Ubuntu that does not support ppc64el - deb_files=$amd_files - fi - for file in $deb_files; do - reprepro -b ./ includedeb $dist $file; - done - done - #create the mklocalrepo script - cat << '__EOF__' > mklocalrepo.sh -. /etc/lsb-release -cd `dirname $0` -host_arch=`uname -m` -if [ "$host_arch" != "ppc64le" ];then - host_arch="amd64" -else - host_arch="ppc64el" -fi -echo deb [arch=$host_arch] file://"`pwd`" $DISTRIB_CODENAME main > /etc/apt/sources.list.d/xcat-core.list -__EOF__ - - chmod 775 mklocalrepo.sh - - # - # Add a buildinfo file into the tar.bz2 file to track information about the build - # - BUILDINFO=$local_core_repo_path/buildinfo - echo "VERSION=$ver" > $BUILDINFO - echo "RELEASE=$xcat_release" >> $BUILDINFO - echo "BUILD_TIME=$build_time" >> $BUILDINFO - echo "BUILD_MACHINE=$build_machine" >> $BUILDINFO - echo "COMMIT_ID=$commit_id" >> $BUILDINFO - echo "COMMIT_ID_LONG=$commit_id_long" >> $BUILDINFO - - #create the xcat-core.list file - - cd ../ - if ! grep xcat /etc/group ; then - groupadd xcat - fi - - chgrp -R root xcat-core - chmod -R g+w xcat-core - - #build the tar ball - echo "Creating `pwd`/$tar_name ..." - tar -hjcf $tar_name xcat-core - chgrp root $tar_name - chmod g+w $tar_name - - if [ -n "$DEST" ]; then - ln -sf $(basename `pwd`)/$tar_name ../$tar_name - if [ $? != 0 ]; then - echo "ERROR: Failed to make symbol link $DEST/$tar_name" - fi - fi - - if [ ! -e core-snap ]; then - ln -s xcat-core core-snap - fi - - cd $old_pwd - exit 0 -fi - -if [ "$d_flag" ] -then - echo "################################" - echo "# Creating xcat-dep repository #" - echo "################################" - - #the path of ubuntu xcat-dep deb packages on GSA - GSA="/gsa/pokgsa/projects/x/xcat/build/ubuntu/xcat-dep" - if [ ! -d $GSA ]; then - echo "build-ubunturepo: It appears that you do not have GSA to access the xcat-dep pkgs." - exit 1; - fi - - #define the dep source code path, core build target path and dep build target path - if [ -z "$DEST" ]; then - local_dep_repo_path="$curdir/../../xcat-dep/xcat-dep" - else - local_dep_repo_path="$DEST/xcat-dep/xcat-dep" - fi - - # Sync from the GSA master copy of the dep rpms - echo "Creating directory $local_dep_repo_path" - mkdir -p $local_dep_repo_path/ - - echo "Syncing RPMs from $GSA/ to $local_dep_repo_path/../ ..." - rsync -ilrtpu --delete $GSA/ $local_dep_repo_path/../ - if [ $? -ne 0 ]; then - echo "Error from rsync, cannot continue!" - exit 1 - fi - - #clean all old files - if [ -e $local_dep_repo_path ];then - rm -rf $local_dep_repo_path - fi - mkdir -p $local_dep_repo_path - cd $local_dep_repo_path - mkdir conf - - - #create the conf/distributions file - for dist in $dists; do - tmp_out_arch="amd64 ppc64el" - if [ "$dist" = "saucy" ]; then - # for older releases of Ubuntu that does not support ppc64el - tmp_out_arch="amd64" - fi - cat << __EOF__ >> conf/distributions -Origin: xCAT internal repository -Label: xcat-dep bazaar repository -Codename: $dist -Architectures: $tmp_out_arch -Components: main -Description: Repository automatically genereted conf -__EOF__ - - if [ "$GPGSIGN" = "0" ];then - echo "GPGSIGN=$GPGSIGN specified, the repo will not be signed" - echo "" >> conf/distributions - else - keyid=$(gpg --list-keys --keyid-format long "$GPG_KEY_ID" | grep '^pub' | sed -e 's/.*\///' -e 's/ .*//') - echo "SignWith: $keyid" >> conf/distributions - echo "" >> conf/distributions - fi - - done - - - - if [ -n "$GPG_HOME" ]; then - cat << __EOF__ > conf/options -verbose -basedir . -__EOF__ - else - cat << __EOF__ > conf/options -verbose -ask-passphrase -basedir . -__EOF__ - fi - - #import the deb packages into the repo - amd_files=`ls ../debs/*.deb | grep -v "ppc64el"` - all_files=`ls ../debs/*.deb` - for dist in $dists; do - deb_files=$all_files - if [ "$dist" = "saucy" ]; then - # for older releases of Ubuntu that does not support ppc64el - deb_files=$amd_files - fi - for file in $deb_files; do - reprepro -b ./ includedeb $dist $file; - done - done - - cat << '__EOF__' > mklocalrepo.sh -. /etc/lsb-release -cd `dirname $0` -host_arch=`uname -m` -if [ "$host_arch" != "ppc64le" ];then - host_arch="amd64" -else - host_arch="ppc64el" -fi -echo deb [arch=$host_arch] file://"`pwd`" $DISTRIB_CODENAME main > /etc/apt/sources.list.d/xcat-dep.list -__EOF__ - - chmod 775 mklocalrepo.sh - - cd .. - if ! grep xcat /etc/group ; then - groupadd xcat - fi - - chgrp -R root xcat-dep - chmod -R g+w xcat-dep - - #create the tar ball - dep_tar_name=xcat-dep-ubuntu-`date +%Y%m%d%H%M`.tar.bz2 - tar -hjcf $dep_tar_name xcat-dep - chgrp root $dep_tar_name - chmod g+w $dep_tar_name - - - USER="xcat" - SERVER="xcat.org" - FILES_PATH="files" - FRS="/var/www/${SERVER}/${FILES_PATH}" - APT_DIR="${FRS}/xcat" - APT_REPO_DIR="${APT_DIR}/repos/apt/devel" - - # Decide whether to upload the xcat-dep package or NOT (default is to NOT upload xcat-dep - if [ "$UP" != "1" ]; then - echo "Upload not specified, Done! (rerun with UP=1, to upload)" - cd $old_pwd - exit 0 - fi - - #upload the dep packages - i=0 - echo "Uploading debs from xcat-dep to ${APT_REPO_DIR}/xcat-dep/ ..." - while [ $((i+=1)) -le 5 ] && ! rsync -urLv --delete xcat-dep $USER@${SERVER}:${APT_REPO_DIR}/ - do : ; done - - #upload the tarball - i=0 - echo "Uploading $dep_tar_name to ${APT_DIR}/xcat-dep/2.x_Ubuntu/ ..." - while [ $((i+=1)) -le 5 ] && ! rsync -v --force $dep_tar_name $USER@${SERVER}:${APT_DIR}/xcat-dep/2.x_Ubuntu/ - do : ; done - - #upload the README file - cd debs - i=0 - echo "Uploading README to ${APT_DIR}/xcat-dep/2.x_Ubuntu/ ..." - while [ $((i+=1)) -le 5 ] && ! rsync -v --force README $USER@${SERVER}:${APT_DIR}/xcat-dep/2.x_Ubuntu/ - do : ; done - -fi - -cd $old_pwd -exit 0 diff --git a/builddebs.pl b/builddebs.pl new file mode 100644 index 000000000..35bd2c169 --- /dev/null +++ b/builddebs.pl @@ -0,0 +1,422 @@ +#!/usr/bin/perl +# Build the xcat-core Debian packages and assemble a signed apt repository. +# +# Replaces build-ubunturepo. The shape mirrors buildrpms.pl -- Getopt::Long options, +# one package list, build then index then sign -- so the two builders read the same way +# and share BuildUtils.pm. +# +# The central fact this design rests on: xcat-core debs are Perl. They are byte-identical +# for every Ubuntu release, so they are built ONCE and the same files are published into +# every codename. Only xCAT, xCATsn and xCAT-genesis-scripts carry an architecture, and +# even there the difference is packaging metadata, not compiled output. That is why this +# needs no sbuild and no per-codename chroot -- unlike xcat-dep, whose compiled packages +# genuinely differ per release. +use strict; +use warnings; +use feature 'say'; + +use Cwd qw(abs_path); +use File::Basename qw(basename); +use File::Copy qw(copy move); +use File::Path qw(make_path remove_tree); +use File::Spec; +use File::Temp qw(tempdir); +use Getopt::Long qw(GetOptions); +use POSIX qw(strftime); +use Pod::Usage qw(pod2usage); + +use FindBin; +use lib $FindBin::Bin; +use BuildUtils qw( + source_date_epoch snap_release deb_version + stage_probe_helpers XCAT_PROBE_HELPERS + deb_package_arches dist_arches default_dists + orig_tarball_name pin_control_version rewrite_changelog_header + reprepro_distributions reprepro_options + lock_id_for take_build_lock sh_quote +); + +# The xcat-core packages that ship as debs. xCAT-openbmc-py, xCAT-rmc and xCAT-release +# are rpm-only and are deliberately absent. +my @PACKAGES = qw( + perl-xCAT + xCAT + xCATsn + xCAT-buildkit + xCAT-client + xCAT-confluent + xCAT-genesis-scripts + xCAT-probe + xCAT-server + xCAT-test + xCAT-vlan +); + +# Releases the repo serves. The same debs are published into each; the list itself +# lives in BuildUtils so the builder and the tests cannot disagree about it. +my @DISTS = default_dists(); + +my %opts; +my (@cli_packages, @cli_dists); +GetOptions( + "dist=s@" => \@cli_dists, + "package=s@" => \@cli_packages, + "dest=s" => \$opts{dest}, + "builddir=s" => \$opts{builddir}, + "release=s" => \$opts{release}, + "gpg-sign" => \$opts{gpg_sign}, + "gpg-home=s" => \$opts{gpg_home}, + "gpg-key-name=s" => \$opts{gpg_key_name}, + "force" => \$opts{force}, + "verbose" => \$opts{verbose}, + "help" => \$opts{help}, +) or usage(); +usage(exitval => 0, verbose => 2) if $opts{help}; + +$opts{packages} = @cli_packages ? \@cli_packages : \@PACKAGES; +$opts{dists} = @cli_dists ? \@cli_dists : \@DISTS; +$opts{gpg_key_name} //= 'xCAT Signing Key'; + +for my $pkg ($opts{packages}->@*) { + die "FATAL: unknown package '$pkg'. Known: @PACKAGES\n" + unless grep { $_ eq $pkg } @PACKAGES; +} + +sub usage { + my (%args) = @_; + pod2usage( + -verbose => $args{verbose} // 1, + -exitval => $args{exitval} // 2, + (defined $args{message} ? (-message => "$args{message}\n") : ()), + ); +} + +sub sh { + my ($cmd) = @_; + say "+ $cmd" if $opts{verbose}; + return system($cmd); +} + +my $ROOT = abs_path($FindBin::Bin); +my $VERSION = do { open my $fh, '<', "$ROOT/Version" or die "Cannot read Version: $!\n"; + my $v = <$fh>; chomp $v; $v }; +my $EPOCH = source_date_epoch(); +my $RELEASE = $opts{release} || snap_release($EPOCH); +my $PKGVER = deb_version($VERSION, $RELEASE); +$ENV{SOURCE_DATE_EPOCH} = $EPOCH; + +# dpkg reads these for the changelog trailer. Fixed, so the packages do not carry +# whoever happened to run the build. +$ENV{DEBFULLNAME} ||= 'xCAT Build'; +$ENV{DEBEMAIL} ||= 'xcat@xcat.org'; +my $MAINTAINER = "$ENV{DEBFULLNAME} <$ENV{DEBEMAIL}>"; +my $DEB_DATE = strftime('%a, %d %b %Y %H:%M:%S +0000', gmtime($EPOCH)); + +# The build lock is scoped to this checkout, not the host -- see BuildUtils::lock_id_for. + +# ------------------------------------------------------------------ staging -- +# +# Each package is prepared, built, and put back exactly as it was. dpkg-buildpackage +# has no --define equivalent, so the version has to be written into the tree; leaving it +# there would dirty the checkout the CD pipeline builds from. +sub with_prepared_tree { + my ($pkg, $arch, $body) = @_; + my $dir = "$ROOT/$pkg"; + my @restore; + + my $save = sub { + my ($rel) = @_; + my $path = "$dir/$rel"; + return unless -f $path; + my $backup = "$path.build.save"; + copy($path, $backup) or die "Cannot back up $path: $!\n"; + push @restore, [$backup, $path]; + }; + + $save->('debian/control'); + $save->('debian/changelog'); + + # Pin the intra-xCAT dependencies to this exact build, so a partial upgrade cannot + # mix versions. + my $control = "$dir/debian/control"; + if (-f $control) { + my $text = do { open my $fh, '<', $control or die; local $/; <$fh> }; + open my $out, '>', $control or die "Cannot write $control: $!\n"; + print {$out} pin_control_version($text, $PKGVER); + close $out; + } + + my $changelog = "$dir/debian/changelog"; + if (-f $changelog) { + my $text = do { open my $fh, '<', $changelog or die; local $/; <$fh> }; + open my $out, '>', $changelog or die "Cannot write $changelog: $!\n"; + print {$out} rewrite_changelog_header($text, $PKGVER, $DEB_DATE, $MAINTAINER); + close $out; + } + unlink glob("$dir/debian/*.dch"); + + my @added; + # xcat-probe reuses functions shipped by xCAT. Copied, not linked: a symlink does + # not survive packaging, and maintaining two copies lets them drift. + if ($pkg eq 'xCAT-probe') { + push @added, stage_probe_helpers("$ROOT/perl-xCAT/xCAT", "$dir/lib/perl/xCAT"); + } + # xCAT ships the genesis bmcsetup/getipmi helpers as postscripts, renamed. + if ($pkg eq 'xCAT') { + for my $f (qw(bmcsetup getipmi)) { + my $src = "$ROOT/xCAT-genesis-scripts/usr/bin/$f"; + next unless -f $src; + my $dst = "$dir/postscripts/$f"; + my $text = do { open my $fh, '<', $src or die; local $/; <$fh> }; + $text =~ s/xcat\.genesis\.\Q$f\E/$f/g; + open my $out, '>', $dst or die "Cannot write $dst: $!\n"; + print {$out} $text; + close $out; + chmod 0755, $dst; + push @added, $dst; + } + } + # xCAT-genesis-scripts keeps a control file per architecture. + if ($pkg eq 'xCAT-genesis-scripts' && $arch ne 'all') { + my $per_arch = "$dir/debian/control-$arch"; + die "FATAL: $per_arch is missing\n" unless -f $per_arch; + my $text = do { open my $fh, '<', $per_arch or die; local $/; <$fh> }; + open my $out, '>', $control or die "Cannot write $control: $!\n"; + print {$out} pin_control_version($text, $PKGVER); + close $out; + } + + my $rc = eval { $body->($dir); 1 } ? 0 : 1; + my $err = $@; + + unlink @added; + for my $pair (reverse @restore) { + my ($backup, $path) = @$pair; + move($backup, $path) or warn "Could not restore $path: $!\n"; + } + die $err if $rc; + return; +} + +sub build_package { + my ($pkg, $arch, $pkgdir) = @_; + say "Building $pkg ($arch) $PKGVER"; + + with_prepared_tree($pkg, $arch, sub { + my ($dir) = @_; + + # A 3.0 (quilt) source package needs its .orig tarball beside the tree. + my $format = "$dir/debian/source/format"; + if (-f $format) { + my $text = do { open my $fh, '<', $format or die; local $/; <$fh> }; + if ($text =~ /3\.0 \(quilt\)/) { + my $tar = "$ROOT/" . orig_tarball_name($pkg, $PKGVER); + unless (-f $tar) { + sh(sprintf('tar czf %s --exclude debian -C %s .', + sh_quote($tar), sh_quote($dir))) == 0 + or die "FATAL: could not create $tar\n"; + } + } + } + + my $arch_flag = $arch eq 'all' ? '' : " -a$arch"; + my $quiet = $opts{verbose} ? '' : ' >/dev/null'; + sh("cd " . sh_quote($dir) . " && dpkg-buildpackage -rfakeroot -uc -us$arch_flag$quiet") == 0 + or die "FATAL: dpkg-buildpackage failed for $pkg ($arch)\n"; + }); + + # dpkg-buildpackage writes its output beside the package directory. + my $moved = 0; + for my $deb (glob("$ROOT/*.deb")) { + move($deb, "$pkgdir/" . basename($deb)) + or die "Cannot move $deb into $pkgdir: $!\n"; + $moved++; + } + die "FATAL: $pkg ($arch) produced no .deb\n" unless $moved; + # The rest of the dpkg output is build residue, not an artifact. + unlink glob("$ROOT/*.buildinfo"), glob("$ROOT/*.changes"), glob("$ROOT/*.dsc"), + glob("$ROOT/*.tar.xz"), glob("$ROOT/*.tar.gz"); + return $moved; +} + +# ------------------------------------------------------------- apt assembly -- +sub gpg_key_id { + my ($name) = @_; + my $out = `gpg --list-keys --keyid-format long @{[ sh_quote($name) ]} 2>/dev/null`; + my ($id) = $out =~ m{^pub\s+\S+/(\S+)}m; + die "FATAL: no gpg key matching '$name'\n" unless $id; + return $id; +} + +sub assemble_repo { + my ($pkgdir, $repodir) = @_; + make_path("$repodir/conf"); + + my $keyid; + if ($opts{gpg_sign}) { + local $ENV{GNUPGHOME} = $opts{gpg_home} if $opts{gpg_home}; + $keyid = gpg_key_id($opts{gpg_key_name}); + } + + open my $d, '>', "$repodir/conf/distributions" or die "Cannot write conf/distributions: $!\n"; + print {$d} reprepro_distributions($opts{dists}, $keyid); + close $d; + + open my $o, '>', "$repodir/conf/options" or die "Cannot write conf/options: $!\n"; + print {$o} reprepro_options($opts{gpg_home}); + close $o; + + local $ENV{GNUPGHOME} = $opts{gpg_home} if $opts{gpg_home}; + my @debs = sort glob("$pkgdir/*.deb"); + die "FATAL: no .deb files to publish in $pkgdir\n" unless @debs; + + for my $dist ($opts{dists}->@*) { + my %ok = map { $_ => 1 } dist_arches($dist); + for my $deb (@debs) { + # A release that predates an architecture must not be handed its packages. + next if basename($deb) =~ /_(\w+)\.deb\z/ && $1 ne 'all' && !$ok{$1}; + sh("cd " . sh_quote($repodir) . " && reprepro -b ./ includedeb " + . sh_quote($dist) . ' ' . sh_quote($deb)) == 0 + or die "FATAL: reprepro could not add $deb to $dist\n"; + } + } + return scalar @debs; +} + +sub write_repo_metadata { + my ($repodir) = @_; + + # Point apt at this directory, for a locally built repo. + open my $m, '>', "$repodir/mklocalrepo.sh" or die "Cannot write mklocalrepo.sh: $!\n"; + print {$m} <<'SCRIPT'; +. /etc/lsb-release +cd `dirname $0` +host_arch=`uname -m` +if [ "$host_arch" != "ppc64le" ];then + host_arch="amd64" +else + host_arch="ppc64el" +fi +echo deb [arch=$host_arch] file://"`pwd`" $DISTRIB_CODENAME main > /etc/apt/sources.list.d/xcat-core.list +SCRIPT + close $m; + chmod 0775, "$repodir/mklocalrepo.sh"; + + my $commit = `git -C @{[ sh_quote($ROOT) ]} rev-parse HEAD 2>/dev/null` || 'unknown'; + chomp $commit; + my $host = `hostname 2>/dev/null` || 'unknown'; chomp $host; + open my $b, '>', "$repodir/buildinfo" or die "Cannot write buildinfo: $!\n"; + print {$b} "VERSION=$VERSION\n", + "RELEASE=$RELEASE\n", + "BUILD_TIME=@{[ strftime('%a %b %d %H:%M:%S %Y', gmtime($EPOCH)) ]}\n", + "BUILD_MACHINE=$host\n", + "COMMIT_ID=@{[ substr($commit, 0, 7) ]}\n", + "COMMIT_ID_LONG=$commit\n"; + close $b; + return; +} + +# ----------------------------------------------------------------- main ------ +my $lock = take_build_lock($ROOT); + +my $dest = $opts{dest} ? abs_path($opts{dest}) : "$ROOT/dist/debs"; +my $pkgdir = "$dest/debs"; +my $repo = "$dest/xcat-core"; +make_path($pkgdir); +remove_tree($repo) if -d $repo && $opts{force}; +make_path($repo); + +say "xcat-core $PKGVER -> $dest"; +say "releases: @{[ join ' ', $opts{dists}->@* ]}"; + +for my $pkg ($opts{packages}->@*) { + for my $arch (deb_package_arches($pkg)) { + build_package($pkg, $arch, $pkgdir); + } +} + +my $count = assemble_repo($pkgdir, $repo); +write_repo_metadata($repo); +say "published $count package(s) into @{[ scalar $opts{dists}->@* ]} release(s) at $repo"; + +__END__ + +=head1 NAME + +builddebs.pl - build the xcat-core Debian packages and an apt repository + +=head1 SYNOPSIS + + perl builddebs.pl [options] + +=head1 DESCRIPTION + +Builds every xcat-core Debian package and assembles a C apt repository +containing them. + +xcat-core packages are Perl. The same binary serves every Ubuntu release, so each +package is built B and the resulting C<.deb> files are published into every +codename the repository declares. Only C, C and C +carry an architecture, and there the difference is packaging metadata rather than +compiled output. Consequently this builder needs no C and no per-codename +chroot. (xcat-dep is different: its packages are compiled, so it builds per codename.) + +Replaces C. The GSA upload paths, the C/C release +flows and the C<-d> xcat-dep repository mode were not carried over: publishing is done +by the CD pipeline's own deploy step, and xcat-dep is built from its own repository. + +=head1 OPTIONS + +=over + +=item B<--dist>=I + +Publish into this release. Repeatable. Defaults to focal, jammy, noble and resolute. + +=item B<--package>=I + +Build only this package. Repeatable. Defaults to every xcat-core deb package. + +=item B<--dest>=I + +Write the build under this directory: packages in C, the apt repository in +C. Defaults to C in the checkout. + +=item B<--release>=I + +Override the C release derived from the commit time. + +=item B<--gpg-sign> + +Sign the repository. Without it the repository is left unsigned. + +=item B<--gpg-home>=I + +Use this directory as C. Implies the key has no passphrase, so the build +never stops to prompt. + +=item B<--gpg-key-name>=I + +The key to sign with. Defaults to C. + +=item B<--force> + +Rebuild the repository from scratch rather than adding to what is there. + +=item B<--verbose> + +Echo each command. + +=item B<--help> + +This message. + +=back + +=head1 EXAMPLES + + ./builddebs.pl + ./builddebs.pl --dist noble --package perl-xCAT + ./builddebs.pl --dest /srv/out --gpg-sign --gpg-home /keys/xcat-gpg-home + +=cut diff --git a/docs/source/developers/guides/code/builds.rst b/docs/source/developers/guides/code/builds.rst index c69420d9d..d7f09e94b 100644 --- a/docs/source/developers/guides/code/builds.rst +++ b/docs/source/developers/guides/code/builds.rst @@ -32,10 +32,33 @@ emitted, because a source-only run has no binary packages to advertise. .. note:: - ``buildcore.sh``, ``makerpm`` and ``buildlocal.sh`` were removed in 2.19. - ``buildrpms.pl`` replaces all three, and its ``--source-only`` replaces the - old ``SRCONLY=1``. Debian and Ubuntu packages are still built by - ``./build-ubunturepo``. + ``buildcore.sh``, ``makerpm``, ``buildlocal.sh`` and ``build-ubunturepo`` + were removed in 2.19. ``buildrpms.pl`` replaces the first three, and its + ``--source-only`` replaces the old ``SRCONLY=1``; ``builddebs.pl`` replaces + ``build-ubunturepo``. + +Debian and Ubuntu packages +-------------------------- + +Build the ``.deb`` packages and an apt repository with ``builddebs.pl``:: + + cd xcat-core + ./builddebs.pl + +The packages land in ``dist/debs/debs/`` and the repository in +``dist/debs/xcat-core/``. Pass ``--dest`` to write them elsewhere, ``--dist`` to +limit which Ubuntu releases the repository serves, and ``--gpg-sign`` (with +``--gpg-home``) to sign it. ``./builddebs.pl --help`` lists the rest. + +xcat-core packages are Perl, so one build serves every Ubuntu release: the +packages are built **once** and the same files are published into every codename +the repository declares. Only ``xCAT``, ``xCATsn`` and ``xCAT-genesis-scripts`` +carry an architecture, and there the difference is packaging metadata rather than +compiled output. That is why this build needs no ``sbuild`` and no per-codename +chroot -- unlike xcat-deps, whose packages are compiled and genuinely differ per +release. + +Helpers shared by both builders live in ``BuildUtils.pm``. ``buildcore.sh`` builds the architecture specific packages (``xCAT``, ``xCATsn``, ``xCAT-genesis-scripts``) for every supported architecture, riscv64 included, with diff --git a/github_action_xcat_test.pl b/github_action_xcat_test.pl index 53e118fbf..e9ca2e5d7 100644 --- a/github_action_xcat_test.pl +++ b/github_action_xcat_test.pl @@ -29,13 +29,16 @@ my $GITHUB_API = "https://api.github.com"; # The workflow starts us in the checked out source tree. The unit tests under # xCAT-test/unit resolve xCAT modules and fixture files relative to that tree -# through FindBin, so they can only be run from a source tree -- but the tree -# does not survive the build. build-ubunturepo sets +# through FindBin, so they can only be run from a source tree. Take a copy +# before building and run the unit tests out of the copy. +# +# This used to be mandatory rather than tidy: build-ubunturepo set # local_core_repo_path="$curdir/../../xcat-core" -# which, under the work// layout GitHub checks out into, resolves -# to the checkout's own parent, and it then rm -rf's that path to make room for -# the apt repository. So take a copy of the tree before building and run the -# unit tests out of the copy. +# which, under the work// layout GitHub checks out into, resolved to +# the checkout's own parent, and it rm -rf'd that path to make room for the apt +# repository -- destroying the tree the tests need. builddebs.pl writes under +# dist/debs INSIDE the checkout and restores every file it edits, so the copy is +# now only isolating the tests from build residue. my $srcdir = getcwd(); my $unitsrc = ($ENV{'RUNNER_TEMP'} ? $ENV{'RUNNER_TEMP'} : "/tmp") . "/xcat-core-unitsrc"; @@ -322,7 +325,7 @@ sub preserve_source_tree{ sub build_xcat_core{ my @output; - my $cmd = "sudo ./build-ubunturepo -c UP=0 BUILDALL=1 GPGSIGN=0"; + my $cmd = "sudo ./builddebs.pl --force"; @output = runcmd("$cmd"); if($::RUNCMD_RC){ my $lastline = $output[-1]; diff --git a/travis.pl b/travis.pl index 5cab615ec..83febfe7f 100644 --- a/travis.pl +++ b/travis.pl @@ -289,7 +289,7 @@ sub build_xcat_core{ # } #} - my $cmd = "sudo ./build-ubunturepo -c UP=0 BUILDALL=1 GPGSIGN=0"; + my $cmd = "sudo ./builddebs.pl --force"; @output = runcmd("$cmd"); print ">>>>>Dumper the output of '$cmd'\n"; print Dumper \@output; diff --git a/xCAT-test/unit/build_ubunturepo_lock.t b/xCAT-test/unit/build_ubunturepo_lock.t deleted file mode 100644 index 8410fdfff..000000000 --- a/xCAT-test/unit/build_ubunturepo_lock.t +++ /dev/null @@ -1,155 +0,0 @@ -#!/usr/bin/env perl -# -# Unit test for the build-ubunturepo build lock (issue VersatusHPC/xcat-core#52). -# -# build-ubunturepo builds its packages in-place in its own source checkout, so the -# resource two concurrent builds contend for is the checkout -- not the host. The lock -# is therefore keyed on the checkout path ($curdir): builds of the SAME checkout -# fail-fast (they would corrupt each other), builds of DISTINCT checkouts get distinct -# locks and run in parallel (this is what lets the devel and stable Ubuntu CD lanes -# build concurrently on one host). -# -# The test extracts the two marked regions from build-ubunturepo VERBATIM so it -# exercises the real code, not a copy: -# * build-lock-id -- derives LOCKFILE from $curdir (pure string computation) -# * build-lock-acquire -- opens fd 8 on LOCKFILE and flock -n's it (needs a writable -# lock dir; skipped with a diag where /var/lock isn't writable) - -use strict; -use warnings; - -use FindBin; -use File::Temp qw(tempdir tempfile); -use IPC::Open2; -use Test::More; - -my $script = "$FindBin::Bin/../../build-ubunturepo"; -ok( -f $script, "found build-ubunturepo at $script" ) - or BAIL_OUT("build-ubunturepo not found"); - -my $src = do { local ( @ARGV, $/ ) = $script; <> }; - -my $workdir = tempdir( CLEANUP => 1 ); # scratch for temp scripts + capture files - -# --- pull the two marked regions out of the script verbatim ----------------- -sub region { - my ($name) = @_; - my ($body) = $src =~ /^# BEGIN \Q$name\E\n(.*?)^# END \Q$name\E\n/ms; - ok( defined $body, "extracted the '$name' region from build-ubunturepo" ) - or BAIL_OUT("marker region '$name' missing -- did the lock block change?"); - return $body; -} -my $id_region = region('build-lock-id'); -my $acquire_region = region('build-lock-acquire'); - -# Write a self-contained shell program (the given region(s) + a tail) to a temp file -# and return its path. Using a file avoids any quoting of the extracted shell. -my $prog_seq = 0; -sub write_prog { - my ($body) = @_; - my $path = "$workdir/prog." . $prog_seq++ . ".sh"; - open( my $fh, '>', $path ) or die "write $path: $!"; - print $fh "set -eu\ncurdir=\"\$1\"\n$body"; - close $fh; - return $path; -} - -sub lockfile_for { - my ($curdir) = @_; - my $prog = write_prog( "$id_region\nprintf '%s\\n' \"\$LOCKFILE\"\n" ); - my $out = qx{bash --noprofile --norc "$prog" "$curdir"}; - chomp $out; - return $out; -} - -# --- LOCKFILE derivation (always runnable -- no filesystem writes) ----------- -subtest 'lock is scoped per checkout, under /var/lock' => sub { - my $a = tempdir( CLEANUP => 1 ); - my $b = tempdir( CLEANUP => 1 ); - my $la = lockfile_for($a); - my $lb = lockfile_for($b); - - like( $la, qr{^/var/lock/xcatbld-[0-9a-f]{12}\.lock$}, - 'LOCKFILE is /var/lock/xcatbld-.lock' ); - is( lockfile_for($a), $la, 'same checkout => same lock (deterministic)' ); - isnt( $la, $lb, 'distinct checkouts => distinct locks' ); - - # the derivation must not create anything inside the checkout itself - ok( !glob("$a/*") && !glob("$a/.*xcatbld*"), - 'source checkout is left byte-pristine (no lock file written into it)' ); -}; - -# --- flock contention (needs a writable lock dir) --------------------------- -# The acquire region hard-codes /var/lock; run it only where that is writable -# (the GitHub Actions runner's /run/lock is sticky world-writable). Elsewhere, -# skip these two with a diag rather than failing on the environment. -my $probe = "/var/lock/.xcatbld-selftest.$$"; -my $lock_writable = open( my $pf, '>', $probe ); -if ($lock_writable) { close $pf; unlink $probe; } - -SKIP: { - skip "/var/lock is not writable here -- flock contention subtests need it", 2 - unless $lock_writable; - - # Launch a holder that acquires the lock for $curdir and blocks (holding fd 8) - # until we send it a newline on stdin. Returns ($pid, $to_child, $from_child, $first). - my $hold_prog = write_prog( - "$id_region\n$acquire_region\nprintf 'ACQUIRED %s\\n' \"\$LOCKFILE\"\nIFS= read -r _ || true\n" - ); - my $holder = sub { - my ($curdir) = @_; - my $pid = open2( my $out, my $in, 'bash', '--noprofile', '--norc', - $hold_prog, $curdir ); - my $first = <$out>; # blocks until the holder has the lock (or died) - return ( $pid, $in, $out, $first ); - }; - - # A contender that tries to acquire and, if it gets past the lock, prints MARK. - my $try_prog = sub { - my ($mark) = @_; - return write_prog( - "$id_region\n$acquire_region\nprintf '%s\\n' '$mark'\n" ); - }; - my $run = sub { - my ( $prog, $curdir ) = @_; - my $cap = "$workdir/cap." . $prog_seq++ . ".out"; - my $rc = system("bash --noprofile --norc \"$prog\" \"$curdir\" >\"$cap\" 2>&1"); - my $out = do { local ( @ARGV, $/ ) = $cap; <> }; - $out = '' unless defined $out; - return ( $rc, $out ); - }; - - subtest 'same checkout: second build fails fast' => sub { - my $dir = tempdir( CLEANUP => 1 ); - my ( $pid, $in, $out, $first ) = $holder->($dir); - like( $first, qr/^ACQUIRED /, 'first build acquired the checkout lock' ) - or BAIL_OUT('holder never acquired -- cannot test contention'); - - my ( $rc, $output ) = $run->( $try_prog->('SHOULD_NOT_REACH'), $dir ); - isnt( $rc, 0, 'second build of the SAME checkout exits non-zero' ); - like( $output, qr/Can't get lock/, 'it reports the lock contention' ); - like( $output, qr/\Q$dir\E/, 'the error names the contended checkout' ); - unlike( $output, qr/SHOULD_NOT_REACH/, 'it did not proceed into the build' ); - - print $in "\n"; close $in; # release the holder - waitpid( $pid, 0 ); - }; - - subtest 'distinct checkouts: both build in parallel' => sub { - my $dir_a = tempdir( CLEANUP => 1 ); - my $dir_b = tempdir( CLEANUP => 1 ); - my ( $pid, $in, $out, $first ) = $holder->($dir_a); - like( $first, qr/^ACQUIRED /, 'checkout A acquired its lock' ); - - # while A still holds its lock, B (a different checkout) must acquire too - my ( $rc, $output ) = $run->( $try_prog->('B_ACQUIRED'), $dir_b ); - is( $rc, 0, 'the other checkout acquires concurrently (exit 0)' ); - like( $output, qr/B_ACQUIRED/, 'it ran past the lock while A held its own' ); - unlike( $output, qr/Can't get lock/, 'no contention between distinct checkouts' ); - - print $in "\n"; close $in; - waitpid( $pid, 0 ); - }; -} - -done_testing; diff --git a/xCAT-test/unit/build_utils.t b/xCAT-test/unit/build_utils.t new file mode 100644 index 000000000..199872a68 --- /dev/null +++ b/xCAT-test/unit/build_utils.t @@ -0,0 +1,182 @@ +#!/usr/bin/env perl +# BuildUtils.pm: the helpers buildrpms.pl and builddebs.pl share. +# +# Every function here is pure, so every assertion below RUNS it. Nothing in this file +# reads the builders' source to check that they call it -- that would pass with the +# call removed. +use strict; +use warnings; + +use File::Path qw(make_path); +use File::Slurper qw(read_text write_text); +use File::Spec; +use File::Temp qw(tempdir); +use FindBin; +use lib "$FindBin::Bin/../lib"; +use lib "$FindBin::Bin/../.."; +use Test::More; + +BEGIN { use_ok('BuildUtils') or BAIL_OUT('BuildUtils.pm does not load'); } + +use BuildUtils qw( + source_date_epoch snap_release deb_version + stage_probe_helpers XCAT_PROBE_HELPERS + deb_package_arches dist_arches + orig_tarball_name pin_control_version rewrite_changelog_header + reprepro_distributions reprepro_options sh_quote +); + +# ------------------------------------------------------------------- versions -- + +is( snap_release(1756000000), 'snap202508240146', + 'the release is the commit time, to the minute, in UTC' ); +is( snap_release(1756000000), snap_release(1756000000), + 'the same commit time always gives the same release' ); +isnt( snap_release(1756000000), snap_release(1756000060), + 'a different commit time gives a different release' ); + +is( deb_version('2.19.0', 'snap202608240826'), '2.19.0-snap202608240826', + 'the deb version is the same Version-Release the rpms carry' ); + +# Gitepoch wins so every arch of one release stamps an identical epoch. +is( source_date_epoch(read_file => sub { '1756000000' }, + git_epoch => sub { '1700000000' }), 1756000000, + 'Gitepoch is preferred over the git log' ); +is( source_date_epoch(read_file => sub { undef }, + git_epoch => sub { "1700000000\n" }), 1700000000, + 'the git commit time is used when Gitepoch is absent' ); +is( source_date_epoch(read_file => sub { "not-a-number\n" }, + git_epoch => sub { '1700000000' }), 1700000000, + 'a corrupt Gitepoch falls through to the git log rather than being trusted' ); +is( source_date_epoch(read_file => sub { undef }, + git_epoch => sub { '' }, now => 42), 42, + 'with no git and no Gitepoch the caller-supplied clock is the last resort' ); + +# ---------------------------------------------------------------- deb layout -- + +is_deeply( [deb_package_arches('perl-xCAT')], ['all'], + 'a Perl package is built once, arch-independent' ); +is_deeply( [deb_package_arches('xCAT-probe')], ['all'], + 'xCAT-probe is arch-independent too' ); +for my $pkg (qw(xCAT xCATsn xCAT-genesis-scripts)) { + is_deeply( [deb_package_arches($pkg)], ['amd64', 'ppc64el'], + "$pkg is built per architecture" ); +} +is_deeply( [deb_package_arches(undef)], ['all'], + 'an undefined package name does not blow up the arch lookup' ); + +is_deeply( [dist_arches('noble')], ['amd64', 'ppc64el'], + 'a current release serves both architectures' ); +is_deeply( [dist_arches('saucy')], ['amd64'], + 'saucy predates ppc64el and serves only amd64' ); + +is( orig_tarball_name('xCAT-server', '2.19.0-snap1'), + 'xcat-server_2.19.0-snap1.orig.tar.gz', + 'the orig tarball name is lower-cased, as dpkg requires' ); + +# ------------------------------------------------------------------- control -- + +my $control = <<'CTRL'; +Package: xCAT +Depends: perl-xCAT (>= 2.13-snap000000000000), xCAT-server (>= 2.13-snap000000000000) +CTRL +my $pinned = pin_control_version($control, '2.19.0-snap202608240826'); +like( $pinned, qr/perl-xCAT \(= 2\.19\.0-snap202608240826\)/, + 'the sentinel dependency is pinned to this exact build' ); +unlike( $pinned, qr/2\.13-snap000000000000/, + 'no sentinel survives, so apt cannot satisfy it with an older xCAT' ); +is( scalar(() = $pinned =~ /= 2\.19\.0-snap202608240826/g), 2, + 'every occurrence is pinned, not just the first' ); +is( pin_control_version(undef, '2.19.0'), undef, + 'an absent control file is passed through rather than dying' ); + +# ----------------------------------------------------------------- changelog -- + +my $changelog = <<'CHANGELOG'; +xcat (2.18.0-snap000000000000) unstable; urgency=low + + * upstream + + -- Somebody Else Mon, 01 Jan 2024 00:00:00 +0000 + +xcat (2.17.0) unstable; urgency=low + + * older + + -- Somebody Else Mon, 01 Jan 2023 00:00:00 +0000 +CHANGELOG + +my $rewritten = rewrite_changelog_header( + $changelog, '2.19.0-snap202608240826', + 'Sat, 24 Aug 2026 08:26:40 +0000', 'xCAT Build '); + +like( $rewritten, qr/\Axcat \(2\.19\.0-snap202608240826\) unstable/, + 'the top stanza carries the version being built' ); +like( $rewritten, qr/^ -- xCAT Build Sat, 24 Aug 2026 08:26:40 \+0000$/m, + 'and the deterministic date, so two builds of one commit match' ); +like( $rewritten, qr/^xcat \(2\.17\.0\) unstable/m, + 'the older stanza is left alone -- the history is not ours to rewrite' ); + +# ------------------------------------------------------------------ reprepro -- + +my $dists = reprepro_distributions([qw(focal noble)], 'DEADBEEF'); +is( scalar(() = $dists =~ /^Codename:/mg), 2, 'one stanza per release' ); +like( $dists, qr/^Codename: focal\nArchitectures: amd64 ppc64el$/m, + 'a release declares both architectures, on the line after its codename' ); +is( scalar(() = $dists =~ /^SignWith: DEADBEEF$/mg), 2, + 'every stanza is signed when a key is given' ); + +my $unsigned = reprepro_distributions([qw(noble)], undef); +unlike( $unsigned, qr/SignWith/, 'no SignWith line without a key' ); +like( $unsigned, qr/\n\n\z/, 'stanzas stay blank-line separated so reprepro can parse them' ); + +like( reprepro_distributions([qw(saucy)], undef), qr/^Architectures: amd64$/m, + 'saucy declares only the architecture it had' ); + +like( reprepro_options(undef), qr/^ask-passphrase$/m, + 'an interactive build may be asked for a passphrase' ); +unlike( reprepro_options('/some/gnupghome'), qr/ask-passphrase/, + 'a build given a GNUPGHOME must never stop to prompt' ); +like( reprepro_options('/some/gnupghome'), qr/^basedir \.$/m, + 'and still sets its basedir' ); + +# --------------------------------------------------------------------- files -- + +{ + my $root = tempdir(CLEANUP => 1); + my $from = File::Spec->catdir($root, 'perl-xCAT', 'xCAT'); + my $to = File::Spec->catdir($root, 'xCAT-probe', 'lib', 'perl', 'xCAT'); + make_path($from); + write_text(File::Spec->catfile($from, $_), "package $_;\n1;\n") + for XCAT_PROBE_HELPERS; + + my @staged = stage_probe_helpers($from, $to); + is( scalar @staged, scalar(my @h = XCAT_PROBE_HELPERS), + 'every probe helper is staged' ); + for my $helper (XCAT_PROBE_HELPERS) { + my $path = File::Spec->catfile($to, $helper); + ok( -f $path, "$helper reaches the probe tree" ); + is( read_text($path), "package $helper;\n1;\n", + "$helper arrives with its content intact" ); + } + # Copied, not linked: a symlink does not survive packaging. + ok( !-l File::Spec->catfile($to, 'GlobalDef.pm'), + 'the helpers are real files, not symlinks' ); +} + +{ + my $root = tempdir(CLEANUP => 1); + my $ok = eval { + stage_probe_helpers(File::Spec->catdir($root, 'absent'), + File::Spec->catdir($root, 'dest')); + 1; + }; + ok( !$ok, 'a missing helper is fatal rather than a silently incomplete package' ); +} + +# -------------------------------------------------------------------- quoting -- + +is( sh_quote(q{it's}), q{'it'"'"'s'}, 'a single quote survives shell quoting' ); +is( sh_quote(undef), q{''}, 'undef quotes to the empty string' ); + +done_testing(); diff --git a/xCAT-test/unit/builddebs_lock.t b/xCAT-test/unit/builddebs_lock.t new file mode 100644 index 000000000..68ae09875 --- /dev/null +++ b/xCAT-test/unit/builddebs_lock.t @@ -0,0 +1,54 @@ +#!/usr/bin/env perl +# The deb build lock (VersatusHPC/xcat-core#52). +# +# builddebs.pl builds in-place in its own checkout -- it rewrites debian/changelog and +# debian/control and runs dpkg-buildpackage inside the package directories -- so two +# builds of the SAME checkout would corrupt each other and must fail fast, while two +# builds of DIFFERENT checkouts share nothing and must run concurrently. The historic +# host-global lock got that backwards and made the devel and stable CD lanes collide. +# +# This drives the real lock. The predecessor extracted a marked region out of +# build-ubunturepo with a regex and ran that; now the lock is a function, so it is +# called directly. +use strict; +use warnings; + +use File::Temp qw(tempdir); +use FindBin; +use lib "$FindBin::Bin/../lib"; +use lib "$FindBin::Bin/../.."; +use Test::More; + +use BuildUtils qw(lock_id_for take_build_lock); + +my $lockdir = tempdir(CLEANUP => 1); + +is( lock_id_for('/opt/builds/devel/xcat-core'), + lock_id_for('/opt/builds/devel/xcat-core'), + 'one checkout always maps to one lock id' ); +isnt( lock_id_for('/opt/builds/devel/xcat-core'), + lock_id_for('/opt/builds/stable/xcat-core'), + 'two checkouts map to different lock ids' ); +like( lock_id_for('/any/path'), qr/\A[0-9a-f]{12}\z/, + 'the id is filesystem-safe, so it can name a file' ); +is( length lock_id_for(''), 12, 'an empty path still yields an id rather than dying' ); + +# Same checkout: the second build must be refused. +my $devel = '/opt/builds/devel/xcat-core'; +my $first = take_build_lock($devel, $lockdir); +ok( $first, 'the first build of a checkout takes the lock' ); + +my $second = eval { take_build_lock($devel, $lockdir) }; +ok( !$second, 'a second build of the SAME checkout is refused' ); +like( $@, qr/already holds/, 'and says which checkout is already building' ); + +# Different checkout: must not be blocked by the first. +my $stable = eval { take_build_lock('/opt/builds/stable/xcat-core', $lockdir) }; +ok( $stable, 'a build of a DIFFERENT checkout runs concurrently' ); + +# Releasing lets the next build in. +close $first; +my $again = eval { take_build_lock($devel, $lockdir) }; +ok( $again, 'the lock is released when the handle is closed' ); + +done_testing(); diff --git a/xCAT-test/unit/ubuntu_2604_pkglist.t b/xCAT-test/unit/ubuntu_2604_pkglist.t index dccb4a6c0..29f6fa8e5 100644 --- a/xCAT-test/unit/ubuntu_2604_pkglist.t +++ b/xCAT-test/unit/ubuntu_2604_pkglist.t @@ -5,6 +5,9 @@ use FindBin; use File::Spec; use Test::More; +use lib "$FindBin::Bin/../.."; +use BuildUtils (); + my $repo_root = File::Spec->catdir( $FindBin::Bin, '..', '..' ); my @pkglist_files = qw( @@ -81,11 +84,14 @@ close($pre_fh); like( $pre, qr/id: efi-part\s+type: partition\s+device: disk-detected\s+size: 512M\s+flag: boot\s+number: 1\s+preserve: false\s+grub_device: true/s, 'subiquity UEFI storage marks the EFI partition as grub device' ); like( $pre, qr/id: efi-part-fs\s+type: format\s+fstype: fat32\s+volume: efi-part/s, 'subiquity UEFI storage formats ESP as fat32' ); -my $repo_builder = File::Spec->catfile( $repo_root, 'build-ubunturepo' ); -open( my $builder_fh, '<', $repo_builder ) or die "Unable to read $repo_builder: $!"; -my $builder = do { local $/; <$builder_fh> }; -close($builder_fh); - -like( $builder, qr/dists="\$\{DISTS:-[^"]*\bresolute\b[^"]*\}"/, 'Ubuntu repo builder includes resolute by default' ); +# The releases the deb builder serves by default. Read from BuildUtils, which is where +# the builder itself reads them, rather than matched against the source that sets them: +# the old assertion passed on any file containing that shell fragment, and broke on a +# reflow that changed nothing. +ok( scalar( grep { $_ eq 'resolute' } BuildUtils::default_dists() ), + 'the Ubuntu repository serves resolute by default' ); +like( BuildUtils::reprepro_distributions( [ BuildUtils::default_dists() ], undef ), + qr/^Codename: resolute$/m, + 'and a resolute stanza reaches conf/distributions' ); done_testing();