From 1a5192d0534b960e181aab72cb3925eced240d1c Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Fri, 4 Sep 2026 12:24:48 -0300 Subject: [PATCH] fix(xcat-core): build-ubunturepo is dead code that still looks buildable builddebs.pl replaced build-ubunturepo, and both CD pipelines prefer it: the fallback to build-ubunturepo fires only for refs that predate builddebs.pl, and such a ref carries its own copy. Nothing on this branch runs the script, so its presence only invites edits that never reach a build. The developer guide said it was kept as a differential oracle until the CD pipelines moved over. They have. Remove the script, and record the removal in the build guide beside the buildcore.sh, makerpm and buildlocal.sh entries. xcat_probe_package_payload.t asserted the Debian staging by matching a `cp -f` line in build-ubunturepo. builddebs.pl stages the helpers through XCAT::BuildUtils::stage_probe_helpers, so the test now calls that function and checks the files it produced. Verified by making stage_probe_helpers skip a helper: the assertion fails. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- build-ubunturepo | 711 ------------------ builddebs.pl | 4 +- docs/source/developers/guides/code/builds.rst | 6 +- github_action_xcat_test.pl | 11 +- xCAT-test/unit/builddebs_lock.t | 4 +- xCAT-test/unit/xcat_probe_package_payload.t | 14 +- 6 files changed, 18 insertions(+), 732 deletions(-) delete mode 100755 build-ubunturepo diff --git a/build-ubunturepo b/build-ubunturepo deleted file mode 100755 index 8a36a17e4..000000000 --- a/build-ubunturepo +++ /dev/null @@ -1,711 +0,0 @@ -#!/bin/bash -# Update GSA Ubuntu Repositories or create a local repository -# -# Author: Leonardo Tonetto (tonetto@linux.vnet.ibm.com) -# Revisor: Arif Ali (aali@ocf.co.uk) -# -# -# Getting Started: -# - Clone the xcat-core git repository under a directory named "xcat-core/src" -# - make sure reprepro is installed on the build machine -# - Run this script from the local git repository you just created. -# ./build-ubunturepo -c BUILDALL=1 - -# Usage: attr=value attr=value ... ./build-ubunturepo { -c | -d } -# PROMOTE=1 - if the attribute "PROMOTE" is specified, means an official dot release. This does not -# actually build xcat, just uploads the most recent snap build to http://xcat.org/files/xcat/ . -# If not specified, a snap build is assumed, which uploads to https://xcat.org/files/xcat/ -# PREGA=1 - use this option with PROMOTE=1 on a branch that already has a released dot release, but this -# build is a GA candidate build, not to be released yet. This will result in the tarball -# being uploaded to http://xcat.org/files/xcat/repos/apt -# (but the tarball file name will be like a released tarball, not a snap build). -# When you are ready to release this build, use PROMOTE=1 without PREGA -# BUILDALL=1 - build all rpms, whether they changed or not. Should be used for snap builds that are in -# prep for a release. -# GPGSIGN=0 - Do not sign the repo in the end of the build. The repo will be signed by default -# -# LOCAL_KEY=1 Use local keys to sign repo instead of WGET from GSA. By default use GSA. -# -# GPG_HOME= - Use the specified directory as GNUPGHOME for signing (no passphrase assumed). -# Bypasses GSA download and LOCAL_KEY. -# -# SETUP=1 Setup environment for build. By default do not setup environment. -# -# LOG= - provide an LOG file option to redirect some output into log file -# -# DEST= - provide a directory to contains the build result -# -# Running builds in parallel on one host: the build lock is scoped to the source checkout (see -# the "build-lock" block below), so two builds from DIFFERENT -# checkouts -- e.g. the devel and stable Ubuntu CD lanes -- run -# concurrently, while two builds of the SAME checkout still fail-fast -# (they build in-place and would corrupt each other). For parallel -# builds give each a separate checkout and a separate output tree -# (a distinct DEST). GPG_HOME may be SHARED between parallel builds -# -- it is used read-only for signing. -# -# For the dependency packages 1. All the xcat dependency deb packages should be uploaded to -# "pokgsa/projects/x/xcat/build/ubuntu/xcat-dep/debs/" on GSA -# 2. run ./build-ubunturepo -d -# -# 3. the built xcat-dep deb packages tarball can be found in "../../xcat-dep" -# related to the path of this script -############################ -printusage() -{ - printf "Usage: %s {-c | -d} \n" $(basename $0) >&2 - echo " -c : Build the xcat-core packages and create the repo" - echo " -d : Create the xcat-dep repo." -} -# For the purpose of getting the distribution name -if [[ ! -f /etc/lsb-release ]]; then - echo "ERROR: Could not find /etc/lsb-release, is this script executed on a Ubuntu machine?" - exit 1 -fi -. /etc/lsb-release - -export HOME=/root - - -# Process cmd line variable assignments, assigning each attr=val pair to a variable of same name -for i in $*; do - echo $i | grep '=' -q - if [ $? != 0 ];then - continue - fi - # upper case the variable name - varstring=`echo "$i"|cut -d '=' -f 1|tr '[a-z]' '[A-Z]'`=`echo "$i"|cut -d '=' -f 2` - export $varstring -done - -#Setup environment so the xcat-deps can be built on a FVT test machine -if [ "$SETUP" = "1" ];then - #Mount GSA - POKGSA="/gsa/pokgsa" - POKGSA2="/gsa/pokgsa-p2" - POKGSAIBM="pokgsa.ibm.com" - if [ ! -d $POKGSA ];then - mkdir -p $POKGSA - mount ${POKGSAIBM}:${POKGSA} ${POKGSA} - fi - if [ ! -d $POKGSA2 ];then - mkdir -p $POKGSA2 - mount ${POKGSAIBM}:${POKGSA2} ${POKGSA2} - fi - - # Verify needed packages installed - REPREPO="reprepro" - DEVSCRIPTS="devscripts" - DEBHELPER="debhelper" - QUILT="quilt" - - apt-get -y install $REPREPO $DEVSCRIPTS $DEBHELPER $QUILT - - echo "Finished setup for xcat-dep build. Rerun this script with SETUP=0 LOCAL_KEY=1 flags" - exit 1 -fi - -# Check the necessary packages before starting the build -declare -a packages=( "reprepro" "devscripts" "debhelper" "libsoap-lite-perl" "libdbi-perl" "quilt" "git") - -for package in ${packages[@]}; do - RC=`dpkg -l | grep $package >> /dev/null 2>&1; echo $?` - if [[ ${RC} != 0 ]]; then - echo "ERROR: Could not find $package, install using 'apt-get install $package' to continue" - exit 1 - fi -done - -# Supported distributions. Set DISTS="jammy noble resolute" to limit local validation builds. -dists="${DISTS:-saucy trusty utopic xenial bionic focal jammy noble resolute}" - -# GPG key used to sign the apt repo (reprepro SignWith). Defaults to the historic -# name. Override with GPG_KEY_ID= (space-free, since it is passed via -# the attr=value parser above), e.g. GPG_KEY_ID=xcat-build@xcat.org -GPG_KEY_ID="${GPG_KEY_ID:-xCAT Automatic Signing Key}" - -c_flag= # xcat-core (trunk-delvel) path -d_flag= # xcat-dep (trunk) path -r_flag= #genesis base rpm package path - -while getopts 'cdr:' OPTION -do - case $OPTION in - c) c_flag=1 - ;; - d) d_flag=1 - ;; - r) r_flag=1 - genesis_rpm_path="$OPTARG" - ;; - ?) printusage - exit 2 - ;; - esac -done -shift $(($OPTIND - 1)) - -if [ -z "$c_flag" -a -z "$d_flag" ];then - printusage - exit 2 -fi - -if [ "$c_flag" -a "$d_flag" ];then - printusage - exit 2 -fi - -if [ -z "$BUILDALL" ]; then - BUILDALL=1 -fi - -# Find where this script is located to set some build variables -old_pwd=`pwd` -cd `dirname $0` -curdir=`pwd` - -# Scope the build lock to THIS checkout. build-ubunturepo builds the packages in-place -# in its own source tree (it rewrites debian/changelog and debian/control, drops -# *.orig.tar.gz at the checkout root and runs dpkg-buildpackage inside the package -# dirs), so the resource two builds actually contend for is the checkout -- not the -# host. The historic single /var/lock/xcatbld.lock was host-global and fail-fast, so -# two builds from *different* checkouts (e.g. the devel and stable Ubuntu CD lanes on -# one build host) collided and the loser failed the pipeline even though they share -# nothing. Key the lock on the checkout path instead: builds of the SAME checkout -# still fail-fast (they would corrupt each other in-place), while builds of DISTINCT -# checkouts get distinct locks and run in parallel. The lock file stays on the local -# /var/lock (reliable flock; the checkout may live on NFS/virtiofs where flock is not) -# and the source tree is left byte-pristine. -# -# NOTE: the two marked regions below are extracted verbatim and exercised by the unit -# test xCAT-test/unit/build_ubunturepo_lock.t (which runs them with a chosen $curdir) -# -- keep the markers, and keep each region self-contained. -# BEGIN build-lock-id -lock_id_for() { printf '%s' "$1" | md5sum | cut -c1-12; } -LOCKFILE="/var/lock/xcatbld-$(lock_id_for "$curdir").lock" -# END build-lock-id -# BEGIN build-lock-acquire -exec 8>"$LOCKFILE" -if ! flock -n 8; then - echo "ERROR: Can't get lock $LOCKFILE for checkout $curdir. Another build is already using this checkout. Exiting...." - exit 1 -fi -# END build-lock-acquire - -# for the git case, query the current branch and set REL (changing master to devel if necessary) -function setbranch { - # Get the current branch name. safe.directory='*' so this still works when the - # build runs as root against a repo owned by another user (otherwise git errors - # with "dubious ownership", returns empty, and REL collapses to an unstable value). - branch=`git -c safe.directory='*' rev-parse --abbrev-ref HEAD 2>/dev/null` - if [ "$branch" = "master" ]; then - REL="devel" - elif [ "$branch" = "HEAD" ] || [ -z "$branch" ]; then - # Special handling when in a 'detached HEAD' state - branch=`git -c safe.directory='*' describe --abbrev=0 HEAD 2>/dev/null` - [[ -n "$branch" ]] && REL=`echo $branch|cut -d. -f 1,2` - else - REL=$branch - fi -} - -WGET_CMD="wget" -if [ ! -z ${LOG} ]; then - WGET_CMD="wget -o ${LOG}" -fi - -if [ "$GPGSIGN" = "0" ];then - echo "GPGSIGN=$GPGSIGN specified, skip gnupg key downloading" -elif [ -n "$GPG_HOME" ];then - echo "GPG_HOME=$GPG_HOME specified, using provided GNUPGHOME" - export GNUPGHOME="$GPG_HOME" -else - #sync the gpg key to the build machine local - gsa_url=http://pokgsa.ibm.com/projects/x/xcat/build/linux - mkdir -p $HOME/.gnupg - for key_name in pubring.gpg secring.gpg trustdb.gpg; do - if [ "$LOCAL_KEY" = "1" ];then - # Keys are already in the local $HOME/.gnupg directory - chmod 600 $HOME/.gnupg/$key_name - else - # Need to download keys from GSA - if [ ! -f $HOME/.gnupg/$key_name ] || [ `wc -c $HOME/.gnupg/$key_name|cut -f 1 -d' '` == 0 ]; then - rm -f $HOME/.gnupg/$key_name - ${WGET_CMD} -P $HOME/.gnupg $gsa_url/keys/$key_name - chmod 600 $HOME/.gnupg/$key_name - fi - fi - done -fi - -REL=xcat-core -if [ "$c_flag" ] -then - setbranch - # Sanitize REL into a stable, filesystem-safe token: replace any character that - # isn't [A-Za-z0-9._-] (e.g. the '/' in a branch like feat/ubuntu-e2e, which would - # otherwise create nested dirs) with '-', and never let it be empty. - REL=${REL//[^A-Za-z0-9._-]/-} - [ -z "$REL" ] && REL="local" - package_dir_name=debs$REL - - #define the dep source code path, core build target path and dep build target path - if [ -z "$DEST" ]; then - local_core_repo_path="$curdir/../../xcat-core" - PKGDIR="../../$package_dir_name" - else - local_core_repo_path="$DEST/$package_dir_name/xcat-core" - PKGDIR="$DEST/$package_dir_name/$package_dir_name" - fi - if [ ! -d "$PKGDIR" ];then - mkdir -p "$PKGDIR" - fi - - echo "#############################################################" - echo "Building xcat-core on branch ($REL) to $local_core_repo_path" - echo "#############################################################" - if [ "$PROMOTE" != 1 ]; then - code_change=0 - update_log='' - - if [ -z "$GITUP" ];then - update_log=../coregitup - echo "git pull > $update_log" - git pull > $update_log - else - update_log=$GITUP - fi - - if ! grep -q 'Already up-to-date' $update_log; then - code_change=1 - fi - ver=`cat Version` - short_ver=`cat Version|cut -d. -f 1,2` - short_short_ver=`cat Version|cut -d. -f 1` - commit_id_long=`git rev-parse HEAD` - commit_id="${commit_id_long:0:7}" - if [ -f Gitepoch ]; then - source_date_epoch=$(cat Gitepoch) - else - source_date_epoch=$(git log -1 --format=%ct HEAD 2>/dev/null || date +%s) - fi - export SOURCE_DATE_EPOCH="$source_date_epoch" - export DEBEMAIL="xcat-build@xcat.org" - export DEBFULLNAME="xCAT Build" - build_time=$(date -d "@$source_date_epoch" --utc '+%Y-%m-%dT%H:%M:%SZ' 2>/dev/null || date -u) - build_machine=`hostname` - - if [ $code_change == 0 -a "$UP" != 1 -a "$BUILDALL" != 1 ]; then - echo "Nothing new detected. Exiting...." - exit 0 - fi - - echo "###############################" - echo "# Building xcat-core packages #" - echo "###############################" - - #the package type: local | snap | alpha - #the build introduce string - build_string="Snap_Build" - if [ -f Release ]; then - xcat_release=$(cat Release) - else - xcat_release="snap$(date -d "@$source_date_epoch" --utc '+%Y%m%d%H%M')" - fi - pkg_version="${ver}-${xcat_release}" - - packages="xCAT-client xCAT-genesis-scripts perl-xCAT xCAT-server xCAT xCATsn xCAT-test xCAT-buildkit xCAT-vlan xCAT-confluent xCAT-probe" - if [ -n "$PACKAGE" ]; then - match="" - for p in $packages; do - p_low=$(echo "$p" | tr '[A-Z]' '[a-z]') - pkg_low=$(echo "$PACKAGE" | tr '[A-Z]' '[a-z]') - if [ "$p_low" = "$pkg_low" ]; then - match="$p" - break - fi - done - if [ -z "$match" ]; then - echo "ERROR: Package '$PACKAGE' not found. Valid packages: $packages" - exit 1 - fi - packages="$match" - fi - target_archs=(amd64 ppc64el) - for file in $packages - do - file_low=`echo $file | tr '[A-Z]' '[a-z]'` - if [ "$file" = "xCAT" -o "$file" = "xCAT-genesis-scripts" -o "$file" = "xCATsn" ]; then - target_archs="amd64 ppc64el" - else - target_archs="all" - fi - for target_arch in $target_archs - do - tar_orig="${file_low}_${ver}.orig.tar.gz" - if grep -q "3.0 (quilt)" "${file}/debian/source/format" && [ ! -f "$tar_orig" ]; then - tar czf "$tar_orig" --exclude debian -C "$file" . - fi - - if grep -q $file $update_log || [ "$BUILDALL" == 1 -o "$file" = "perl-xCAT" ]; then - rm -f $PKGDIR/${file_low}_*.$target_arch.deb - cd $file - CURDIR=$(pwd) - - find . -name '*.dch' -delete - deterministic_date=$(date -R -d "@$SOURCE_DATE_EPOCH" --utc 2>/dev/null || date -R --utc) - sed -i "1s/(.*)/(${pkg_version})/" debian/changelog - sed -i "s/^ -- .*/ -- $DEBFULLNAME <$DEBEMAIL> $deterministic_date/" debian/changelog - if [ "$target_arch" = "all" ]; then - #xcat probe use some functions shipped by xCAT, for below reasons we need to copy files to xCAT-probe directory - #1 make xcat probe code to be self-contained - #2 don't maintain two files for each script - #3 symbolic link can't work during package - if [ $file_low = "xcat-probe" ]; then - mkdir -p ${CURDIR}/lib/perl/xCAT/ - cp -f ${CURDIR}/../perl-xCAT/xCAT/CommandUtils.pm ${CURDIR}/lib/perl/xCAT/ - cp -f ${CURDIR}/../perl-xCAT/xCAT/NetworkUtils.pm ${CURDIR}/lib/perl/xCAT/ - cp -f ${CURDIR}/../perl-xCAT/xCAT/GlobalDef.pm ${CURDIR}/lib/perl/xCAT/ - cp -f ${CURDIR}/../perl-xCAT/xCAT/ServiceNodeUtils.pm ${CURDIR}/lib/perl/xCAT/ - fi - CURDIR=$(pwd) - cp ${CURDIR}/debian/control ${CURDIR}/debian/control.save.998 - # Magic string used here - sed -i -e "s#>= 2.13-snap000000000000#= ${pkg_version}#g" ${CURDIR}/debian/control - dpkg-buildpackage -rfakeroot -uc -us - mv ${CURDIR}/debian/control.save.998 ${CURDIR}/debian/control - else - if [ "$file" = "xCAT-genesis-scripts" ]; then - echo "Rename control file to build pkg: mv ${CURDIR}/debian/control-${target_arch} ${CURDIR}/debian/control" - cp ${CURDIR}/debian/control-${target_arch} ${CURDIR}/debian/control - elif [ "$file" = "xCAT" ]; then - # shipping bmcsetup and getipmi scripts as part of postscripts - files=("bmcsetup" "getipmi") - for f in "${files[@]}"; do - cp ${CURDIR}/../xCAT-genesis-scripts/usr/bin/$f ${CURDIR}/postscripts/$f - sed -i "s/xcat.genesis.$f/$f/g" ${CURDIR}/postscripts/$f - done - fi - CURDIR=$(pwd) - cp ${CURDIR}/debian/control ${CURDIR}/debian/control.save.998 - # Magic string used here - sed -i -e "s#>= 2.13-snap000000000000#= ${pkg_version}#g" ${CURDIR}/debian/control - dpkg-buildpackage -rfakeroot -uc -us -a$target_arch - mv ${CURDIR}/debian/control.save.998 ${CURDIR}/debian/control - if [ "$file" = "xCAT-genesis-scripts" ]; then - echo "Move control file back: mv ${CURDIR}/debian/control ${CURDIR}/debian/control-${target_arch}" - rm ${CURDIR}/debian/control - elif [ "$file" = "xCAT" ]; then - files=("bmcsetup" "getipmi") - for f in "${files[@]}"; do - rm -f ${CURDIR}/postscripts/$f - done - fi - fi - rc=$? - if [ $rc -gt 0 ]; then - echo "Error: $file build package failed exit code $rc" - exit $rc - fi - cd - - find $file -maxdepth 3 -type d -name "${file_low}*" | grep debian | xargs rm -rf - find $file -maxdepth 3 -type f -name "files" | grep debian | xargs rm -rf - mv ${file_low}* $PKGDIR/ - fi - done - done - - find $PKGDIR/* ! -name '*.deb' | xargs rm -f - fi - - if [ "$PROMOTE" = 1 ]; then - upload_dir="xcat-core" - tar_name="xcat-core-$ver.tar.bz2" - else - upload_dir="core-snap" - tar_name="core-debs-snap.tar.bz2" - fi - - echo "#################################" - echo "# Creating xcat-core repository #" - echo "#################################" - - #clean the repo directory - if [ -e $local_core_repo_path ]; then - rm -rf $local_core_repo_path - fi - mkdir -p $local_core_repo_path - cd $local_core_repo_path - mkdir conf - - for dist in $dists; do - # for all releases moving forward, support amd64 and ppc64el - tmp_out_arch="amd64 ppc64el" - if [ "$dist" = "saucy" ]; then - # for older releases of Ubuntu that does not support ppc64el - tmp_out_arch="amd64" - fi - cat << __EOF__ >> conf/distributions -Origin: xCAT internal repository -Label: xcat-core bazaar repository -Codename: $dist -Architectures: $tmp_out_arch -Components: main -Description: Repository automatically genereted conf -__EOF__ - - if [ "$GPGSIGN" = "0" ];then - #echo "GPGSIGN=$GPGSIGN specified, the repo will not be signed" - echo "" >> conf/distributions - else - keyid=$(gpg --list-keys --keyid-format long "$GPG_KEY_ID" | grep '^pub' | sed -e 's/.*\///' -e 's/ .*//') - echo "SignWith: $keyid" >> conf/distributions - echo "" >> conf/distributions - fi - done - - if [ -n "$GPG_HOME" ]; then - cat << __EOF__ > conf/options -verbose -basedir . -__EOF__ - else - cat << __EOF__ > conf/options -verbose -ask-passphrase -basedir . -__EOF__ - fi - - #import the deb packages into the repo - amd_files=`ls ../$package_dir_name/*.deb | grep -v "ppc64el"` - all_files=`ls ../$package_dir_name/*.deb` - for dist in $dists; do - deb_files=$all_files - if [ "$dist" = "saucy" ]; then - # for older releases of Ubuntu that does not support ppc64el - deb_files=$amd_files - fi - for file in $deb_files; do - reprepro -b ./ includedeb $dist $file; - done - done - #create the mklocalrepo script - cat << '__EOF__' > mklocalrepo.sh -. /etc/lsb-release -cd `dirname $0` -host_arch=`uname -m` -if [ "$host_arch" != "ppc64le" ];then - host_arch="amd64" -else - host_arch="ppc64el" -fi -echo deb [arch=$host_arch] file://"`pwd`" $DISTRIB_CODENAME main > /etc/apt/sources.list.d/xcat-core.list -__EOF__ - - chmod 775 mklocalrepo.sh - - # - # Add a buildinfo file into the tar.bz2 file to track information about the build - # - BUILDINFO=$local_core_repo_path/buildinfo - echo "VERSION=$ver" > $BUILDINFO - echo "RELEASE=$xcat_release" >> $BUILDINFO - echo "BUILD_TIME=$build_time" >> $BUILDINFO - echo "BUILD_MACHINE=$build_machine" >> $BUILDINFO - echo "COMMIT_ID=$commit_id" >> $BUILDINFO - echo "COMMIT_ID_LONG=$commit_id_long" >> $BUILDINFO - - #create the xcat-core.list file - - cd ../ - if ! grep xcat /etc/group ; then - groupadd xcat - fi - - chgrp -R root xcat-core - chmod -R g+w xcat-core - - #build the tar ball - echo "Creating `pwd`/$tar_name ..." - tar -hjcf $tar_name xcat-core - chgrp root $tar_name - chmod g+w $tar_name - - if [ -n "$DEST" ]; then - ln -sf $(basename `pwd`)/$tar_name ../$tar_name - if [ $? != 0 ]; then - echo "ERROR: Failed to make symbol link $DEST/$tar_name" - fi - fi - - if [ ! -e core-snap ]; then - ln -s xcat-core core-snap - fi - - cd $old_pwd - exit 0 -fi - -if [ "$d_flag" ] -then - echo "################################" - echo "# Creating xcat-dep repository #" - echo "################################" - - #the path of ubuntu xcat-dep deb packages on GSA - GSA="/gsa/pokgsa/projects/x/xcat/build/ubuntu/xcat-dep" - if [ ! -d $GSA ]; then - echo "build-ubunturepo: It appears that you do not have GSA to access the xcat-dep pkgs." - exit 1; - fi - - #define the dep source code path, core build target path and dep build target path - if [ -z "$DEST" ]; then - local_dep_repo_path="$curdir/../../xcat-dep/xcat-dep" - else - local_dep_repo_path="$DEST/xcat-dep/xcat-dep" - fi - - # Sync from the GSA master copy of the dep rpms - echo "Creating directory $local_dep_repo_path" - mkdir -p $local_dep_repo_path/ - - echo "Syncing RPMs from $GSA/ to $local_dep_repo_path/../ ..." - rsync -ilrtpu --delete $GSA/ $local_dep_repo_path/../ - if [ $? -ne 0 ]; then - echo "Error from rsync, cannot continue!" - exit 1 - fi - - #clean all old files - if [ -e $local_dep_repo_path ];then - rm -rf $local_dep_repo_path - fi - mkdir -p $local_dep_repo_path - cd $local_dep_repo_path - mkdir conf - - - #create the conf/distributions file - for dist in $dists; do - tmp_out_arch="amd64 ppc64el" - if [ "$dist" = "saucy" ]; then - # for older releases of Ubuntu that does not support ppc64el - tmp_out_arch="amd64" - fi - cat << __EOF__ >> conf/distributions -Origin: xCAT internal repository -Label: xcat-dep bazaar repository -Codename: $dist -Architectures: $tmp_out_arch -Components: main -Description: Repository automatically genereted conf -__EOF__ - - if [ "$GPGSIGN" = "0" ];then - echo "GPGSIGN=$GPGSIGN specified, the repo will not be signed" - echo "" >> conf/distributions - else - keyid=$(gpg --list-keys --keyid-format long "$GPG_KEY_ID" | grep '^pub' | sed -e 's/.*\///' -e 's/ .*//') - echo "SignWith: $keyid" >> conf/distributions - echo "" >> conf/distributions - fi - - done - - - - if [ -n "$GPG_HOME" ]; then - cat << __EOF__ > conf/options -verbose -basedir . -__EOF__ - else - cat << __EOF__ > conf/options -verbose -ask-passphrase -basedir . -__EOF__ - fi - - #import the deb packages into the repo - amd_files=`ls ../debs/*.deb | grep -v "ppc64el"` - all_files=`ls ../debs/*.deb` - for dist in $dists; do - deb_files=$all_files - if [ "$dist" = "saucy" ]; then - # for older releases of Ubuntu that does not support ppc64el - deb_files=$amd_files - fi - for file in $deb_files; do - reprepro -b ./ includedeb $dist $file; - done - done - - cat << '__EOF__' > mklocalrepo.sh -. /etc/lsb-release -cd `dirname $0` -host_arch=`uname -m` -if [ "$host_arch" != "ppc64le" ];then - host_arch="amd64" -else - host_arch="ppc64el" -fi -echo deb [arch=$host_arch] file://"`pwd`" $DISTRIB_CODENAME main > /etc/apt/sources.list.d/xcat-dep.list -__EOF__ - - chmod 775 mklocalrepo.sh - - cd .. - if ! grep xcat /etc/group ; then - groupadd xcat - fi - - chgrp -R root xcat-dep - chmod -R g+w xcat-dep - - #create the tar ball - dep_tar_name=xcat-dep-ubuntu-`date +%Y%m%d%H%M`.tar.bz2 - tar -hjcf $dep_tar_name xcat-dep - chgrp root $dep_tar_name - chmod g+w $dep_tar_name - - - USER="xcat" - SERVER="xcat.org" - FILES_PATH="files" - FRS="/var/www/${SERVER}/${FILES_PATH}" - APT_DIR="${FRS}/xcat" - APT_REPO_DIR="${APT_DIR}/repos/apt/devel" - - # Decide whether to upload the xcat-dep package or NOT (default is to NOT upload xcat-dep - if [ "$UP" != "1" ]; then - echo "Upload not specified, Done! (rerun with UP=1, to upload)" - cd $old_pwd - exit 0 - fi - - #upload the dep packages - i=0 - echo "Uploading debs from xcat-dep to ${APT_REPO_DIR}/xcat-dep/ ..." - while [ $((i+=1)) -le 5 ] && ! rsync -urLv --delete xcat-dep $USER@${SERVER}:${APT_REPO_DIR}/ - do : ; done - - #upload the tarball - i=0 - echo "Uploading $dep_tar_name to ${APT_DIR}/xcat-dep/2.x_Ubuntu/ ..." - while [ $((i+=1)) -le 5 ] && ! rsync -v --force $dep_tar_name $USER@${SERVER}:${APT_DIR}/xcat-dep/2.x_Ubuntu/ - do : ; done - - #upload the README file - cd debs - i=0 - echo "Uploading README to ${APT_DIR}/xcat-dep/2.x_Ubuntu/ ..." - while [ $((i+=1)) -le 5 ] && ! rsync -v --force README $USER@${SERVER}:${APT_DIR}/xcat-dep/2.x_Ubuntu/ - do : ; done - -fi - -cd $old_pwd -exit 0 diff --git a/builddebs.pl b/builddebs.pl index 7e148223e..ea561a367 100755 --- a/builddebs.pl +++ b/builddebs.pl @@ -1,7 +1,7 @@ #!/usr/bin/perl # Build the xcat-core Debian packages and assemble a signed apt repository. # -# Replaces build-ubunturepo. The shape mirrors buildrpms.pl -- Getopt::Long options, +# Builds every xCAT deb and the apt repository. The shape mirrors buildrpms.pl -- Getopt::Long options, # one package list, build then index then sign -- so the two builders read the same way # and share XCAT::BuildUtils. # @@ -385,7 +385,7 @@ carry an architecture, and there the difference is packaging metadata rather tha compiled output. Consequently this builder needs no C and no per-codename chroot. (xcat-dep is different: its packages are compiled, so it builds per codename.) -Replaces C. The GSA upload paths, the C/C release +Replaced C, removed in 2.19. The GSA upload paths, the C/C release flows and the C<-d> xcat-dep repository mode were not carried over: publishing is done by the CD pipeline's own deploy step, and xcat-dep is built from its own repository. diff --git a/docs/source/developers/guides/code/builds.rst b/docs/source/developers/guides/code/builds.rst index 607075ce5..e870a39f0 100644 --- a/docs/source/developers/guides/code/builds.rst +++ b/docs/source/developers/guides/code/builds.rst @@ -36,10 +36,8 @@ emitted, because a source-only run has no binary packages to advertise. ``buildrpms.pl`` replaces all three, and its ``--source-only`` replaces the old ``SRCONLY=1``. - ``build-ubunturepo`` is superseded by ``builddebs.pl`` but is **still in the - tree for now**, as a differential oracle: it is the reference the new builder - is checked against, and it is removed once the CD pipelines have been moved - over. Do not add features to it. + ``build-ubunturepo`` was removed in 2.19. ``builddebs.pl`` replaces it, and the + CD pipelines build every Ubuntu target with it. Debian and Ubuntu packages -------------------------- diff --git a/github_action_xcat_test.pl b/github_action_xcat_test.pl index f4480771b..c345e38c3 100644 --- a/github_action_xcat_test.pl +++ b/github_action_xcat_test.pl @@ -32,13 +32,10 @@ my $GITHUB_API = "https://api.github.com"; # through FindBin, so they can only be run from a source tree. Take a copy # before building and run the unit tests out of the copy. # -# This used to be mandatory rather than tidy: build-ubunturepo set -# local_core_repo_path="$curdir/../../xcat-core" -# which, under the work// layout GitHub checks out into, resolved to -# the checkout's own parent, and it rm -rf'd that path to make room for the apt -# repository -- destroying the tree the tests need. builddebs.pl writes under -# dist/debs INSIDE the checkout and restores every file it edits, so the copy is -# now only isolating the tests from build residue. +# The copy is tidiness, not a requirement: builddebs.pl writes under dist/debs +# inside the checkout and restores every file it edits, so it isolates the tests +# from build residue and nothing more. Its predecessor deleted the checkout's +# parent directory, which is why the copy was added. my $srcdir = getcwd(); my $unitsrc = ($ENV{'RUNNER_TEMP'} ? $ENV{'RUNNER_TEMP'} : "/tmp") . "/xcat-core-unitsrc"; diff --git a/xCAT-test/unit/builddebs_lock.t b/xCAT-test/unit/builddebs_lock.t index 5166dcba9..cba190133 100644 --- a/xCAT-test/unit/builddebs_lock.t +++ b/xCAT-test/unit/builddebs_lock.t @@ -7,9 +7,7 @@ # builds of DIFFERENT checkouts share nothing and must run concurrently. The historic # host-global lock got that backwards and made the devel and stable CD lanes collide. # -# This drives the real lock. The predecessor extracted a marked region out of -# build-ubunturepo with a regex and ran that; now the lock is a function, so it is -# called directly. +# This drives the real lock. The lock is a function, so it is called directly. use strict; use warnings; diff --git a/xCAT-test/unit/xcat_probe_package_payload.t b/xCAT-test/unit/xcat_probe_package_payload.t index 26659bcdf..58ff90990 100644 --- a/xCAT-test/unit/xcat_probe_package_payload.t +++ b/xCAT-test/unit/xcat_probe_package_payload.t @@ -12,7 +12,7 @@ use lib "$FindBin::Bin/../lib"; use lib "$FindBin::Bin/../../build-utils/lib"; use Test::More; -use XCAT::BuildUtils qw(XCAT_PROBE_HELPERS); +use XCAT::BuildUtils qw(XCAT_PROBE_HELPERS stage_probe_helpers); use XCAT::Test::File qw(repo_path slurp_repo_file); my @helpers = qw( @@ -29,7 +29,6 @@ my @affected_subcommands = qw( ); my $builder = slurp_repo_file('buildrpms.pl'); -my $debian_builder = slurp_repo_file('build-ubunturepo'); my $installed_probe_test = slurp_repo_file('xCAT-test/autotest/testcase/probe/xcatproble_list'); my $rpm_spec = slurp_repo_file('xCAT-probe/xCAT-probe.spec'); @@ -67,6 +66,12 @@ like( 'Debian package requires ss or the legacy netstat provider' ); +# The Debian builder stages the helpers by calling stage_probe_helpers, so run it and +# look at what it produced. The predecessor matched a `cp -f` line in build-ubunturepo, +# which passed whenever that text was reformatted and failed whenever it moved. +my $staged_probe_dir = File::Spec->catdir(tempdir(CLEANUP => 1), 'lib', 'perl', 'xCAT'); +stage_probe_helpers(repo_path(File::Spec->catdir('perl-xCAT', 'xCAT')), $staged_probe_dir); + for my $helper (@helpers) { my $source = repo_path(File::Spec->catfile('perl-xCAT', 'xCAT', $helper)); ok(-f $source, "$helper source exists"); @@ -75,9 +80,8 @@ for my $helper (@helpers) { scalar(grep { $_ eq $helper } XCAT_PROBE_HELPERS), "the shared builder helper list carries $helper" ); - like( - $debian_builder, - qr{cp -f [^\n]*/perl-xCAT/xCAT/\Q$helper\E\s+[^\n]*/lib/perl/xCAT/}, + ok( + -f File::Spec->catfile($staged_probe_dir, $helper), "Debian builder stages $helper" ); like(