diff --git a/Version b/Version index cf8690732..a36e9b090 100644 --- a/Version +++ b/Version @@ -1 +1 @@ -2.18.0 +2.18.1 diff --git a/buildrpms.pl b/buildrpms.pl index 6054b4a68..b4461ad4a 100755 --- a/buildrpms.pl +++ b/buildrpms.pl @@ -19,7 +19,7 @@ sub install_deps { esac dnf install -y perl-generators https://dl.fedoraproject.org/pub/epel/epel-release-latest-10.noarch.rpm dnf install -y \$(/usr/lib/rpm/perl.req $0) - dnf install -y tar mock nginx createrepo podman rpmdevtools rpm-sign + dnf install -y tar mock nginx createrepo_c podman rpmdevtools rpm-sign systemctl enable --now nginx @@ -40,6 +40,7 @@ use Data::Dumper; use File::Copy qw(cp); use File::Path qw(make_path remove_tree); use File::Slurper qw(read_text write_text); +use File::Temp qw(tempdir tempfile); use FindBin qw($Bin); use Getopt::Long qw(GetOptions); use POSIX qw(strftime); @@ -50,8 +51,19 @@ use autodie; use autodie qw(cp); my $SOURCES = "$ENV{HOME}/rpmbuild/SOURCES"; +# Ensure the rpmbuild tree exists. buildrpms stages source tarballs into $SOURCES, but it only +# runs rpmdev-setuptree in the one-time env-setup path -- so on a host where that never ran (or +# $HOME/rpmbuild was cleaned) source staging fails with "SOURCES/...: No such file or directory", +# no srpms/rpms are produced, and the run still exits 0. Create the tree up front so a build never +# depends on prior manual setup. +system('mkdir', '-p', map { "$ENV{HOME}/rpmbuild/$_" } qw(SOURCES SPECS BUILD BUILDROOT RPMS SRPMS)); my $VERSION = read_text("Version"); my $PWD = Cwd::cwd(); +my @XCAT_PROBE_HELPERS = qw( + GlobalDef.pm + NetworkUtils.pm + ServiceNodeUtils.pm +); chomp($VERSION); @@ -101,6 +113,13 @@ my $DISTRO = $OS{ID}; # almalinux+epel-* config, so translate the os-release ID accordingly. $DISTRO = "alma" if $DISTRO eq "almalinux"; +# xCAT-genesis-base is intentionally NOT in the default build set below. Its +# payload is a dracut-built initramfs that bundles the build chroot's kernel + +# glibc/busybox/perl, so it is OS-dependent (an el10 build cannot boot el8/el9 +# nodes). It is built per target by the xcat-dep pipeline +# (xcat-dep/mockbuild-all.pl, via `buildrpms.pl --package xCAT-genesis-base`) +# and shipped in the per-EL repo xcat-dep/rh, NOT in the flat xcat-core. The +# build logic further down still supports `--package xCAT-genesis-base`. my @PACKAGES = qw( perl-xCAT xCAT @@ -108,7 +127,6 @@ my @PACKAGES = qw( xCAT-buildkit xCAT-client xCAT-confluent - xCAT-genesis-base xCAT-genesis-scripts xCAT-openbmc-py xCAT-probe @@ -129,7 +147,7 @@ my %opts = ( configure_nginx => 0, force => 0, gpg_home => "", - gpg_key_name => "xCAT Automatic Signing Key", + gpg_key_name => "xCAT Signing Key", gpg_sign => 0, help => 0, mock_uniqueext => "", @@ -143,6 +161,7 @@ my %opts = ( xcat_dep_path => "$PWD/../xcat-dep/", ); +my @cli_packages; GetOptions( "configure_nginx" => \$opts{configure_nginx}, "force" => \$opts{force}, @@ -153,19 +172,27 @@ GetOptions( "mock-uniqueext=s" => \$opts{mock_uniqueext}, "nginx_port" => \$opts{nginx_port}, "nproc=i" => \$opts{nproc}, - "package=s@" => \$opts{packages}, + "package=s@" => \@cli_packages, "release=s" => \$opts{release}, "repo-mode=s" => \$opts{repo_mode}, "target=s@" => \$opts{targets}, "verbose" => \$opts{verbose}, "xcat_dep_path=s" => \$opts{xcat_dep_path}, "setup_local_repos" => \$opts{setup_local_repos}, + "finalize-core=s" => \$opts{finalize_core}, ) or usage(); -# Release is regenerated at each run so every build gets a fresh snapshot -# release, unless pinned with --release (e.g. to rebuild a single package -# matching the release the rest of the repo was built with). -my $RELEASE = $opts{release} || strftime("snap%Y%m%d%H%M", localtime); +# --package REPLACES the default set (build exactly what was asked), so +# `--package xCAT-genesis-base` builds only genesis-base for the dep pipeline. +# The full default set is built on every arch (x86_64 and ppc64le alike), so each +# arch produces a complete, self-contained xcat-core repo. +$opts{packages} = \@cli_packages if @cli_packages; + +# Release is derived from SOURCE_DATE_EPOCH (the git commit time), NOT wall-clock, +# so identical sources -> identical Version-Release -> bit-reproducible packages +# (a hard requirement for the content-addressed/Merkle-DAG CI). Override with +# --release to rebuild a single package matching an existing repo's release. +my $RELEASE = $opts{release} || strftime("snap%Y%m%d%H%M", gmtime($SOURCE_DATE_EPOCH)); write_text("Release", "$RELEASE\n"); sub usage { @@ -294,6 +321,38 @@ sub buildsources_genesis_base($) { remove_tree($staging_parent); } +sub prepare_xcat_probe_source_tar { + my $staging_parent = tempdir("xcat-probe-source.XXXXXX", TMPDIR => 1, CLEANUP => 1); + my $staging_root = "$staging_parent/xCAT-probe"; + my $helper_dir = "$staging_root/lib/perl/xCAT"; + my $source_tarball = "$SOURCES/xCAT-probe-$VERSION.tar.gz"; + + sh(qq(cp -a "xCAT-probe" "$staging_root")) + and die "Error staging xCAT-probe sources"; + + remove_tree($helper_dir) if -e $helper_dir; + make_path($helper_dir); + chmod 0755, $helper_dir; + for my $helper (@XCAT_PROBE_HELPERS) { + my $destination = "$helper_dir/$helper"; + cp "perl-xCAT/xCAT/$helper", $destination; + chmod 0644, $destination; + } + + my ($archive_fh, $archive_path) = tempfile( + ".xCAT-probe-$VERSION.XXXXXX", + DIR => $SOURCES, + UNLINK => 1, + ); + close $archive_fh; + + sh(qq(tar --sort=name --owner=0 --group=0 --numeric-owner --mtime="\@$SOURCE_DATE_EPOCH" --use-compress-program="gzip -n" -cf "$archive_path" -C "$staging_parent" xCAT-probe)) + and die "Error creating $source_tarball"; + + chmod 0644, $archive_path; + rename $archive_path, $source_tarball; +} + sub buildsources { my ($pkg, $target) = @_; @@ -329,6 +388,9 @@ EOF EOF # xCATsn.spec consumes templates from xCAT shared templates payload. sh qq(tar --sort=name --owner=0 --group=0 --mtime="\@$SOURCE_DATE_EPOCH" -czf "$SOURCES/templates.tar.gz" xCAT/templates) unless -f "$SOURCES/templates.tar.gz"; + } elsif ($pkg eq "xCAT-probe") { + # Prepared once before target builds fork so workers only read a complete archive. + return; } else { sh qq(tar --sort=name --owner=0 --group=0 --mtime="\@$SOURCE_DATE_EPOCH" -czf "$SOURCES/$pkg-$VERSION.tar.gz" $pkg); } @@ -344,8 +406,8 @@ sub buildspkgs { my $diskcache = ( $pkg eq 'xCAT-genesis-scripts' || $pkg eq 'xCAT-genesis-base' - ) ? "dist/$target/srpms/$pkg-$genesis_tarch-$VERSION-$RELEASE.src.rpm" - : "dist/$target/srpms/$pkg-$VERSION-$RELEASE.src.rpm"; + ) ? "dist/$target/rpms/SRPMS/$pkg-$genesis_tarch-$VERSION-$RELEASE.src.rpm" + : "dist/$target/rpms/SRPMS/$pkg-$VERSION-$RELEASE.src.rpm"; return if -f $diskcache and not $opts{force}; my $dir = sub { @@ -373,7 +435,7 @@ mock -r $chroot \\ --buildsrpm \\ --spec $dir/$pkg.spec \\ --sources $SOURCES \\ - --resultdir "dist/$target/srpms/" + --resultdir "dist/$target/rpms/SRPMS/" EOF } @@ -385,6 +447,7 @@ sub buildpkgs { my @native_pkgs = qw( xCAT + xCATsn xCAT-genesis-scripts ); @@ -427,7 +490,7 @@ mock -r $chroot \\ --define "clamp_mtime_to_source_date_epoch 1" \\ --define "_buildhost xcat-build" \\ --resultdir "dist/$target/rpms/" \\ - --rebuild dist/$target/srpms/$spkgname + --rebuild dist/$target/rpms/SRPMS/$spkgname EOF } @@ -546,40 +609,162 @@ sub setup_local_repos { } +# Index one repo dir with deterministic, upstream-matching metadata. createrepo_c's +# defaults already emit primary/filelists/other as *.xml.zst plus *.sqlite.bz2 +# (--database), exactly the upstream shape; --set-timestamp-to-revision pins the +# repomd timestamp to SOURCE_DATE_EPOCH. +sub createrepo_dir { + my ($dir, $extra) = @_; + $extra //= ''; + sh(qq(createrepo_c --update --database ) + . qq(--revision "$SOURCE_DATE_EPOCH" --set-timestamp-to-revision $extra "$dir")) + and die "Failed to createrepo_c $dir\n"; +} + +# A core repo dir holds binaries flat plus a SRPMS/ subdir carrying its own +# repodata (the upstream xcat.org layout). mock --rebuild re-emits the .src.rpm +# into the binary resultdir, but the canonical copy lives in SRPMS/, so drop the +# top-level strays; then index the binaries EXCLUDING the SRPMS/ subdir so no +# src.rpm enters the binary repomd, and index the SRPMS repo separately. +sub index_repo { + my ($repodir) = @_; + say "Creating repository $repodir"; + # Drop the top-level stray src.rpm and the mock logs (build.log/root.log/...) + # that mock leaves in the resultdir, so the dir is directly deployable (upstream + # ships neither). The canonical src.rpm lives in SRPMS/. + unlink($_) for glob("$repodir/*.src.rpm"), glob("$repodir/*.log"), + glob("$repodir/SRPMS/*.log"); + createrepo_dir($repodir, "--excludes 'SRPMS/*' --excludes '*.src.rpm'"); + createrepo_dir("$repodir/SRPMS") if -d "$repodir/SRPMS"; +} + sub update_repo { my ($target) = @_; - say "Creating repository dist/$target/rpms"; - `find dist/$target/rpms -name ".src.rpm" -delete`; - `createrepo --update dist/$target/rpms`; + index_repo("dist/$target/rpms"); } sub sign_rpms { my ($target) = @_; - my $key_name = $opts{gpg_key_name}; - my $repodir = "dist/$target/rpms"; + sign_repo_dir("dist/$target/rpms", $opts{gpg_key_name}); +} + +# Sign every rpm in a core repo dir -- the top-level binaries AND SRPMS/*.src.rpm -- +# then re-index (signing rewrites the rpms, invalidating checksums) and detach-sign +# + export the key into BOTH the binary and the SRPMS repodata dirs. +sub sign_repo_dir { + my ($repodir, $key_name) = @_; say "Signing RPMs in $repodir"; - my @rpms = glob("$repodir/*.rpm"); - if (@rpms) { - my $rpm_list = join " ", map { qq("$_") } @rpms; - sh(qq(rpmsign --define "%_gpg_name $key_name" --addsign $rpm_list)) + my @bin = glob("$repodir/*.rpm"); + if (@bin) { + sh(qq(rpmsign --define "%_gpg_name $key_name" --addsign ) + . join(" ", map { qq("$_") } @bin)) and die "Failed to sign RPMs in $repodir"; } + my @src = glob("$repodir/SRPMS/*.src.rpm"); + if (@src) { + sh(qq(rpmsign --define "%_gpg_name $key_name" --addsign ) + . join(" ", map { qq("$_") } @src)) + and die "Failed to sign SRPMs in $repodir/SRPMS"; + } - # rpmsign --addsign rewrites the rpm files, so the checksums recorded by the - # earlier createrepo no longer match and dnf rejects them. Regenerate the repo - # metadata now (after signing, before signing repomd.xml) so it stays consistent. - say "Regenerating repo metadata after signing $repodir"; - sh(qq(createrepo --update "$repodir")) - and die "Failed to regenerate repo metadata after signing"; + # Regenerate both indexes (binary + SRPMS) after signing, before signing repomd. + index_repo($repodir); - say "Signing repomd.xml for $target"; - my $repomd = "$repodir/repodata/repomd.xml"; - unlink "$repomd.asc" if -f "$repomd.asc"; - sh(qq(gpg -a --detach-sign --default-key "$key_name" "$repomd")) - and die "Failed to sign $repomd"; - sh(qq(gpg -a --export "$key_name" > "$repomd.key")) - and die "Failed to export public key"; + for my $rd ("$repodir/repodata", + (-d "$repodir/SRPMS/repodata" ? ("$repodir/SRPMS/repodata") : ())) { + my $repomd = "$rd/repomd.xml"; + next unless -f $repomd; + say "Signing $repomd"; + unlink "$repomd.asc" if -f "$repomd.asc"; + sh(qq(gpg -a --detach-sign --default-key "$key_name" "$repomd")) + and die "Failed to sign $repomd"; + sh(qq(gpg -a --export "$key_name" > "$rd/repomd.xml.key")) + and die "Failed to export public key to $rd"; + } +} + +# Emit the deployable repo metadata into dist/$target/rpms: xcat-core.repo, +# mklocalrepo.sh and buildinfo.txt (templates ported from buildcore.sh). This makes +# the built tree directly deployable to xcat.org and removes the need for +# cluster-test.pl to re-collect / re-createrepo the dist output. +sub write_repo_metadata { + my ($target) = @_; + write_repo_metadata_dir("dist/$target/rpms"); +} + +sub write_repo_metadata_dir { + my ($repodir) = @_; + return unless -d $repodir; + + # Shipped baseurl points at xcat.org; mklocalrepo.sh rewrites baseurl/gpgkey to + # file:// at deploy time for local use. + my $baseurl = "https://xcat.org/files/xcat/repos/yum/devel/xcat-core"; + my $gpgcheck = $opts{gpg_sign} ? 1 : 0; + my $gpgkey_line = $opts{gpg_sign} + ? "gpgkey=$baseurl/repodata/repomd.xml.key" + : "# gpgkey="; + write_text("$repodir/xcat-core.repo", <<"EOF"); +[xcat-core] +name=xCAT 2 Core packages +baseurl=$baseurl +enabled=1 +gpgcheck=$gpgcheck +$gpgkey_line +EOF + + write_text("$repodir/mklocalrepo.sh", <<'EOF2'); +#!/bin/sh +cd `dirname $0` +REPOFILE=`basename xcat-*.repo` +if [[ $REPOFILE == "xcat-*.repo" ]]; then + echo "ERROR: For xcat-dep, please execute $0 in the correct / subdirectory" + exit 1 +fi +# +# default to RHEL yum, if doesn't exist try Zypper +# +DIRECTORY="/etc/yum.repos.d" +if [ ! -d "$DIRECTORY" ]; then + DIRECTORY="/etc/zypp/repos.d" +fi +sed -e 's|baseurl=.*|baseurl=file://'"`pwd`"'|' $REPOFILE | sed -e 's|gpgkey=.*|gpgkey=file://'"`pwd`"'/repodata/repomd.xml.key|' > "$DIRECTORY/$REPOFILE" +if [ -f "$DIRECTORY/xCAT-core.repo" ]; then + mv "$DIRECTORY/xCAT-core.repo" "$DIRECTORY/xCAT-core.repo.nouse" +fi +cd - +EOF2 + chmod 0775, "$repodir/mklocalrepo.sh"; + + # BUILD_TIME from SOURCE_DATE_EPOCH keeps buildinfo reproducible across rebuilds. + my $build_time = strftime("%a %b %e %H:%M:%S %Z %Y", gmtime($SOURCE_DATE_EPOCH)); + my $build_machine = `hostname`; chomp $build_machine; + my $commit_short = substr($GITINFO, 0, 7); + write_text("$repodir/buildinfo.txt", <<"EOF"); +VERSION=$VERSION +RELEASE=$RELEASE +BUILD_TIME=$build_time +BUILD_MACHINE=$build_machine +COMMIT_ID=$commit_short +COMMIT_ID_LONG=$GITINFO +EOF +} + +# Turn an already-populated core dir into a signed repo in the upstream xcat.org +# layout, reusing the same index/sign/metadata code as a per-target build. Used to +# assemble the flat MULTI-ARCH core: the caller rsyncs each arch's dist//rpms/ +# (excluding repodata/) into first, then this does the single final +# createrepo_c + repomd signing so no packages are moved by hand. +sub finalize_core { + my $dir = $opts{finalize_core}; + die "FATAL: --finalize-core dir '$dir' does not exist\n" unless -d $dir; + index_repo($dir); + if ($opts{gpg_sign}) { + $ENV{GNUPGHOME} = $opts{gpg_home} if $opts{gpg_home}; + sign_repo_dir($dir, $opts{gpg_key_name}); + } + write_repo_metadata_dir($dir); + return 0; } sub main { @@ -590,6 +775,10 @@ sub main { return exit(configure_nginx()) if $opts{configure_nginx}; return exit(setup_local_repos()) if $opts{setup_local_repos}; + return exit(finalize_core()) if $opts{finalize_core}; + + prepare_xcat_probe_source_tar() + if grep { $_ eq "xCAT-probe" } $opts{packages}->@*; my @rpms = product($opts{packages}, $opts{targets}); my $pm = Parallel::ForkManager->new($opts{nproc}); @@ -621,6 +810,12 @@ sub main { } } + # Emit deployable repo metadata (after signing, so the .repo gpgkey line matches + # the freshly written repomd.xml.key). + for my $target ($opts{targets}->@*) { + write_repo_metadata($target); + } + exit(0); } diff --git a/xCAT-OpenStack-baremetal/xCAT-OpenStack-baremetal.spec b/xCAT-OpenStack-baremetal/xCAT-OpenStack-baremetal.spec index 0a38609de..71fa9b96b 100644 --- a/xCAT-OpenStack-baremetal/xCAT-OpenStack-baremetal.spec +++ b/xCAT-OpenStack-baremetal/xCAT-OpenStack-baremetal.spec @@ -20,7 +20,10 @@ BuildArch: noarch Provides: xCAT-OpenStack-baremetal = %{epoch}:%{version} Requires: xCAT-client +# Pod::Html ships inside the core perl package on SUSE; there is no perl-Pod-Html +%if !0%{?suse_version} BuildRequires: perl-Pod-Html +%endif %description xCAT-OpenStack-baremetal provides the baremetal driver for OpenStack. diff --git a/xCAT-SoftLayer/xCAT-SoftLayer.spec b/xCAT-SoftLayer/xCAT-SoftLayer.spec index 0991a419c..a2355ddba 100644 --- a/xCAT-SoftLayer/xCAT-SoftLayer.spec +++ b/xCAT-SoftLayer/xCAT-SoftLayer.spec @@ -15,7 +15,10 @@ BuildRoot: /var/tmp/%{name}-%{version}-%{release}-root BuildArch: noarch %endif Requires: xCAT-server +# Pod::Html ships inside the core perl package on SUSE; there is no perl-Pod-Html +%if !0%{?suse_version} BuildRequires: perl-Pod-Html +%endif # perl-ExtUtils-MakeMaker, perl-CPAN, perl-Test-Harness are only available in rhel. # When this rpm supports being installed in sles, need to add these to xcat-dep. diff --git a/xCAT-buildkit/xCAT-buildkit.spec b/xCAT-buildkit/xCAT-buildkit.spec index 037dfcfc4..4bc6d948b 100644 --- a/xCAT-buildkit/xCAT-buildkit.spec +++ b/xCAT-buildkit/xCAT-buildkit.spec @@ -12,7 +12,10 @@ Distribution: %{?_distribution:%{_distribution}}%{!?_distribution:%{_vendor}} Prefix: /opt/xcat BuildRoot: /var/tmp/%{name}-%{version}-%{release}-root +# Pod::Html ships inside the core perl package on SUSE; there is no perl-Pod-Html +%if !0%{?suse_version} BuildRequires: perl-Pod-Html +%endif #%ifnos linux AutoReqProv: no diff --git a/xCAT-client/xCAT-client.spec b/xCAT-client/xCAT-client.spec index b35349324..0fc3e717f 100644 --- a/xCAT-client/xCAT-client.spec +++ b/xCAT-client/xCAT-client.spec @@ -19,7 +19,10 @@ BuildRoot: /var/tmp/%{name}-%{version}-%{release}-root %define s390x %(if [ "$s390x" = "1" ];then echo 1; else echo 0; fi) %define nots390x %(if [ "$s390x" = "1" ];then echo 0; else echo 1; fi) +# Pod::Html ships inside the core perl package on SUSE; there is no perl-Pod-Html +%if !0%{?suse_version} BuildRequires: perl-Pod-Html +%endif # AIX will build with an arch of "ppc" %ifos linux @@ -31,7 +34,10 @@ Requires: cpio # fping or nmap is needed by pping (in case xCAT-client is installed by itself on a remote client) %ifos linux +# perl-generators is a RHEL/Fedora-only build helper; SUSE generates perl deps itself +%if !0%{?suse_version} BuildRequires: perl-generators +%endif Requires: nmap perl-XML-Simple perl-XML-Parser %else Requires: expat diff --git a/xCAT-confluent/xCAT-confluent.spec b/xCAT-confluent/xCAT-confluent.spec index b0bf55e48..9a940fb30 100644 --- a/xCAT-confluent/xCAT-confluent.spec +++ b/xCAT-confluent/xCAT-confluent.spec @@ -17,7 +17,10 @@ BuildArch: noarch Requires: confluent_server Provides: xCAT-confluent = %{epoch}:%{version} +# Pod::Html ships inside the core perl package on SUSE; there is no perl-Pod-Html +%if !0%{?suse_version} BuildRequires: perl-Pod-Html +%endif %description xCAT confluent provides the necessary integration pieces to utilize the confluent diff --git a/xCAT-genesis-builder/xCAT-genesis-base.spec b/xCAT-genesis-builder/xCAT-genesis-base.spec index aea7027f9..fe544f319 100644 --- a/xCAT-genesis-builder/xCAT-genesis-base.spec +++ b/xCAT-genesis-builder/xCAT-genesis-base.spec @@ -131,13 +131,38 @@ dracut --compress gzip -m "xcat base" --no-early-microcode -N -f "$DRACUT_IMAGE" zcat "$DRACUT_IMAGE" | cpio -dumi ) -%if 0%{?rhel} > 0 && 0%{?rhel} <= 9 -# EL9 upgrade safety depends on this remaining a real directory. +# usrmerge collapse: on a usr-merged build host the extracted genesis fs can +# contain /bin,/sbin,/lib,/lib64 as real directories that duplicate the files +# already under /usr/*, which makes rpm reject the package with file conflicts. +# Fold the top-level dirs into /usr and replace them with symlinks (a no-op on +# hosts where the image already ships them as symlinks). +for _d in bin sbin lib lib64; do + if [ -d "$GENESIS_FS/$_d" ] && [ ! -L "$GENESIS_FS/$_d" ] && [ -d "$GENESIS_FS/usr/$_d" ]; then + cp -a "$GENESIS_FS/$_d/." "$GENESIS_FS/usr/$_d/" 2>/dev/null || true + rm -rf "$GENESIS_FS/$_d" + ln -s "usr/$_d" "$GENESIS_FS/$_d" + fi +done + +# xCAT 2.14.5 genesis payloads shipped usr/lib/dracut/hooks as a real directory. +# Newer dracut (EL8+) makes it a symlink to ../../../var/lib/dracut/hooks. RPM +# cannot replace a directory with a symlink across an upgrade, so a 2.17 -> 2.18 +# upgrade aborts with a file conflict on this path. Materialize the symlink back +# into a real directory (with the hook contents) so the payload type matches the +# installed 2.14.5 layout and the upgrade is conflict-free on EL8/EL9/EL10. +if [ -L "$GENESIS_FS/usr/lib/dracut/hooks" ]; then + hooks_target="$GENESIS_FS/var/lib/dracut/hooks" + rm -f "$GENESIS_FS/usr/lib/dracut/hooks" + if [ -d "$hooks_target" ]; then + cp -a "$hooks_target" "$GENESIS_FS/usr/lib/dracut/hooks" + else + mkdir -p "$GENESIS_FS/usr/lib/dracut/hooks" + fi +fi if [ ! -d "$GENESIS_FS/usr/lib/dracut/hooks" ] || [ -L "$GENESIS_FS/usr/lib/dracut/hooks" ]; then - echo "EL%{?rhel} genesis payload has invalid usr/lib/dracut/hooks layout" >&2 + echo "genesis payload has invalid usr/lib/dracut/hooks layout" >&2 exit 1 fi -%endif for script in \ "$GENESIS_FS/sbin/dhclient-script" \ @@ -207,7 +232,7 @@ local tail_leaf_prefix = '`-- ' local link_prefix = ' -> ' local function printf(...) - io.write(string.format(unpack(arg))) + io.write(string.format(table.unpack({...}))) end local function remove_directory(directory, level, prefix) @@ -215,7 +240,7 @@ local function remove_directory(directory, level, prefix) local num_files = 0 if posix.access(directory, "rw") then local files = posix.dir(directory) - local last_file_index = table.getn(files) + local last_file_index = #files table.sort(files) for i, name in ipairs(files) do if name ~= '.' and name ~= '..' then @@ -277,8 +302,11 @@ remove_directory_deep("/opt/xcat/share/xcat/netboot/genesis/%{tarch}/fs/var/run" if [ "$1" == "2" ]; then #only on upgrade, as on install it's probably not going to work... if [ -f "/proc/cmdline" ]; then # prevent running it during install into chroot image . /etc/profile.d/xcat.sh - mknb %{tarch} - echo "If you are installing/updating xCAT-genesis-base separately, not as part of installing/updating all of xCAT, run 'mknb ' manually" + # During a full 'dnf update xCAT', xcatd is stopped while xCAT is being + # upgraded, so mknb cannot reach it and exits non-zero. That must not fail + # the rpm transaction: drop the genesis-base-updated marker so the netboot + # image is regenerated later (xcatd post-start / manual 'mknb '). + mknb %{tarch} || echo "mknb %{tarch} deferred (xcatd not reachable during upgrade); run 'mknb %{tarch}' after xcatd is up" mkdir -p /etc/xcat touch /etc/xcat/genesis-base-updated fi diff --git a/xCAT-genesis-scripts/xCAT-genesis-scripts.spec b/xCAT-genesis-scripts/xCAT-genesis-scripts.spec index 5202c1ed0..79c70749a 100644 --- a/xCAT-genesis-scripts/xCAT-genesis-scripts.spec +++ b/xCAT-genesis-scripts/xCAT-genesis-scripts.spec @@ -32,7 +32,15 @@ Vendor: IBM Corp. Summary: xCAT Genesis netboot image - Core content URL: https://xcat.org/ Source1: xCAT-genesis-scripts.tar.bz2 -Requires: xCAT-genesis-base-%{tarch} = 2:%{version}-%{release} +# Require a fixed genesis-base floor, NOT the exact snap and NOT %{version}. genesis-base +# lives in the per-EL xcat-dep repo while this package ships in the flat xcat-core; +# pinning the exact -%{release} forced the two repos to be republished in lockstep on +# every core rebuild, and pinning >= 2:%{version} did the same on every marketing-version +# bump (a 2.18.1 core then demanded a 2.18.1 genesis-base). genesis-base changes rarely, +# so floor it at the 2.18.0 baseline: any 2.18.0-or-newer genesis-base snap satisfies it, +# and core version bumps no longer force a genesis-base re-publish. (RPM ignores release +# when the dependency omits it.) +Requires: xCAT-genesis-base-%{tarch} >= 2:2.18.0 Buildroot: %{_localstatedir}/tmp/xCAT-genesis Packager: IBM Corp. diff --git a/xCAT-server/lib/xcat/plugins/kvm.pm b/xCAT-server/lib/xcat/plugins/kvm.pm index 546c9984d..1e81317ae 100644 --- a/xCAT-server/lib/xcat/plugins/kvm.pm +++ b/xCAT-server/lib/xcat/plugins/kvm.pm @@ -720,7 +720,11 @@ sub build_xmldesc { $xtree{name}->{content} = $node; $xtree{uuid}->{content} = getNodeUUID($node); $xtree{os} = build_oshash(); - if (defined($hypcpumodel) and $hypcpumodel eq "ppc64") { + # ppc64le hypervisors report cpumodel "ppc64le" (not "ppc64"); both are pseries + # guests whose libvirt arch is "ppc64". Without this the guest is emitted + # as an x86-style domain (no machine, plus the pae/acpi/apic below) which libvirt + # rejects on ppc64le hosts: "machine type 'pseries-*' does not support ACPI". + if (defined($hypcpumodel) and ($hypcpumodel eq "ppc64" or $hypcpumodel eq "ppc64le")) { $xtree{os}->{type}->{arch} = "ppc64"; $xtree{os}->{type}->{machine} = "pseries"; delete $xtree{os}->{bios}; @@ -936,9 +940,13 @@ sub build_xmldesc { } } - $xtree{features}->{pae} = {}; - $xtree{features}->{acpi} = {}; - $xtree{features}->{apic} = {}; + # pae/acpi/apic are x86 features; pseries (ppc64/ppc64le) guests do not support + # them and libvirt rejects the domain if they are present. + unless (defined($hypcpumodel) and ($hypcpumodel eq "ppc64" or $hypcpumodel eq "ppc64le")) { + $xtree{features}->{pae} = {}; + $xtree{features}->{acpi} = {}; + $xtree{features}->{apic} = {}; + } $xtree{features}->{content} = "\n"; ($xtree{devices}->{disk}, my $errstr) = build_diskstruct($cdloc); if ($errstr) { diff --git a/xCAT-server/share/xcat/cons/kvm b/xCAT-server/share/xcat/cons/kvm index 4a8d07ce2..9cd5fb6dd 100755 --- a/xCAT-server/share/xcat/cons/kvm +++ b/xCAT-server/share/xcat/cons/kvm @@ -69,12 +69,24 @@ until ($dsthost and $speed and $dstty) { } release_lock(); -# The screen command needs the TERM env var, -# TERM might be empty for some unknown reasons, -# for example, on SLES 12 and on PowerKVM +# TERM may be empty (SLES 12, PowerKVM); a terminal multiplexer needs it. if (!$ENV{'TERM'}) { $ENV{'TERM'} = "vt100"; } -exec "ssh -t $dsthost screen -U -a -O -e ^]a -d -R -S serial-" . $ARGV[0] . "-cons -A $dstty $speed"; - +# Attach to the KVM guest's serial console inside a shared tmux session on the +# hypervisor. tmux (unlike screen) is present on every libvirt host we target, +# incl. ppc64le power hosts that ship no screen; `new-session -A` gives the same +# detach/reattach + multi-client behaviour screen provided (goconserver and an +# interactive rcons share one session). The console itself is opened by +# `virsh console` keyed on the domain name (== the xCAT node name for KVM), which +# is stable across guest reboots (the raw serial pty is not). -x/-y size the +# pane so the guest sees a sane terminal geometry. +my $node = $ARGV[0]; +my $session = "serial-$node-cons"; +my $virshcons = "virsh -c qemu:///system console --force $node"; +# Create the session once, detached, with the status bar OFF so the captured +# console log stays clean serial (no tmux chrome/clock redraws) -- matching what +# screen gave; then attach. `new-session -d -A` is a no-op if it already exists, +# so concurrent clients (goconserver + rcons) share the single session. +exec "ssh -t $dsthost \"tmux -u new-session -d -A -x 200 -y 50 -s $session '$virshcons'; tmux set-option -t $session status off; exec tmux -u attach -t $session\""; diff --git a/xCAT-server/share/xcat/install/alma/service.alma10.ppc64le.otherpkgs.pkglist b/xCAT-server/share/xcat/install/alma/service.alma10.ppc64le.otherpkgs.pkglist new file mode 120000 index 000000000..ba418ca8a --- /dev/null +++ b/xCAT-server/share/xcat/install/alma/service.alma10.ppc64le.otherpkgs.pkglist @@ -0,0 +1 @@ +../rh/service.rhels10.ppc64le.otherpkgs.pkglist \ No newline at end of file diff --git a/xCAT-server/share/xcat/install/rh/service.rhels10.ppc64le.otherpkgs.pkglist b/xCAT-server/share/xcat/install/rh/service.rhels10.ppc64le.otherpkgs.pkglist new file mode 100644 index 000000000..0abdc95c3 --- /dev/null +++ b/xCAT-server/share/xcat/install/rh/service.rhels10.ppc64le.otherpkgs.pkglist @@ -0,0 +1,2 @@ +xcat/xcat-core/xCATsn +xcat/xcat-dep/rh10/ppc64le/goconserver diff --git a/xCAT-server/xCAT-server.spec b/xCAT-server/xCAT-server.spec index e0b1ec8c1..bc1d82438 100644 --- a/xCAT-server/xCAT-server.spec +++ b/xCAT-server/xCAT-server.spec @@ -68,7 +68,10 @@ Requires: perl-HTTP-Async >= 0.30-3 Requires: initscripts Requires: chkconfig %endif -%if 0%{?rhel} >= 10 +# openssl is present on all EL; require it uniformly (not just el10) so a single +# flat xcat-core build is correct everywhere. Excluded on SLES, where the package +# name differs and it was never required before. +%if !0%{?suse_version} Requires: openssl %endif diff --git a/xCAT-test/autotest/bundle/alma10_x86_daily.bundle b/xCAT-test/autotest/bundle/alma10_x86_daily.bundle index 26f22b0b9..f985cfd5c 100644 --- a/xCAT-test/autotest/bundle/alma10_x86_daily.bundle +++ b/xCAT-test/autotest/bundle/alma10_x86_daily.bundle @@ -1,3 +1,4 @@ setup_vm #INCLUDE:rhels_x86_daily.bundle# +xcatprobe_work clean_up_env diff --git a/xCAT-test/autotest/testcase/probe/xcatproble_list b/xCAT-test/autotest/testcase/probe/xcatproble_list index 214711885..e5471aae9 100644 --- a/xCAT-test/autotest/testcase/probe/xcatproble_list +++ b/xCAT-test/autotest/testcase/probe/xcatproble_list @@ -5,6 +5,8 @@ cmd:mkdir -p /tmp/xcatprobe_l cmd:xcatprobe -l check:rc==0 check:output=~Supported sub commands are: +cmd:for module in GlobalDef.pm NetworkUtils.pm ServiceNodeUtils.pm; do test -r "/opt/xcat/probe/lib/perl/xCAT/$module" || exit 1; done +check:rc==0 cmd:xcatprobe -l|grep -v "Supported sub commands are" |awk '/^[[:graph:]]/ {print $1}'|sort > /tmp/xcatprobe_l/subcmd_from_xcatprobe_l cmd:ls -l /opt/xcat/probe/subcmds/ |awk '/^-/ {print $9}'|sort > /tmp/xcatprobe_l/subcmd_under_subcmds_dir cmd:diff -y /tmp/xcatprobe_l/subcmd_from_xcatprobe_l /tmp/xcatprobe_l/subcmd_under_subcmds_dir diff --git a/xCAT-test/unit/xcat_probe_package_payload.t b/xCAT-test/unit/xcat_probe_package_payload.t new file mode 100644 index 000000000..a4fad8aaf --- /dev/null +++ b/xCAT-test/unit/xcat_probe_package_payload.t @@ -0,0 +1,137 @@ +#!/usr/bin/env perl +use strict; +use warnings; + +use File::Copy qw(copy); +use File::Path qw(make_path); +use File::Spec; +use File::Temp qw(tempdir); +use FindBin; +use Test::More; + +my $repo_root = File::Spec->rel2abs(File::Spec->catdir($FindBin::Bin, '..', '..')); +my @helpers = qw( + GlobalDef.pm + NetworkUtils.pm + ServiceNodeUtils.pm +); +my @affected_subcommands = qw( + code_template + discovery + osdeploy + xcatmn +); + +my $builder = read_file('buildrpms.pl'); +like($builder, qr/sub prepare_xcat_probe_source_tar\b/, 'RPM builder has dedicated xCAT-probe source preparation'); +like( + $builder, + qr/for my \$helper \(\@XCAT_PROBE_HELPERS\).*?cp "perl-xCAT\/xCAT\/\$helper", \$destination;/s, + 'RPM builder copies every declared helper into the staged package tree' +); +like($builder, qr/tempfile\(.*?DIR\s*=>\s*\$SOURCES/s, 'RPM builder writes a unique archive in the source directory'); +like($builder, qr/--use-compress-program="gzip -n"/, 'RPM builder normalizes gzip metadata'); +like($builder, qr/rename\s+\$archive_path,\s*\$source_tarball/, 'RPM builder publishes the source archive atomically'); +like( + $builder, + qr/elsif \(\$pkg eq "xCAT-probe"\)\s*\{.*?\breturn;/s, + 'target workers reuse the source archive prepared before the fork' +); + +my $prepare_call = rindex($builder, 'prepare_xcat_probe_source_tar()'); +my $worker_fanout = index($builder, 'Parallel::ForkManager->new'); +ok( + $prepare_call >= 0 && $worker_fanout >= 0 && $prepare_call < $worker_fanout, + 'xCAT-probe source preparation runs before worker processes fork' +); + +for my $helper (@helpers) { + my $source = File::Spec->catfile($repo_root, 'perl-xCAT', 'xCAT', $helper); + ok(-f $source, "$helper source exists"); + like($builder, qr/^\s*\Q$helper\E\s*$/m, "RPM builder stages $helper"); +} + +my $tmpdir = tempdir(CLEANUP => 1); +my $xcatroot = File::Spec->catdir($tmpdir, 'opt', 'xcat'); +my $probe_root = File::Spec->catdir($xcatroot, 'probe'); +my $bin_dir = File::Spec->catdir($xcatroot, 'bin'); +my $subcmd_dir = File::Spec->catdir($probe_root, 'subcmds'); +my $helper_dir = File::Spec->catdir($probe_root, 'lib', 'perl', 'xCAT'); + +make_path($probe_root, $bin_dir); +copy_tree(File::Spec->catdir($repo_root, 'xCAT-probe', 'lib'), File::Spec->catdir($probe_root, 'lib')); +copy_tree(File::Spec->catdir($repo_root, 'xCAT-probe', 'subcmds'), $subcmd_dir); + +my $xcatprobe_source = File::Spec->catfile($repo_root, 'xCAT-probe', 'xcatprobe'); +my $xcatprobe = File::Spec->catfile($bin_dir, 'xcatprobe'); +copy($xcatprobe_source, $xcatprobe) or die "copy $xcatprobe_source: $!"; +chmod 0755, $xcatprobe or die "chmod $xcatprobe: $!"; + +make_path($helper_dir, File::Spec->catdir($subcmd_dir, 'bin')); +for my $helper (@helpers) { + my $source = File::Spec->catfile($repo_root, 'perl-xCAT', 'xCAT', $helper); + my $destination = File::Spec->catfile($helper_dir, $helper); + copy($source, $destination) or die "copy $source: $!"; + chmod 0644, $destination or die "chmod $destination: $!"; +} + +my $xcatclient = File::Spec->catfile($bin_dir, 'xcatclient'); +write_file($xcatclient, "#!/bin/sh\nprintf '[ok]:dummy xcatclient\\n'\n"); +chmod 0755, $xcatclient or die "chmod $xcatclient: $!"; + +local $ENV{XCATROOT} = $xcatroot; +local $ENV{PATH} = "$bin_dir:$ENV{PATH}"; +local $ENV{PERL5LIB}; +local $ENV{PERL5OPT}; +local $ENV{PERLLIB}; +delete $ENV{PERL5LIB}; +delete $ENV{PERL5OPT}; +delete $ENV{PERLLIB}; + +for my $subcommand (@affected_subcommands) { + my $command = File::Spec->catfile($subcmd_dir, $subcommand); + my ($rc, $output) = run_command($command, '-T'); + is($rc, 0, "$subcommand self-test exits successfully") or diag($output); + like($output, qr/^\[ok\]\s*:/m, "$subcommand self-test reports ready"); +} + +my ($list_rc, $list_output) = run_command($xcatprobe, '-l'); +is($list_rc, 0, 'xcatprobe list exits successfully') or diag($list_output); +my %listed = map { /^([^\s].*?)\s/ ? ($1 => 1) : () } split /\n/, $list_output; +for my $subcommand (@affected_subcommands) { + ok($listed{$subcommand}, "xcatprobe lists $subcommand") or diag($list_output); +} + +done_testing(); + +sub read_file { + my ($file) = @_; + my $path = File::Spec->catfile($repo_root, $file); + + open(my $fh, '<', $path) or die "open $path: $!"; + my $contents = do { local $/; <$fh> }; + close($fh) or die "close $path: $!"; + return $contents; +} + +sub copy_tree { + my ($source, $destination) = @_; + my $rc = system('cp', '-R', $source, $destination); + is($rc, 0, "copied $source into the package fixture") + or BAIL_OUT("unable to create package fixture from $source"); +} + +sub run_command { + my (@command) = @_; + open(my $fh, '-|', @command) or die "run @command: $!"; + my $output = do { local $/; <$fh> }; + close($fh); + return ($? >> 8, $output // ''); +} + +sub write_file { + my ($path, $contents) = @_; + open(my $fh, '>', $path) or die "open $path: $!"; + print {$fh} $contents; + close($fh) or die "close $path: $!"; +} diff --git a/xCAT-test/xCAT-test.spec b/xCAT-test/xCAT-test.spec index ed97a2d0d..d4f759221 100644 --- a/xCAT-test/xCAT-test.spec +++ b/xCAT-test/xCAT-test.spec @@ -11,7 +11,10 @@ Vendor: IBM Corp. Distribution: %{?_distribution:%{_distribution}}%{!?_distribution:%{_vendor}} Prefix: /opt/xcat BuildRoot: /var/tmp/%{name}-%{version}-%{release}-root +# Pod::Html ships inside the core perl package on SUSE; there is no perl-Pod-Html +%if !0%{?suse_version} BuildRequires: perl-Pod-Html +%endif # AIX will build with an arch of "ppc" %ifos linux diff --git a/xCAT-vlan/xCAT-vlan.spec b/xCAT-vlan/xCAT-vlan.spec index 69e3c5cbe..883f714d1 100644 --- a/xCAT-vlan/xCAT-vlan.spec +++ b/xCAT-vlan/xCAT-vlan.spec @@ -11,7 +11,10 @@ Vendor: IBM Corp. Distribution: %{?_distribution:%{_distribution}}%{!?_distribution:%{_vendor}} Prefix: /opt/xcat BuildRoot: /var/tmp/%{name}-%{version}-%{release}-root +# Pod::Html ships inside the core perl package on SUSE; there is no perl-Pod-Html +%if !0%{?suse_version} BuildRequires: perl-Pod-Html +%endif %ifos linux BuildArch: noarch diff --git a/xCAT/xCAT.spec b/xCAT/xCAT.spec index b5e665def..0ccc8a945 100644 --- a/xCAT/xCAT.spec +++ b/xCAT/xCAT.spec @@ -64,13 +64,14 @@ Requires: httpd nfs-utils nmap bind perl(CGI) # on RHEL7, need to specify it explicitly Requires: net-tools Requires: /usr/bin/killall -# On RHEL this pulls in dhcp, on SLES it pulls in dhcp-server. EL10 uses Kea. -%if 0%{?rhel} >= 10 -Requires: kea -Requires: kea-hooks -%else -Requires: /usr/sbin/dhcpd -%endif +# DHCP backend resolved at INSTALL time (not build time) via an RPM rich +# dependency, so a single flat xcat-core build is correct on every EL: el10+ +# dropped ISC dhcp from its distro and uses Kea; el8/el9 use ISC dhcpd. SLES +# has no "system-release" provide, so the condition is false there and it +# falls to dhcp-server (/usr/sbin/dhcpd), preserving prior behavior. +# system-release is versioned per release package (el10=10.x, el9=9.x, el8=8.x). +Requires: (kea if (system-release >= 10) else /usr/sbin/dhcpd) +Requires: (kea-hooks if (system-release >= 10)) # On RHEL this pulls in openssh-server, on SLES it pulls in openssh Requires: /usr/bin/ssh %if %nots390x diff --git a/xCATsn/xCATsn.spec b/xCATsn/xCATsn.spec index b184bc02e..a24b6979d 100644 --- a/xCATsn/xCATsn.spec +++ b/xCATsn/xCATsn.spec @@ -48,13 +48,14 @@ Requires: /usr/bin/killall Requires: /usr/bin/bc # yaboot-xcat is pulled in so any SN can manage ppc nodes Requires: httpd nfs-utils nmap bind -# On RHEL this pulls in dhcp, on SLES it pulls in dhcp-server. EL10 uses Kea. -%if 0%{?rhel} >= 10 -Requires: kea -Requires: kea-hooks -%else -Requires: /usr/sbin/dhcpd -%endif +# DHCP backend resolved at INSTALL time (not build time) via an RPM rich +# dependency, so a single flat xcat-core build is correct on every EL: el10+ +# dropped ISC dhcp from its distro and uses Kea; el8/el9 use ISC dhcpd. SLES +# has no "system-release" provide, so the condition is false there and it +# falls to dhcp-server (/usr/sbin/dhcpd), preserving prior behavior. +# system-release is versioned per release package (el10=10.x, el9=9.x, el8=8.x). +Requires: (kea if (system-release >= 10) else /usr/sbin/dhcpd) +Requires: (kea-hooks if (system-release >= 10)) # On RHEL this pulls in openssh-server, on SLES it pulls in openssh Requires: /usr/bin/ssh %ifnarch s390x