diff --git a/.github/workflows/genesis_openembedded.yml b/.github/workflows/genesis_openembedded.yml new file mode 100644 index 000000000..b2ee14521 --- /dev/null +++ b/.github/workflows/genesis_openembedded.yml @@ -0,0 +1,54 @@ +name: genesis_openembedded +on: + pull_request: + paths: + - 'xCAT-genesis-builder/oe/**' + - 'xCAT-genesis-scripts/**' + - '.github/workflows/genesis_openembedded.yml' + workflow_dispatch: +jobs: + genesis_openembedded: + name: OpenEmbedded metadata + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + with: + python-version: '3.12' + cache: pip + cache-dependency-path: xCAT-genesis-builder/oe/requirements.txt + - name: Install OpenEmbedded host dependencies + run: >- + sudo env DEBIAN_FRONTEND=noninteractive apt-get install -y + --no-install-recommends --no-install-suggests + build-essential chrpath cpio diffstat file gawk git locales + lz4 openssl socat texinfo unzip wget xz-utils zstd + - name: Install KAS + run: python -m pip install -r xCAT-genesis-builder/oe/requirements.txt + - name: Enable BitBake user namespaces + run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 + - name: Validate every architecture configuration + shell: bash + env: + KAS_WORK_DIR: ${{ runner.temp }}/kas-work + run: | + set -euo pipefail + for architecture in x86 x86_64 ppc64 ppc64le armv7hf aarch64 riscv64 s390x; do + export KAS_BUILD_DIR="${RUNNER_TEMP}/kas-build-${architecture}" + configuration="xCAT-genesis-builder/oe/kas/${architecture}.yml" + kas dump "${configuration}" >/dev/null + kas shell "${configuration}" -c 'bitbake -p' + done + - name: Validate the image task graph + shell: bash + env: + KAS_WORK_DIR: ${{ runner.temp }}/kas-work + KAS_BUILD_DIR: ${{ runner.temp }}/kas-build-x86_64 + run: | + set -euo pipefail + kas shell xCAT-genesis-builder/oe/kas/x86_64.yml -c \ + 'bitbake -n xcat-genesis-image xcat-genesis-extension-smoke' + export KAS_BUILD_DIR="${RUNNER_TEMP}/kas-build-s390x" + kas shell xCAT-genesis-builder/oe/kas/s390x.yml -c \ + 'bitbake -n xcat-genesis-image xcat-genesis-extension-smoke' diff --git a/.github/workflows/xcat_test.yml b/.github/workflows/xcat_test.yml index 04d50aff0..d23185138 100644 --- a/.github/workflows/xcat_test.yml +++ b/.github/workflows/xcat_test.yml @@ -10,49 +10,3 @@ jobs: run: sudo env DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends --no-install-suggests bats build-essential fakeroot reprepro devscripts debhelper libcapture-tiny-perl libfile-slurper-perl libjson-perl libparallel-forkmanager-perl libsoap-lite-perl libdbi-perl libcgi-pm-perl quilt openssh-server dpkg looptools genometools software-properties-common - name: Run tests run: perl github_action_xcat_test.pl - - genesis_openembedded: - name: OpenEmbedded metadata - runs-on: ubuntu-24.04 - timeout-minutes: 30 - steps: - - uses: actions/checkout@v6 - - uses: actions/setup-python@v6 - with: - python-version: '3.12' - cache: pip - cache-dependency-path: xCAT-genesis-builder/oe/requirements.txt - - name: Install OpenEmbedded host dependencies - run: >- - sudo env DEBIAN_FRONTEND=noninteractive apt-get install -y - --no-install-recommends --no-install-suggests - build-essential chrpath cpio diffstat file gawk git locales - lz4 openssl socat texinfo unzip wget xz-utils zstd - - name: Install KAS - run: python -m pip install -r xCAT-genesis-builder/oe/requirements.txt - - name: Enable BitBake user namespaces - run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 - - name: Validate every architecture configuration - shell: bash - env: - KAS_WORK_DIR: ${{ runner.temp }}/kas-work - run: | - set -euo pipefail - for architecture in x86 x86_64 ppc64 ppc64le armv7hf aarch64 riscv64 s390x; do - export KAS_BUILD_DIR="${RUNNER_TEMP}/kas-build-${architecture}" - configuration="xCAT-genesis-builder/oe/kas/${architecture}.yml" - kas dump "${configuration}" >/dev/null - kas shell "${configuration}" -c 'bitbake -p' - done - - name: Validate the image task graph - shell: bash - env: - KAS_WORK_DIR: ${{ runner.temp }}/kas-work - KAS_BUILD_DIR: ${{ runner.temp }}/kas-build-x86_64 - run: | - set -euo pipefail - kas shell xCAT-genesis-builder/oe/kas/x86_64.yml -c \ - 'bitbake -n xcat-genesis-image xcat-genesis-extension-smoke' - export KAS_BUILD_DIR="${RUNNER_TEMP}/kas-build-s390x" - kas shell xCAT-genesis-builder/oe/kas/s390x.yml -c \ - 'bitbake -n xcat-genesis-image xcat-genesis-extension-smoke' diff --git a/Version b/Version index ef0f38abe..7329e21c3 100644 --- a/Version +++ b/Version @@ -1 +1 @@ -2.19.0 +2.20.0 diff --git a/buildrpms.pl b/buildrpms.pl index fe2768d0f..42b87bb7a 100755 --- a/buildrpms.pl +++ b/buildrpms.pl @@ -657,14 +657,20 @@ sub setup_local_repos { } -# Index one repo dir with deterministic, upstream-matching metadata. createrepo_c's -# defaults already emit primary/filelists/other as *.xml.zst plus *.sqlite.bz2 -# (--database), exactly the upstream shape; --set-timestamp-to-revision pins the -# repomd timestamp to SOURCE_DATE_EPOCH. +# Index one repo dir with deterministic, upstream-matching metadata: primary/filelists/other as +# *.xml.zst, with --set-timestamp-to-revision pinning the repomd timestamp to SOURCE_DATE_EPOCH. +# +# NO --database. It writes *.sqlite.bz2, and building those needs SQLite, which needs POSIX +# locks. A build tree can live on an NFS re-export, where the kernel refuses locks outright: +# every attempt answers errno 524, and createrepo_c then dies on every target with +# "Cannot open .repodata/primary.sqlite: Can not create db_info table: disk I/O error". +# Without --database it succeeds there. +# +# Nothing this project ships reads the sqlite metadata. dnf on el8+ and zypper both read the XML. sub createrepo_dir { my ($dir, $extra) = @_; $extra //= ''; - sh_or_die(qq(createrepo_c --update --database ) + sh_or_die(qq(createrepo_c --update ) . qq(--revision "$SOURCE_DATE_EPOCH" --set-timestamp-to-revision $extra "$dir"), "Failed to createrepo_c $dir\n"); } diff --git a/docs/source/developers/index.rst b/docs/source/developers/index.rst index a56f0e63a..5bca03e07 100644 --- a/docs/source/developers/index.rst +++ b/docs/source/developers/index.rst @@ -10,3 +10,4 @@ This page is for developers interested in working with xCAT. license/index.rst github/index.rst guides/index.rst + releases/index.rst diff --git a/docs/source/developers/releases/checklist.rst b/docs/source/developers/releases/checklist.rst new file mode 100644 index 000000000..db884ad2d --- /dev/null +++ b/docs/source/developers/releases/checklist.rst @@ -0,0 +1,321 @@ +Release Checklist +================= + +A maintainer with publish access to the download server does these steps. The build hosts, the +signing key and the publish procedure are in the private repository of the maintainers. This page +does not repeat them. + +The steps use these names: + +* ``X.Y.Z``: the new release, for example ``2.19.1``. +* ``P``: the tag of the previous release in the same series, for example ``2.19.0``. For X.Y.0, + use the tag of the previous minor release. +* ````: the full id of the release commit. +* ````: the name of the release notes page on the wiki. + +Plan +---- + +#. Open the milestone of the release: ``X.Y.Z`` for a patch release, ``X.Y`` for X.Y.0. Make sure + that each closed pull request in it is merged on the ``X.Y`` branch. For a backported fix, the + ``[Backport X.Y]`` pull request must also be merged. + +#. Move each open item to the next milestone, or finish it before the release candidates start. + +Start a Minor Release +--------------------- + +Do these steps once for X.Y.0, when the release candidates start. You must have admin rights on +``xcat2/xcat-core``. + +#. Create the ``X.Y`` branch from the head of ``master``: :: + + $ gh api -X POST repos/xcat2/xcat-core/git/refs -f ref=refs/heads/X.Y -f sha= + +#. Add ``refs/heads/X.Y`` to the target branches of the ``release-branches`` ruleset, in + Settings, Rules, Rulesets. + +#. Create the backport label: :: + + $ gh label create "backport X.Y" -R xcat2/xcat-core --color 0e8a16 \ + --description "Backport this PR to the X.Y maintenance branch" + +#. Create the milestone ``X.Y.1`` and the milestone of the next minor release, if they do not + exist. + +#. Open a pull request against ``master`` that sets ``Version`` and the ``release`` value in + ``docs/source/conf.py`` to the next minor release, for example ``2.21.0`` after the ``2.20`` + branch is created. + +After these steps, a fix for X.Y.0 goes to ``master`` with the label ``backport X.Y`` and the +milestone ``X.Y``. + +When a series gets no more releases, delete its backport label. + +Prepare the Release Branch +-------------------------- + +#. Add a row to ``docs/source/overview/_files/X.Y.x.csv``. For X.Y.0, create the file and add a + section for it at the top of ``docs/source/overview/xcat2_release.rst``. A row has this form: :: + + 2.18.0,2026-06-22,"RHEL 10,AlmaLinux 10",`2.18.0 Release Notes `_ + + Open the change against ``master`` with the label ``backport X.Y``, so that the row is on both + branches before the tag. + +#. Make sure that ``Version`` and the ``release`` value in ``docs/source/conf.py`` on the ``X.Y`` + branch are both ``X.Y.Z``. + +Release Candidates +------------------ + +A release candidate is a build of the head of the ``X.Y`` branch. Do not create a branch or a tag +for a candidate, such as ``release/X.Y-rc1`` or ``X.Y.Z-rc1``. + +#. Build a candidate from the head of the ``X.Y`` branch, and sign the packages and the + repository metadata. Build xcat-dep too if its packages changed after the previous release. + +#. Publish the candidate to a staging location only. Do not publish a candidate to ``X.Y`` or to + ``latest``. + +#. Test the candidate on each operating system and architecture that the release notes will name. + Install a management node, and provision stateful and stateless compute nodes. Record what you + tested, because the release notes report it. + +#. If a test fails, fix the problem on ``master``, backport the fix, and build the next candidate + from the new head of ``X.Y``. + +The ``COMMIT_ID_LONG`` line in the ``buildinfo.txt`` file of a candidate identifies it. + +Build and Publish +----------------- + +The release is the last candidate that passed the tests. + +#. Record the full id of the release commit. This is the ``COMMIT_ID_LONG`` of that candidate. + +#. Publish the packages of that candidate to ``repos/yum/X.Y/`` and ``repos/apt/X.Y/``. Do not + build them again. A new build from the same commit can give different packages, for example a + Genesis image with a newer kernel. If you must build a package again, test the new build as a + candidate before you publish it. + +#. Upload the offline bundles: :: + + xcat-core/X.Y.x_Linux/xcat-core-X.Y.Z-linux.tar.bz2 + xcat-core/X.Y.x_Ubuntu/xcat-core-X.Y.Z-ubuntu.tar.bz2 + xcat-dep/2.x_Linux/xcat-dep-X.Y.Z-linux.tar.bz2 + xcat-dep/2.x_Ubuntu/xcat-dep-X.Y.Z-ubuntu.tar.bz2 + +#. Make sure that the ``.repo`` files in the published repositories point at the published + location, not at ``devel``. + +#. If X.Y is the newest series, point ``latest`` at ``X.Y`` for yum and for apt. + +#. Make sure that ``devel`` does not serve a build that is older than the release. + +Verify the Published Packages +----------------------------- + +#. Check that ``buildinfo.txt`` shows ``VERSION=X.Y.Z`` and ``COMMIT_ID_LONG=``. If you + changed ``latest``, check it too: :: + + $ curl -fsS https://xcat.org/files/xcat/repos/yum/X.Y/xcat-core/buildinfo.txt + $ curl -fsS https://xcat.org/files/xcat/repos/yum/latest/xcat-core/buildinfo.txt + +#. Check the signatures of the repository metadata against the key that ``xCAT-release`` ships. + Use a keyring that holds only ``xCAT-release/RPM-GPG-KEY-xCAT``, so that no other key can pass + the check: :: + + $ gpg --dearmor xcat-release.gpg + $ curl -fsSO https://xcat.org/files/xcat/repos/yum/X.Y/xcat-core/repodata/repomd.xml + $ curl -fsSO https://xcat.org/files/xcat/repos/yum/X.Y/xcat-core/repodata/repomd.xml.asc + $ gpgv --keyring ./xcat-release.gpg repomd.xml.asc repomd.xml + + Do the same for ``xcat-dep/common`` and for each ``xcat-dep/rh/`` directory. For apt, + check ``dists//InRelease`` of xcat-core and xcat-dep for each codename: :: + + $ curl -fsSO https://xcat.org/files/xcat/repos/apt/X.Y/xcat-core/dists//InRelease + $ gpgv --keyring ./xcat-release.gpg InRelease + +#. Check that each offline bundle URL answers with HTTP 200: :: + + $ curl -fsSI https://xcat.org/files/xcat/xcat-core/X.Y.x_Linux/xcat-core-X.Y.Z-linux.tar.bz2 + +#. Install xCAT with signature checks on a new EL host and on a new Ubuntu host. ``go-xcat`` + turns these checks off with ``dnf --nogpgcheck`` and ``apt-get --allow-unauthenticated``, so a + ``go-xcat`` installation does not prove the signatures. + + On EL, use the three published ``.repo`` files, which set ``gpgcheck=1``. Enable the other + repositories that the installation guide requires first. The published files point at + ``latest``. For an older series, replace ``latest`` with ``X.Y`` in the three files. Install the + Genesis image for the host architecture by name, because ``xCAT`` only recommends it. :: + + $ curl -fsSo /etc/yum.repos.d/xcat-core.repo \ + https://xcat.org/files/xcat/repos/yum/X.Y/xcat-core/xcat-core.repo + $ curl -fsSo /etc/yum.repos.d/xcat-dep.repo \ + https://xcat.org/files/xcat/repos/yum/X.Y/xcat-dep/rh//xcat-dep.repo + $ curl -fsSo /etc/yum.repos.d/xcat-dep-common.repo \ + https://xcat.org/files/xcat/repos/yum/X.Y/xcat-dep/common/xcat-dep-common.repo + $ dnf install -y xCAT xCAT-genesis-openembedded- + $ lsxcatd -v + + On Ubuntu, add the repositories with their signing keys. ``apt-get update`` must not report a + signature error. :: + + $ url=https://xcat.org/files/xcat/repos/apt/X.Y + $ codename=$(. /etc/os-release && echo "$VERSION_CODENAME") + $ curl -fsSL $url/xcat-core/apt.key | gpg --dearmor -o /usr/share/keyrings/xcat-core.gpg + $ curl -fsSL $url/xcat-dep/apt.key | gpg --dearmor -o /usr/share/keyrings/xcat-dep.gpg + $ echo "deb [signed-by=/usr/share/keyrings/xcat-core.gpg] $url/xcat-core $codename main" \ + >/etc/apt/sources.list.d/xcat.list + $ echo "deb [signed-by=/usr/share/keyrings/xcat-dep.gpg] $url/xcat-dep $codename main" \ + >>/etc/apt/sources.list.d/xcat.list + $ apt-get update + $ apt-get install -y xcat + $ lsxcatd -v + +#. Install xCAT with ``go-xcat`` on another new EL host and on a new Ubuntu host, and check the + version: :: + + $ ./go-xcat -x X.Y -y install + $ lsxcatd -v + + If you changed ``latest``, also install once without ``-x X.Y``. + +Release Notes +------------- + +The release notes are a page on the `xcat-core wiki `_. + +#. Name the page ``XCAT_X.Y_Release_Notes`` for X.Y.0, and ``XCAT_X.Y.Z_Release_Notes`` for a + patch release. Do not rename the page later, because other pages and https://xcat.org link to + it. + +#. Start from the notes of the previous release of the same kind, and keep the order of the + sections: :: + + # xCAT X.Y.Z Release Notes (Month D, YYYY) + ## Operating System Support + ## Highlighted Changes + ## Download xCAT + ### Offline tarball bundles + ## Validation + ## Key Issues Resolved + ## Documentation + +#. Run each command in "Download xCAT" on a new host, and open each link. + +#. Report only the tests that were done, with the operating system, the architecture and the + node types. + +#. For a patch release, add a line at the top of the X.Y notes page that links to the new page. + +Tag +--- + +Create a signed annotated tag on the release commit, and push only the tag: :: + + $ git fetch upstream + $ git tag -s X.Y.Z -m "xCAT X.Y.Z" + $ git tag -v X.Y.Z + $ git push upstream refs/tags/X.Y.Z + +Do not create the tag from the GitHub release page. That page creates a lightweight tag with no +signature. + +GitHub Release +-------------- + +Write the release text to a file, for example ``notes.md``: :: + + The release notes for X.Y.Z are available at https://github.com/xcat2/xcat-core/wiki/ + + Full changelog: https://github.com/xcat2/xcat-core/compare/P...X.Y.Z + +Then create the release from the tag: :: + + $ gh release create X.Y.Z -R xcat2/xcat-core --verify-tag --title X.Y.Z --notes-file notes.md + +For a patch release of a series that is not the newest, add ``--latest=false``. + +Read the Docs +------------- + +Read the Docs builds a documentation version for each new tag, and ``stable`` follows the highest +version tag. + +#. Open https://xcat-docs.readthedocs.io/en/X.Y.Z/, and check that the build passed and that the + title shows X.Y.Z. + +#. If X.Y.Z is the highest release, check that https://xcat-docs.readthedocs.io/en/stable/ shows + X.Y.Z. For a patch release of an older series, check that ``stable`` did not change. + +Wiki Index Pages +---------------- + +#. On the ``test_sidebar`` page, which holds the News list, add a line at the top: :: + + * Mon DD, YYYY: [xCAT X.Y.Z]() released. + +#. For X.Y.0, add a row for X.Y to the "General Release Information and Planning" table on the + ``Home`` page, and move the ``(stable)`` marker to it. + +Website +------- + +The ``xcat2/xcat2.github.io`` repository holds the pages of https://xcat.org. The default +downloads on these pages must stay on the same series as ``latest``. + +If X.Y is the newest series: + +#. In ``index.html``, change the release line and the release notes link. + +#. In ``download.html``, change the version and the offline bundle links. + +#. In ``footer.html``, change the release news link. + +For a patch release of an older series, do not change the default version on these pages. Add the +older release to ``download.html`` as a separate entry. + +Commit the change in the repository, copy the changed files to the web server, and open each +changed link on https://xcat.org. + +Announcement +------------ + +Send an email to xcat-user@lists.sourceforge.net with the subject +``Announcement: xCAT X.Y.Z released``. Use the announcement of the previous release as the model: :: + + Dear xCAT community, + + We are pleased to announce the release of xCAT X.Y.Z. + + + Highlights: + * + - + + Full release notes: https://github.com/xcat2/xcat-core/wiki/ + Downloads: https://github.com/xcat2/xcat-core/releases/tag/X.Y.Z + + + + We welcome your feedback, bug reports, and contributions: + https://github.com/xcat2/xcat-core + + Best regards, + + on behalf of the xCAT Consortium + +Keep the announcement of a patch release short, and name the fixes it contains. + +After the Release +----------------- + +#. Close the milestone of the release. Create the milestone ``X.Y.(Z+1)`` if it does not exist, + and move the open items to it. + +#. Open a pull request against ``X.Y`` that sets ``Version`` and the ``release`` value in + ``docs/source/conf.py`` to ``X.Y.(Z+1)``. + +#. For X.Y.0, make sure that the steps in "Start a Minor Release" are complete. diff --git a/docs/source/developers/releases/index.rst b/docs/source/developers/releases/index.rst new file mode 100644 index 000000000..56f082c9b --- /dev/null +++ b/docs/source/developers/releases/index.rst @@ -0,0 +1,68 @@ +Releases +======== + +This section describes how xCAT versions are numbered, how a fix reaches a maintained release, +and how a maintainer publishes a release. + +.. toctree:: + :maxdepth: 2 + + pull_requests.rst + checklist.rst + +Branches and Versions +--------------------- + +xCAT has one development branch and one maintenance branch for each minor release. + +* ``master`` is the development line of the next minor release. +* ``X.Y``, for example ``2.19``, is the maintenance branch of the X.Y series. It receives only + fixes that are backported from ``master``. The patch releases X.Y.1, X.Y.2 and later are tagged + on it. + +The ``Version`` file at the top of the tree is the only place that holds the version. +``buildrpms.pl`` and ``builddebs.pl`` read it, and each build adds the release string +``snapYYYYMMDDHHMM`` from the commit time. A release is the build of the tagged commit. There is no +separate release build. + +.. list-table:: + :header-rows: 1 + + * - Branch + - ``Version`` holds + - Example after 2.19.0 is released + * - ``master`` + - the next minor release + - ``2.20.0`` + * - ``X.Y`` + - the next release of the series + - ``2.19.1`` + +The ``release`` value in ``docs/source/conf.py`` must be the same as ``Version`` on the same +branch, because Read the Docs shows it in the title of each page. Change the two files in the same +pull request. + +A released version number is never used again. The :doc:`checklist` changes ``Version`` so that +no build after a release carries the number of that release. + +Tags +---- + +Each release has a signed annotated tag named ``X.Y.Z``, for example ``2.19.1``. The tag points at +the commit that the published packages were built from. Release candidates do not get tags, +because Read the Docs builds a documentation version for each new tag. + +Package Channels +---------------- + +The download server has three channels under https://xcat.org/files/xcat/repos/yum/ and +https://xcat.org/files/xcat/repos/apt/: + +* ``devel``: development snapshots from ``master``, in ``devel/core-snap`` and ``devel/xcat-dep``. +* ``X.Y``: the most recent release of the X.Y series. A patch release replaces the contents of + its series directory. +* ``latest``: the most recent release series. ``go-xcat`` installs from ``latest`` by default, so + when ``latest`` changes, new installations get the new release. + +The offline bundles are in https://xcat.org/files/xcat/xcat-core/ and +https://xcat.org/files/xcat/xcat-dep/. diff --git a/docs/source/developers/releases/pull_requests.rst b/docs/source/developers/releases/pull_requests.rst new file mode 100644 index 000000000..51b3758ea --- /dev/null +++ b/docs/source/developers/releases/pull_requests.rst @@ -0,0 +1,64 @@ +Pull Requests and Backports +=========================== + +Open every change against ``master``. A fix reaches a maintenance branch as a backport of the +merged ``master`` pull request. + +Labels and Milestones +--------------------- + +Set a label and a milestone on each pull request against ``master``. The milestone is the first +release that contains the change. + +.. list-table:: + :header-rows: 1 + :widths: 50 20 30 + + * - Change + - Label + - Milestone + * - A fix that users of the maintained series need: a regression, a failure on a supported + platform, a security fix, or a change that the maintenance branch needs to build or test + - ``backport X.Y`` + - the next release of the series, for example ``2.19.1`` + * - Any other change: a feature, a refactor, a change to tests only, documentation, or a fix + for an old or high-risk problem + - none + - the next minor release, for example ``2.20`` + +Do not add a backport label to a change of the ``Version`` file. + +When a patch release ships, the next patch milestone replaces it. After 2.19.1, the milestone for +backported fixes is ``2.19.2``. + +Automatic Backports +------------------- + +The ``backport`` workflow in ``.github/workflows/backport.yml`` starts when a pull request with a +``backport X.Y`` label is merged. It also starts when the label is added to a pull request that is +already merged. The workflow: + +#. cherry-picks the commits of the pull request onto the ``X.Y`` branch, +#. opens a pull request named ``[Backport X.Y] ``, +#. approves that pull request and turns on auto-merge. + +The backport pull request merges when ``xcat_pr_test`` passes. Do not set a milestone on it. The +milestone stays on the original pull request. + +Manual Backports +---------------- + +If the cherry-pick has a conflict, the workflow fails and comments on the original pull request. +Then do the backport by hand from your fork: :: + + $ git fetch upstream + $ git switch -c backport--to-X.Y upstream/X.Y + $ git cherry-pick -x ... + $ git push origin backport--to-X.Y + +Resolve each conflict before you continue the cherry-pick. Open a pull request against ``X.Y`` +named ``[Backport X.Y] ``, and set the patch release milestone on it. + +The ``release-branches`` ruleset protects each maintenance branch in the same way as ``master``. +A change needs a pull request, an approval and a passing ``xcat_pr_test``, and the branch refuses +direct pushes. diff --git a/xCAT-client/bin/pgsqlsetup b/xCAT-client/bin/pgsqlsetup index f13ca21dc..9a7c962a1 100755 --- a/xCAT-client/bin/pgsqlsetup +++ b/xCAT-client/bin/pgsqlsetup @@ -1095,6 +1095,23 @@ sub setupxcatdb } $pgsql->soft_close(); + # PostgreSQL 15 removed the default CREATE privilege on schema public + # for PUBLIC, so xcatadm must be granted it explicitly before xcatd + # can create the xCAT tables. No-op on earlier releases. + my $grantsql = "GRANT USAGE, CREATE ON SCHEMA public TO xcatadm;"; + my $grantcmd; + if ($::osname eq 'AIX') + { + $grantcmd = "$::installdir/bin/psql -d $::dbname -U postgres -v ON_ERROR_STOP=1 -c \"$grantsql\""; + } else { + $grantcmd = "$::pgcmddir/psql -d $::dbname -U postgres -v ON_ERROR_STOP=1 -c \"$grantsql\""; + } + $rc = &runpgcmd_chkoutput($grantcmd); + if ($rc > 0) { + xCAT::MsgUtils->message("E", + "Failed granting xcatadm privileges on schema public in $::dbname"); + exit(1); + } # setup root user $pgsql = new Expect; diff --git a/xCAT-server/share/xcat/netboot/ubuntu/genimage b/xCAT-server/share/xcat/netboot/ubuntu/genimage index 17005cdb5..970a97d59 100755 --- a/xCAT-server/share/xcat/netboot/ubuntu/genimage +++ b/xCAT-server/share/xcat/netboot/ubuntu/genimage @@ -1827,8 +1827,13 @@ EOMS } } - if (-d "$rootimg_dir/lib/firmware/") { - system("cp -r $rootimg_dir/lib/firmware/* /tmp/xcatinitrd.$$/lib/firmware"); + # The whole firmware tree is 666 MB on Ubuntu 26.04, and grub2 on a pseries node cannot + # load an initrd of that size. The initrd only has to reach the network. + foreach my $firmware ( + initrd_firmware_files($rootimg_dir, \@filestoadd, $kernelver, $customdir, $pathtofiles)) + { + copy_initrd_file("$rootimg_dir/lib/firmware/$firmware", + "/tmp/xcatinitrd.$$/lib/firmware/$firmware"); } if (-d "$rootimg_dir/lib/modules/$kernelver/") { @@ -1892,6 +1897,79 @@ sub isnetdriver { } } +#------------------------------------------------------------------------------- + +=head3 initrd_firmware_files + + Descriptions: Select the firmware that the drivers in the netboot initrd ask for. + Arguments: + rootimg_dir - the root image directory + filestoadd - reference to the list of files that go into the initrd. A module + is either a plain path or a [ source, destination ] pair. + kernelver - the kernel release, which names one of the firmware directories + module_dirs - directories searched for a module before the root image, in the + same order the copy step searches them + Returns: the firmware files that exist, as paths below lib/firmware + +=cut + +#------------------------------------------------------------------------------- +sub initrd_firmware_files { + my ($rootimg_dir, $filestoadd, $kernelver, @module_dirs) = @_; + + return () unless (-d "$rootimg_dir/lib/firmware"); + + my %wanted; + foreach my $entry (@$filestoadd) { + my $module = ref($entry) ? $entry->[0] : $entry; + next unless ($module =~ /\.ko(?:\.(?:gz|xz|zst))?$/); + + # Ask about the module the copy step takes, which is the custom one where there is + # one. A custom driver is not in the root image at all. + my $source; + foreach my $dir (grep { defined && length } @module_dirs, $rootimg_dir) { + next unless (-f "$dir/$module"); + $source = "$dir/$module"; + last; + } + next unless (defined $source); + + my $out = `modinfo -F firmware "$source" 2>/dev/null`; + if ($?) { + print "Warning: cannot read the firmware of $module\n"; + next; + } + foreach my $name (split(/\n/, $out)) { + $name =~ s/^\s+|\s+$//g; + $wanted{$name} = 1 if (length $name); + } + } + + # The kernel looks for a firmware name in these directories below lib/firmware, in this + # order, and loads the first file it finds. Keep every one that exists, so an override + # still wins on the node. + my @dirs = ("updates", ""); + if (defined $kernelver and length $kernelver) { + unshift @dirs, "updates/$kernelver"; + splice(@dirs, 2, 0, $kernelver); + } + + my @files; + foreach my $name (sort keys %wanted) { + foreach my $dir (@dirs) { + my $prefix = length($dir) ? "$dir/" : ""; + + # Ubuntu keeps the firmware compressed and the kernel asks for the plain name. + foreach my $candidate ($name, "$name.zst", "$name.xz", "$name.gz") { + next unless (-e "$rootimg_dir/lib/firmware/$prefix$candidate"); + push @files, "$prefix$candidate"; + last; + } + } + } + return @files; +} + sub find_rootimg_file { my ($file) = @_; diff --git a/xCAT-test/unit/ubuntu_genimage_initrd_firmware.t b/xCAT-test/unit/ubuntu_genimage_initrd_firmware.t new file mode 100755 index 000000000..d01242385 --- /dev/null +++ b/xCAT-test/unit/ubuntu_genimage_initrd_firmware.t @@ -0,0 +1,148 @@ +#!/usr/bin/env perl +use strict; +use warnings; + +use File::Path qw(make_path); +use File::Spec; +use File::Temp qw(tempdir); +use FindBin; +use Test::More; + +# grub2 on a pseries node cannot load the netboot initrd when it carries the whole Ubuntu +# firmware tree. Run genimage's firmware step over a root image that holds firmware no +# driver in the initrd asks for, and read what the step put in the initrd. + +my $repo_root = File::Spec->rel2abs(File::Spec->catdir($FindBin::Bin, '..', '..')); +my $genimage = File::Spec->catfile( + $repo_root, 'xCAT-server', 'share', 'xcat', 'netboot', 'ubuntu', 'genimage'); +die "genimage not found at $genimage\n" unless -f $genimage; + +my $src = do { local $/; open my $fh, '<', $genimage or die $!; <$fh> }; + +# The step sits between the loop that copies the initrd files and the copy of the module +# index. Both anchors hold whatever the firmware step itself looks like. +my ($step) = $src =~ m{ + copy_initrd_file\(\$srcpath,\ "/tmp/xcatinitrd\.\$\$/\$_"\);\n + \ {8}\}\n\ {4}\}\n + (.*?) + \n\ {4}if\ \(-d\ "\$rootimg_dir/lib/modules/\$kernelver/"\)\ \{ +}sx; +die "genimage no longer has a firmware step between the initrd file loop and the module index\n" + unless defined $step; + +my ($copy_initrd_file) = $src =~ m{^(sub copy_initrd_file \{.*?\n\}\n)}ms; +die "genimage no longer defines copy_initrd_file\n" unless defined $copy_initrd_file; + +# The helper the step calls once the firmware copy is filtered. A genimage that copies the +# tree whole has no such routine, and the step below then needs none. +my ($helper) = $src =~ m{^(sub initrd_firmware_files \{.*?\n\}\n)}ms; + +my $scratch = tempdir(CLEANUP => 1); +my $rootimg = "$scratch/rootimg"; +my $initrd_dir = "$scratch/initrd"; + +my $rewritten = ($step =~ s{/tmp/xcatinitrd\.\$\$}{$initrd_dir}g); +die "the firmware step no longer writes to /tmp/xcatinitrd.\$\$\n" unless $rewritten; + +# modinfo reports the firmware of a module. Answer for the modules of this root image only. +my $bin = "$scratch/bin"; +make_path($bin); +open(my $fake, '>', "$bin/modinfo") or die $!; +print $fake <<'SH'; +#!/bin/sh +case "$*" in + *mlx5_core*) echo mellanox/fw-a.mfa2 ;; + *bnx2x*) echo bnx2x/bnx2x-e2.fw ;; + *e1000e*) echo intel/absent-from-this-image.bin ;; + *custom_nic*) echo custom/custom-nic.bin ;; + *virtio_net*) : ;; + *) exit 1 ;; +esac +SH +close($fake); +chmod 0755, "$bin/modinfo"; +$ENV{PATH} = "$bin:$ENV{PATH}"; + +foreach my $file ( + 'lib/firmware/mellanox/fw-a.mfa2', + 'lib/firmware/updates/7.0.0/mellanox/fw-a.mfa2', + 'lib/firmware/custom/custom-nic.bin', + 'lib/firmware/bnx2x/bnx2x-e2.fw.zst', + 'lib/firmware/amdgpu/never-asked-for.bin', + 'lib/firmware/qcom/never-asked-for-either.bin', + 'lib/modules/7.0.0/kernel/drivers/net/virtio_net.ko', + 'lib/modules/7.0.0/kernel/drivers/net/mlx5_core.ko', + 'lib/modules/7.0.0/kernel/drivers/net/bnx2x.ko.zst', + 'lib/modules/7.0.0/kernel/drivers/net/e1000e.ko', + 'bin/busybox', + ) +{ + my $full = "$rootimg/$file"; + ($full =~ m{^(.*)/[^/]+$}) and make_path($1); + open(my $fh, '>', $full) or die $!; + print $fh "content of $file\n"; + close($fh); +} +make_path("$initrd_dir/lib/firmware"); + +# A driver the administrator put in the custom directory. genimage takes the module from there +# and not from the root image, so that is the file its firmware has to be read from. +my $customdir = "$scratch/custom"; +my $pathtofiles = "$scratch/pathtofiles"; +make_path("$customdir/lib/modules/7.0.0/kernel/drivers/net"); +make_path($pathtofiles); +open(my $custom, '>', "$customdir/lib/modules/7.0.0/kernel/drivers/net/custom_nic.ko") or die $!; +print $custom "content of a custom driver\n"; +close($custom); + +{ + package Scratch; + use strict; + use warnings; + use File::Basename; + use File::Copy; + use File::Path qw(mkpath); + our $rootimg_dir; + our $customdir; + our $pathtofiles; + our $kernelver; + our @filestoadd; + sub xdie { die @_ } +} + +## no critic (BuiltinFunctions::ProhibitStringyEval) +eval "package Scratch;\n$copy_initrd_file\n1" or die $@; +if (defined $helper) { + eval "package Scratch;\n$helper\n1" or die $@; +} + +$Scratch::rootimg_dir = $rootimg; +$Scratch::customdir = $customdir; +$Scratch::pathtofiles = $pathtofiles; +$Scratch::kernelver = '7.0.0'; +@Scratch::filestoadd = ( + [ 'lib/modules/7.0.0/kernel/drivers/net/virtio_net.ko', 'lib/virtio_net.ko' ], + [ 'lib/modules/7.0.0/kernel/drivers/net/mlx5_core.ko', 'lib/mlx5_core.ko' ], + [ 'lib/modules/7.0.0/kernel/drivers/net/bnx2x.ko.zst', 'lib/bnx2x.ko' ], + [ 'lib/modules/7.0.0/kernel/drivers/net/e1000e.ko', 'lib/e1000e.ko' ], + [ 'lib/modules/7.0.0/kernel/drivers/net/custom_nic.ko', 'lib/custom_nic.ko' ], + [ 'bin/busybox', 'bin/busybox' ], +); +eval "package Scratch;\nno strict 'vars';\n$step\n1" or die $@; + +ok(-e "$initrd_dir/lib/firmware/mellanox/fw-a.mfa2", + 'the initrd keeps the firmware a driver in it asks for'); +ok(-e "$initrd_dir/lib/firmware/bnx2x/bnx2x-e2.fw.zst", + 'a compressed firmware file answers the plain name the driver asks for'); +ok(!-e "$initrd_dir/lib/firmware/amdgpu/never-asked-for.bin", + 'the initrd does not carry firmware no driver in it asks for'); +ok(!-e "$initrd_dir/lib/firmware/qcom/never-asked-for-either.bin", + 'the whole firmware tree does not reach the initrd'); +ok(!-e "$initrd_dir/lib/firmware/intel/absent-from-this-image.bin", + 'a firmware name the root image does not have is left out'); +ok(-e "$initrd_dir/lib/firmware/custom/custom-nic.bin", + 'the firmware of a driver taken from the custom directory reaches the initrd'); +ok(-e "$initrd_dir/lib/firmware/updates/7.0.0/mellanox/fw-a.mfa2", + 'a firmware override under updates/ reaches the initrd'); + +done_testing();