From 9c2068e26c404002b31546c92ea2ed05d315254e Mon Sep 17 00:00:00 2001
From: Daniel Hilst <392820+dhilst@users.noreply.github.com>
Date: Mon, 14 Sep 2026 07:54:15 -0300
Subject: [PATCH 01/10] test(xcat-core): the Ubuntu netboot initrd carries the
 whole firmware tree

The Ubuntu genimage copies every file under the root image lib/firmware into the
netboot initrd. On Ubuntu 26.04 that tree is 666 MB, and grub2 on a pseries node
cannot load the initrd it produces.

The test runs the firmware step of genimage over a root image whose firmware tree
holds files no driver in the initrd asks for, and reads the initrd that comes out.
It fails on the present code, which copies the tree whole.

Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com>
---
 .../unit/ubuntu_genimage_initrd_firmware.t    | 124 ++++++++++++++++++
 1 file changed, 124 insertions(+)
 create mode 100755 xCAT-test/unit/ubuntu_genimage_initrd_firmware.t

diff --git a/xCAT-test/unit/ubuntu_genimage_initrd_firmware.t b/xCAT-test/unit/ubuntu_genimage_initrd_firmware.t
new file mode 100755
index 000000000..b51bc6ba1
--- /dev/null
+++ b/xCAT-test/unit/ubuntu_genimage_initrd_firmware.t
@@ -0,0 +1,124 @@
+#!/usr/bin/env perl
+use strict;
+use warnings;
+
+use File::Path qw(make_path);
+use File::Spec;
+use File::Temp qw(tempdir);
+use FindBin;
+use Test::More;
+
+# grub2 on a pseries node cannot load the netboot initrd when it carries the whole Ubuntu
+# firmware tree. Run genimage's firmware step over a root image that holds firmware no
+# driver in the initrd asks for, and read what the step put in the initrd.
+
+my $repo_root = File::Spec->rel2abs(File::Spec->catdir($FindBin::Bin, '..', '..'));
+my $genimage  = File::Spec->catfile(
+    $repo_root, 'xCAT-server', 'share', 'xcat', 'netboot', 'ubuntu', 'genimage');
+plan skip_all => "genimage not found at $genimage" unless -f $genimage;
+
+my $src = do { local $/; open my $fh, '<', $genimage or die $!; <$fh> };
+
+# The step sits between the loop that copies the initrd files and the copy of the module
+# index. Both anchors hold whatever the firmware step itself looks like.
+my ($step) = $src =~ m{
+    copy_initrd_file\(\$srcpath,\ "/tmp/xcatinitrd\.\$\$/\$_"\);\n
+    \ {8}\}\n\ {4}\}\n
+    (.*?)
+    \n\ {4}if\ \(-d\ "\$rootimg_dir/lib/modules/\$kernelver/"\)\ \{
+}sx;
+die "genimage no longer has a firmware step between the initrd file loop and the module index\n"
+  unless defined $step;
+
+my ($copy_initrd_file) = $src =~ m{^(sub copy_initrd_file \{.*?\n\}\n)}ms;
+die "genimage no longer defines copy_initrd_file\n" unless defined $copy_initrd_file;
+
+# The helper the step calls once the firmware copy is filtered. A genimage that copies the
+# tree whole has no such routine, and the step below then needs none.
+my ($helper) = $src =~ m{^(sub initrd_firmware_files \{.*?\n\}\n)}ms;
+
+my $scratch    = tempdir(CLEANUP => 1);
+my $rootimg    = "$scratch/rootimg";
+my $initrd_dir = "$scratch/initrd";
+
+my $rewritten = ($step =~ s{/tmp/xcatinitrd\.\$\$}{$initrd_dir}g);
+die "the firmware step no longer writes to /tmp/xcatinitrd.\$\$\n" unless $rewritten;
+
+# modinfo reports the firmware of a module. Answer for the modules of this root image only.
+my $bin = "$scratch/bin";
+make_path($bin);
+open(my $fake, '>', "$bin/modinfo") or die $!;
+print $fake <<'SH';
+#!/bin/sh
+case "$*" in
+    *mlx5_core*)  echo mellanox/fw-a.mfa2 ;;
+    *bnx2x*)      echo bnx2x/bnx2x-e2.fw ;;
+    *e1000e*)     echo intel/absent-from-this-image.bin ;;
+    *virtio_net*) : ;;
+    *)            exit 1 ;;
+esac
+SH
+close($fake);
+chmod 0755, "$bin/modinfo";
+$ENV{PATH} = "$bin:$ENV{PATH}";
+
+foreach my $file (
+    'lib/firmware/mellanox/fw-a.mfa2',
+    'lib/firmware/bnx2x/bnx2x-e2.fw.zst',
+    'lib/firmware/amdgpu/never-asked-for.bin',
+    'lib/firmware/qcom/never-asked-for-either.bin',
+    'lib/modules/7.0.0/kernel/drivers/net/virtio_net.ko',
+    'lib/modules/7.0.0/kernel/drivers/net/mlx5_core.ko',
+    'lib/modules/7.0.0/kernel/drivers/net/bnx2x.ko.zst',
+    'lib/modules/7.0.0/kernel/drivers/net/e1000e.ko',
+    'bin/busybox',
+  )
+{
+    my $full = "$rootimg/$file";
+    ($full =~ m{^(.*)/[^/]+$}) and make_path($1);
+    open(my $fh, '>', $full) or die $!;
+    print $fh "content of $file\n";
+    close($fh);
+}
+make_path("$initrd_dir/lib/firmware");
+
+{
+    package Scratch;
+    use strict;
+    use warnings;
+    use File::Basename;
+    use File::Copy;
+    use File::Path qw(mkpath);
+    our $rootimg_dir;
+    our @filestoadd;
+    sub xdie { die @_ }
+}
+
+## no critic (BuiltinFunctions::ProhibitStringyEval)
+eval "package Scratch;\n$copy_initrd_file\n1" or die $@;
+if (defined $helper) {
+    eval "package Scratch;\n$helper\n1" or die $@;
+}
+
+$Scratch::rootimg_dir = $rootimg;
+@Scratch::filestoadd  = (
+    [ 'lib/modules/7.0.0/kernel/drivers/net/virtio_net.ko', 'lib/virtio_net.ko' ],
+    [ 'lib/modules/7.0.0/kernel/drivers/net/mlx5_core.ko',  'lib/mlx5_core.ko' ],
+    [ 'lib/modules/7.0.0/kernel/drivers/net/bnx2x.ko.zst',  'lib/bnx2x.ko' ],
+    [ 'lib/modules/7.0.0/kernel/drivers/net/e1000e.ko',     'lib/e1000e.ko' ],
+    [ 'bin/busybox',                                        'bin/busybox' ],
+);
+eval "package Scratch;\nno strict 'vars';\n$step\n1" or die $@;
+
+ok(-e "$initrd_dir/lib/firmware/mellanox/fw-a.mfa2",
+    'the initrd keeps the firmware a driver in it asks for');
+ok(-e "$initrd_dir/lib/firmware/bnx2x/bnx2x-e2.fw.zst",
+    'a compressed firmware file answers the plain name the driver asks for');
+ok(!-e "$initrd_dir/lib/firmware/amdgpu/never-asked-for.bin",
+    'the initrd does not carry firmware no driver in it asks for');
+ok(!-e "$initrd_dir/lib/firmware/qcom/never-asked-for-either.bin",
+    'the whole firmware tree does not reach the initrd');
+ok(!-e "$initrd_dir/lib/firmware/intel/absent-from-this-image.bin",
+    'a firmware name the root image does not have is left out');
+
+done_testing();

From 3fcc781fdd88c05da4572220e01f8703f24c7d79 Mon Sep 17 00:00:00 2001
From: Daniel Hilst <392820+dhilst@users.noreply.github.com>
Date: Mon, 14 Sep 2026 07:54:23 -0300
Subject: [PATCH 02/10] fix(xcat-core): grub2 cannot load the Ubuntu 26.04
 ppc64el netboot initrd

The ubuntu-26.04-ppc64el diskless node never starts its kernel. SLOF reports
W3411 and E3406 and falls through to disk. The node fetches the kernel and the
initrd in under a minute and fails six minutes later, with no network activity in
between: grub2 has the payload and cannot start it.

genimage copies the whole lib/firmware tree of the root image into the initrd.
That tree is 666 MB on Ubuntu 26.04, which takes the initrd to 719 MB. A 687 MB
initrd boots the same kernel on the same node; a 719 MB one does not.

The firmware copy now takes only the firmware that the drivers in the initrd ask
for, which modinfo reports for each module. The root image keeps its whole tree,
so the node that boots is unchanged.

ubuntu_genimage_initrd_firmware.t drives the firmware step over a root image whose
firmware tree holds files no driver asks for. It fails on the previous commit.

Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com>
---
 .../share/xcat/netboot/ubuntu/genimage        | 56 ++++++++++++++++++-
 1 file changed, 54 insertions(+), 2 deletions(-)

diff --git a/xCAT-server/share/xcat/netboot/ubuntu/genimage b/xCAT-server/share/xcat/netboot/ubuntu/genimage
index 17005cdb5..9fe0803b4 100755
--- a/xCAT-server/share/xcat/netboot/ubuntu/genimage
+++ b/xCAT-server/share/xcat/netboot/ubuntu/genimage
@@ -1827,8 +1827,11 @@ EOMS
         }
     }
 
-    if (-d "$rootimg_dir/lib/firmware/") {
-        system("cp -r $rootimg_dir/lib/firmware/* /tmp/xcatinitrd.$$/lib/firmware");
+    # The whole firmware tree is 666 MB on Ubuntu 26.04, and grub2 on a pseries node cannot
+    # load an initrd of that size. The initrd only has to reach the network.
+    foreach my $firmware (initrd_firmware_files($rootimg_dir, \@filestoadd)) {
+        copy_initrd_file("$rootimg_dir/lib/firmware/$firmware",
+            "/tmp/xcatinitrd.$$/lib/firmware/$firmware");
     }
 
     if (-d "$rootimg_dir/lib/modules/$kernelver/") {
@@ -1892,6 +1895,55 @@ sub isnetdriver {
     }
 }
 
+#-------------------------------------------------------------------------------
+
+=head3 initrd_firmware_files
+
+    Descriptions: Select the firmware that the drivers in the netboot initrd ask for.
+    Arguments:
+        rootimg_dir - the root image directory
+        filestoadd  - reference to the list of files that go into the initrd. A module
+                      is either a plain path or a [ source, destination ] pair.
+    Returns: the firmware files that exist, as paths below lib/firmware
+
+=cut
+
+#-------------------------------------------------------------------------------
+sub initrd_firmware_files {
+    my ($rootimg_dir, $filestoadd) = @_;
+
+    return () unless (-d "$rootimg_dir/lib/firmware");
+
+    my %wanted;
+    foreach my $entry (@$filestoadd) {
+        my $module = ref($entry) ? $entry->[0] : $entry;
+        next unless ($module =~ /\.ko(?:\.(?:gz|xz|zst))?$/);
+        next unless (-f "$rootimg_dir/$module");
+
+        my $out = `modinfo -F firmware "$rootimg_dir/$module" 2>/dev/null`;
+        if ($?) {
+            print "Warning: cannot read the firmware of $module\n";
+            next;
+        }
+        foreach my $name (split(/\n/, $out)) {
+            $name =~ s/^\s+|\s+$//g;
+            $wanted{$name} = 1 if (length $name);
+        }
+    }
+
+    my @files;
+    foreach my $name (sort keys %wanted) {
+
+        # Ubuntu keeps the firmware compressed and the kernel asks for the plain name.
+        foreach my $candidate ($name, "$name.zst", "$name.xz", "$name.gz") {
+            next unless (-e "$rootimg_dir/lib/firmware/$candidate");
+            push @files, $candidate;
+            last;
+        }
+    }
+    return @files;
+}
+
 sub find_rootimg_file {
     my ($file) = @_;
 

From 3c104421a28fc354a00b90ce27619487dc60960c Mon Sep 17 00:00:00 2001
From: Daniel Hilst <392820+dhilst@users.noreply.github.com>
Date: Tue, 15 Sep 2026 21:11:18 -0300
Subject: [PATCH 03/10] fix(xcat-core): ubuntu_genimage_initrd_firmware.t
 passes when the file it reads is missing

ubuntu_genimage_initrd_firmware.t called plan skip_all when xCAT-server/share/xcat/netboot/ubuntu/genimage was absent, so a checkout that lost
the file reported 0 tests and exit 0. A test that cannot fail measures nothing.

Die instead, which is what makentp_ntp_deps.t already does for setupntp.

With xCAT-server/share/xcat/netboot/ubuntu/genimage moved aside the file now exits 2 and prints "genimage not found at <path>";
before this change it exited 0 and printed "1..0 # SKIP genimage not found at <path>". With the file
present the test passes either way.

Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com>
---
 xCAT-test/unit/ubuntu_genimage_initrd_firmware.t | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/xCAT-test/unit/ubuntu_genimage_initrd_firmware.t b/xCAT-test/unit/ubuntu_genimage_initrd_firmware.t
index b51bc6ba1..8408545b9 100755
--- a/xCAT-test/unit/ubuntu_genimage_initrd_firmware.t
+++ b/xCAT-test/unit/ubuntu_genimage_initrd_firmware.t
@@ -15,7 +15,7 @@ use Test::More;
 my $repo_root = File::Spec->rel2abs(File::Spec->catdir($FindBin::Bin, '..', '..'));
 my $genimage  = File::Spec->catfile(
     $repo_root, 'xCAT-server', 'share', 'xcat', 'netboot', 'ubuntu', 'genimage');
-plan skip_all => "genimage not found at $genimage" unless -f $genimage;
+die "genimage not found at $genimage\n" unless -f $genimage;
 
 my $src = do { local $/; open my $fh, '<', $genimage or die $!; <$fh> };
 

From 2523a06b1650658a5f0506ab6e679e4913eaef5f Mon Sep 17 00:00:00 2001
From: Daniel Hilst <392820+dhilst@users.noreply.github.com>
Date: Thu, 17 Sep 2026 07:37:56 -0300
Subject: [PATCH 04/10] test(xcat-core): the initrd firmware step ignores
 custom drivers and firmware overrides

ubuntu_genimage_initrd_firmware.t drove the firmware step over a root image that
holds every module and every firmware file. It covered neither of the two places
genimage reads from beside the root image.

The test now puts a driver in the custom directory, with its firmware in the
root image, and a firmware override under lib/firmware/updates/<kernel>. Both
are red: the step asks modinfo about the module under the root image, where a
custom driver is not, and it looks for a firmware name under lib/firmware only,
where an override is not.

The five assertions that were there stay green.

Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com>
---
 .../unit/ubuntu_genimage_initrd_firmware.t    | 21 +++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/xCAT-test/unit/ubuntu_genimage_initrd_firmware.t b/xCAT-test/unit/ubuntu_genimage_initrd_firmware.t
index 8408545b9..0d162dd68 100755
--- a/xCAT-test/unit/ubuntu_genimage_initrd_firmware.t
+++ b/xCAT-test/unit/ubuntu_genimage_initrd_firmware.t
@@ -54,6 +54,7 @@ case "$*" in
     *mlx5_core*)  echo mellanox/fw-a.mfa2 ;;
     *bnx2x*)      echo bnx2x/bnx2x-e2.fw ;;
     *e1000e*)     echo intel/absent-from-this-image.bin ;;
+    *custom_nic*) echo custom/custom-nic.bin ;;
     *virtio_net*) : ;;
     *)            exit 1 ;;
 esac
@@ -64,6 +65,8 @@ $ENV{PATH} = "$bin:$ENV{PATH}";
 
 foreach my $file (
     'lib/firmware/mellanox/fw-a.mfa2',
+    'lib/firmware/updates/7.0.0/mellanox/fw-a.mfa2',
+    'lib/firmware/custom/custom-nic.bin',
     'lib/firmware/bnx2x/bnx2x-e2.fw.zst',
     'lib/firmware/amdgpu/never-asked-for.bin',
     'lib/firmware/qcom/never-asked-for-either.bin',
@@ -82,6 +85,16 @@ foreach my $file (
 }
 make_path("$initrd_dir/lib/firmware");
 
+# A driver the administrator put in the custom directory. genimage takes the module from there
+# and not from the root image, so that is the file its firmware has to be read from.
+my $customdir   = "$scratch/custom";
+my $pathtofiles = "$scratch/pathtofiles";
+make_path("$customdir/lib/modules/7.0.0/kernel/drivers/net");
+make_path($pathtofiles);
+open(my $custom, '>', "$customdir/lib/modules/7.0.0/kernel/drivers/net/custom_nic.ko") or die $!;
+print $custom "content of a custom driver\n";
+close($custom);
+
 {
     package Scratch;
     use strict;
@@ -101,11 +114,15 @@ if (defined $helper) {
 }
 
 $Scratch::rootimg_dir = $rootimg;
+$Scratch::customdir   = $customdir;
+$Scratch::pathtofiles = $pathtofiles;
+$Scratch::kernelver   = '7.0.0';
 @Scratch::filestoadd  = (
     [ 'lib/modules/7.0.0/kernel/drivers/net/virtio_net.ko', 'lib/virtio_net.ko' ],
     [ 'lib/modules/7.0.0/kernel/drivers/net/mlx5_core.ko',  'lib/mlx5_core.ko' ],
     [ 'lib/modules/7.0.0/kernel/drivers/net/bnx2x.ko.zst',  'lib/bnx2x.ko' ],
     [ 'lib/modules/7.0.0/kernel/drivers/net/e1000e.ko',     'lib/e1000e.ko' ],
+    [ 'lib/modules/7.0.0/kernel/drivers/net/custom_nic.ko', 'lib/custom_nic.ko' ],
     [ 'bin/busybox',                                        'bin/busybox' ],
 );
 eval "package Scratch;\nno strict 'vars';\n$step\n1" or die $@;
@@ -120,5 +137,9 @@ ok(!-e "$initrd_dir/lib/firmware/qcom/never-asked-for-either.bin",
     'the whole firmware tree does not reach the initrd');
 ok(!-e "$initrd_dir/lib/firmware/intel/absent-from-this-image.bin",
     'a firmware name the root image does not have is left out');
+ok(-e "$initrd_dir/lib/firmware/custom/custom-nic.bin",
+    'the firmware of a driver taken from the custom directory reaches the initrd');
+ok(-e "$initrd_dir/lib/firmware/updates/7.0.0/mellanox/fw-a.mfa2",
+    'a firmware override under updates/<kernel> reaches the initrd');
 
 done_testing();

From 5554b79c2ded72025f6b07cfe7bbaf4ec6e928b6 Mon Sep 17 00:00:00 2001
From: Daniel Hilst <392820+dhilst@users.noreply.github.com>
Date: Thu, 17 Sep 2026 07:38:47 -0300
Subject: [PATCH 05/10] fix(xcat-core): the initrd firmware step reads the root
 image only

The step that fills lib/firmware in the Ubuntu netboot initrd asked modinfo
about $rootimg_dir/$module and looked a firmware name up under lib/firmware.
The copy step beside it takes a module from $customdir or $pathtofiles first,
and the kernel looks a firmware name up under updates/<kernel>, updates/,
<kernel>/ and lib/firmware. So a custom driver reached the initrd with no
firmware, even when the root image carried it, and a firmware override was left
out of the initrd altogether.

initrd_firmware_files now takes the module directories the copy step searches
and the kernel release. It resolves each module in that order before asking
modinfo, and keeps every firmware file that exists in the four directories the
kernel searches, so the override still wins on the node.

ubuntu_genimage_initrd_firmware.t covers both: its two new cases are red on the
commit before this one.

Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com>
---
 .../share/xcat/netboot/ubuntu/genimage        | 44 +++++++++++++++----
 .../unit/ubuntu_genimage_initrd_firmware.t    |  3 ++
 2 files changed, 38 insertions(+), 9 deletions(-)

diff --git a/xCAT-server/share/xcat/netboot/ubuntu/genimage b/xCAT-server/share/xcat/netboot/ubuntu/genimage
index 9fe0803b4..970a97d59 100755
--- a/xCAT-server/share/xcat/netboot/ubuntu/genimage
+++ b/xCAT-server/share/xcat/netboot/ubuntu/genimage
@@ -1829,7 +1829,9 @@ EOMS
 
     # The whole firmware tree is 666 MB on Ubuntu 26.04, and grub2 on a pseries node cannot
     # load an initrd of that size. The initrd only has to reach the network.
-    foreach my $firmware (initrd_firmware_files($rootimg_dir, \@filestoadd)) {
+    foreach my $firmware (
+        initrd_firmware_files($rootimg_dir, \@filestoadd, $kernelver, $customdir, $pathtofiles))
+    {
         copy_initrd_file("$rootimg_dir/lib/firmware/$firmware",
             "/tmp/xcatinitrd.$$/lib/firmware/$firmware");
     }
@@ -1904,13 +1906,16 @@ sub isnetdriver {
         rootimg_dir - the root image directory
         filestoadd  - reference to the list of files that go into the initrd. A module
                       is either a plain path or a [ source, destination ] pair.
+        kernelver   - the kernel release, which names one of the firmware directories
+        module_dirs - directories searched for a module before the root image, in the
+                      same order the copy step searches them
     Returns: the firmware files that exist, as paths below lib/firmware
 
 =cut
 
 #-------------------------------------------------------------------------------
 sub initrd_firmware_files {
-    my ($rootimg_dir, $filestoadd) = @_;
+    my ($rootimg_dir, $filestoadd, $kernelver, @module_dirs) = @_;
 
     return () unless (-d "$rootimg_dir/lib/firmware");
 
@@ -1918,9 +1923,18 @@ sub initrd_firmware_files {
     foreach my $entry (@$filestoadd) {
         my $module = ref($entry) ? $entry->[0] : $entry;
         next unless ($module =~ /\.ko(?:\.(?:gz|xz|zst))?$/);
-        next unless (-f "$rootimg_dir/$module");
 
-        my $out = `modinfo -F firmware "$rootimg_dir/$module" 2>/dev/null`;
+        # Ask about the module the copy step takes, which is the custom one where there is
+        # one. A custom driver is not in the root image at all.
+        my $source;
+        foreach my $dir (grep { defined && length } @module_dirs, $rootimg_dir) {
+            next unless (-f "$dir/$module");
+            $source = "$dir/$module";
+            last;
+        }
+        next unless (defined $source);
+
+        my $out = `modinfo -F firmware "$source" 2>/dev/null`;
         if ($?) {
             print "Warning: cannot read the firmware of $module\n";
             next;
@@ -1931,14 +1945,26 @@ sub initrd_firmware_files {
         }
     }
 
+    # The kernel looks for a firmware name in these directories below lib/firmware, in this
+    # order, and loads the first file it finds. Keep every one that exists, so an override
+    # still wins on the node.
+    my @dirs = ("updates", "");
+    if (defined $kernelver and length $kernelver) {
+        unshift @dirs, "updates/$kernelver";
+        splice(@dirs, 2, 0, $kernelver);
+    }
+
     my @files;
     foreach my $name (sort keys %wanted) {
+        foreach my $dir (@dirs) {
+            my $prefix = length($dir) ? "$dir/" : "";
 
-        # Ubuntu keeps the firmware compressed and the kernel asks for the plain name.
-        foreach my $candidate ($name, "$name.zst", "$name.xz", "$name.gz") {
-            next unless (-e "$rootimg_dir/lib/firmware/$candidate");
-            push @files, $candidate;
-            last;
+            # Ubuntu keeps the firmware compressed and the kernel asks for the plain name.
+            foreach my $candidate ($name, "$name.zst", "$name.xz", "$name.gz") {
+                next unless (-e "$rootimg_dir/lib/firmware/$prefix$candidate");
+                push @files, "$prefix$candidate";
+                last;
+            }
         }
     }
     return @files;
diff --git a/xCAT-test/unit/ubuntu_genimage_initrd_firmware.t b/xCAT-test/unit/ubuntu_genimage_initrd_firmware.t
index 0d162dd68..d01242385 100755
--- a/xCAT-test/unit/ubuntu_genimage_initrd_firmware.t
+++ b/xCAT-test/unit/ubuntu_genimage_initrd_firmware.t
@@ -103,6 +103,9 @@ close($custom);
     use File::Copy;
     use File::Path qw(mkpath);
     our $rootimg_dir;
+    our $customdir;
+    our $pathtofiles;
+    our $kernelver;
     our @filestoadd;
     sub xdie { die @_ }
 }

From 199de7f4f899056fabd451a98a6f69b40119162c Mon Sep 17 00:00:00 2001
From: Gary Skouson <gskouson@yahoo.com>
Date: Fri, 18 Sep 2026 08:37:38 -0400
Subject: [PATCH 06/10] grant CREATE for xcatadm on pgsql15+

---
 xCAT-client/bin/pgsqlsetup | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/xCAT-client/bin/pgsqlsetup b/xCAT-client/bin/pgsqlsetup
index f13ca21dc..9a7c962a1 100755
--- a/xCAT-client/bin/pgsqlsetup
+++ b/xCAT-client/bin/pgsqlsetup
@@ -1095,6 +1095,23 @@ sub setupxcatdb
 
     }
     $pgsql->soft_close();
+    # PostgreSQL 15 removed the default CREATE privilege on schema public
+    # for PUBLIC, so xcatadm must be granted it explicitly before xcatd
+    # can create the xCAT tables. No-op on earlier releases.
+    my $grantsql = "GRANT USAGE, CREATE ON SCHEMA public TO xcatadm;";
+    my $grantcmd;
+    if ($::osname eq 'AIX')
+    {
+        $grantcmd = "$::installdir/bin/psql -d $::dbname -U postgres -v ON_ERROR_STOP=1 -c \"$grantsql\"";
+    } else {
+        $grantcmd = "$::pgcmddir/psql -d $::dbname -U postgres -v ON_ERROR_STOP=1 -c \"$grantsql\"";
+    }
+    $rc = &runpgcmd_chkoutput($grantcmd);
+    if ($rc > 0) {
+        xCAT::MsgUtils->message("E",
+            "Failed granting xcatadm privileges on schema public in $::dbname");
+        exit(1);
+    }
 
     # setup root user
     $pgsql = new Expect;

From 0e6196524f9aae0147f945e3b55d3ec3c1b2d0b6 Mon Sep 17 00:00:00 2001
From: Daniel Hilst <392820+dhilst@users.noreply.github.com>
Date: Wed, 23 Sep 2026 10:39:13 -0300
Subject: [PATCH 07/10] fix(xcat-core): repository indexing fails on every
 target on a shared build tree

createrepo_dir passed --database, which writes *.sqlite.bz2. Building those
needs SQLite, and SQLite needs POSIX locks. A build tree can live on an NFS
re-export, where the kernel refuses locks outright: every attempt answers
errno 524.

So every target died with "Cannot open .repodata/primary.sqlite: Can not create
db_info table: disk I/O error", and the build staged nothing.

Measured on such a share, with a local control: a bare sqlite3 connect fails
there and succeeds on local disk; createrepo_c fails with --database and
succeeds without it, emitting primary/filelists/other as *.xml.zst.

Nothing this project ships reads the sqlite metadata. dnf on el8+ and zypper
both read the XML.

Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com>
---
 buildrpms.pl | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/buildrpms.pl b/buildrpms.pl
index e14bdd436..df22aaea2 100755
--- a/buildrpms.pl
+++ b/buildrpms.pl
@@ -657,14 +657,20 @@ sub setup_local_repos {
 }
 
 
-# Index one repo dir with deterministic, upstream-matching metadata. createrepo_c's
-# defaults already emit primary/filelists/other as *.xml.zst plus *.sqlite.bz2
-# (--database), exactly the upstream shape; --set-timestamp-to-revision pins the
-# repomd timestamp to SOURCE_DATE_EPOCH.
+# Index one repo dir with deterministic, upstream-matching metadata: primary/filelists/other as
+# *.xml.zst, with --set-timestamp-to-revision pinning the repomd timestamp to SOURCE_DATE_EPOCH.
+#
+# NO --database. It writes *.sqlite.bz2, and building those needs SQLite, which needs POSIX
+# locks. A build tree can live on an NFS re-export, where the kernel refuses locks outright:
+# every attempt answers errno 524, and createrepo_c then dies on every target with
+# "Cannot open .repodata/primary.sqlite: Can not create db_info table: disk I/O error".
+# Without --database it succeeds there.
+#
+# Nothing this project ships reads the sqlite metadata. dnf on el8+ and zypper both read the XML.
 sub createrepo_dir {
     my ($dir, $extra) = @_;
     $extra //= '';
-    sh_or_die(qq(createrepo_c --update --database )
+    sh_or_die(qq(createrepo_c --update )
        . qq(--revision "$SOURCE_DATE_EPOCH" --set-timestamp-to-revision $extra "$dir"),
         "Failed to createrepo_c $dir\n");
 }

From d1e1c1e3a62955ea8b702b318e6b99291e8539af Mon Sep 17 00:00:00 2001
From: Daniel Hilst <392820+dhilst@users.noreply.github.com>
Date: Sat, 19 Sep 2026 00:00:21 -0300
Subject: [PATCH 08/10] fix(xcat-core): Version still names the released 2.19.0

2.19.0 is released, and master still builds packages that call themselves
2.19.0. Every snapshot built from master since then carries the released
version, so a candidate cannot be told from the release it follows.

Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com>
---
 Version | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Version b/Version
index ef0f38abe..7329e21c3 100644
--- a/Version
+++ b/Version
@@ -1 +1 @@
-2.19.0
+2.20.0

From 771da0e09e3ee26a7790058d3393df9f78a15ab9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?=
 <2031761+viniciusferrao@users.noreply.github.com>
Date: Wed, 23 Sep 2026 13:39:05 -0300
Subject: [PATCH 09/10] ci(genesis): run the OpenEmbedded check only when its
 inputs change

The OpenEmbedded metadata job ran on every pull request and took about
20 minutes. It reads only xCAT-genesis-builder/oe and
xCAT-genesis-scripts, and it pins its upstream sources to fixed commits.
A pull request that changes neither directory gets the same result each
time.

The job moves unchanged to its own workflow, which runs only when those
paths or the workflow file change. xcat_pr_test stays in xcat_test.yml
and runs on every pull request, because a required check that does not
run blocks the merge.
---
 .github/workflows/genesis_openembedded.yml | 54 ++++++++++++++++++++++
 .github/workflows/xcat_test.yml            | 46 ------------------
 2 files changed, 54 insertions(+), 46 deletions(-)
 create mode 100644 .github/workflows/genesis_openembedded.yml

diff --git a/.github/workflows/genesis_openembedded.yml b/.github/workflows/genesis_openembedded.yml
new file mode 100644
index 000000000..b2ee14521
--- /dev/null
+++ b/.github/workflows/genesis_openembedded.yml
@@ -0,0 +1,54 @@
+name: genesis_openembedded
+on:
+  pull_request:
+    paths:
+      - 'xCAT-genesis-builder/oe/**'
+      - 'xCAT-genesis-scripts/**'
+      - '.github/workflows/genesis_openembedded.yml'
+  workflow_dispatch:
+jobs:
+  genesis_openembedded:
+    name: OpenEmbedded metadata
+    runs-on: ubuntu-24.04
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v6
+      - uses: actions/setup-python@v6
+        with:
+          python-version: '3.12'
+          cache: pip
+          cache-dependency-path: xCAT-genesis-builder/oe/requirements.txt
+      - name: Install OpenEmbedded host dependencies
+        run: >-
+          sudo env DEBIAN_FRONTEND=noninteractive apt-get install -y
+          --no-install-recommends --no-install-suggests
+          build-essential chrpath cpio diffstat file gawk git locales
+          lz4 openssl socat texinfo unzip wget xz-utils zstd
+      - name: Install KAS
+        run: python -m pip install -r xCAT-genesis-builder/oe/requirements.txt
+      - name: Enable BitBake user namespaces
+        run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
+      - name: Validate every architecture configuration
+        shell: bash
+        env:
+          KAS_WORK_DIR: ${{ runner.temp }}/kas-work
+        run: |
+          set -euo pipefail
+          for architecture in x86 x86_64 ppc64 ppc64le armv7hf aarch64 riscv64 s390x; do
+              export KAS_BUILD_DIR="${RUNNER_TEMP}/kas-build-${architecture}"
+              configuration="xCAT-genesis-builder/oe/kas/${architecture}.yml"
+              kas dump "${configuration}" >/dev/null
+              kas shell "${configuration}" -c 'bitbake -p'
+          done
+      - name: Validate the image task graph
+        shell: bash
+        env:
+          KAS_WORK_DIR: ${{ runner.temp }}/kas-work
+          KAS_BUILD_DIR: ${{ runner.temp }}/kas-build-x86_64
+        run: |
+          set -euo pipefail
+          kas shell xCAT-genesis-builder/oe/kas/x86_64.yml -c \
+              'bitbake -n xcat-genesis-image xcat-genesis-extension-smoke'
+          export KAS_BUILD_DIR="${RUNNER_TEMP}/kas-build-s390x"
+          kas shell xCAT-genesis-builder/oe/kas/s390x.yml -c \
+              'bitbake -n xcat-genesis-image xcat-genesis-extension-smoke'
diff --git a/.github/workflows/xcat_test.yml b/.github/workflows/xcat_test.yml
index 04d50aff0..d23185138 100644
--- a/.github/workflows/xcat_test.yml
+++ b/.github/workflows/xcat_test.yml
@@ -10,49 +10,3 @@ jobs:
         run: sudo env DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends --no-install-suggests bats build-essential fakeroot reprepro devscripts debhelper libcapture-tiny-perl libfile-slurper-perl libjson-perl libparallel-forkmanager-perl libsoap-lite-perl libdbi-perl libcgi-pm-perl quilt openssh-server dpkg looptools genometools software-properties-common
       - name: Run tests
         run: perl github_action_xcat_test.pl
-
-  genesis_openembedded:
-    name: OpenEmbedded metadata
-    runs-on: ubuntu-24.04
-    timeout-minutes: 30
-    steps:
-      - uses: actions/checkout@v6
-      - uses: actions/setup-python@v6
-        with:
-          python-version: '3.12'
-          cache: pip
-          cache-dependency-path: xCAT-genesis-builder/oe/requirements.txt
-      - name: Install OpenEmbedded host dependencies
-        run: >-
-          sudo env DEBIAN_FRONTEND=noninteractive apt-get install -y
-          --no-install-recommends --no-install-suggests
-          build-essential chrpath cpio diffstat file gawk git locales
-          lz4 openssl socat texinfo unzip wget xz-utils zstd
-      - name: Install KAS
-        run: python -m pip install -r xCAT-genesis-builder/oe/requirements.txt
-      - name: Enable BitBake user namespaces
-        run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
-      - name: Validate every architecture configuration
-        shell: bash
-        env:
-          KAS_WORK_DIR: ${{ runner.temp }}/kas-work
-        run: |
-          set -euo pipefail
-          for architecture in x86 x86_64 ppc64 ppc64le armv7hf aarch64 riscv64 s390x; do
-              export KAS_BUILD_DIR="${RUNNER_TEMP}/kas-build-${architecture}"
-              configuration="xCAT-genesis-builder/oe/kas/${architecture}.yml"
-              kas dump "${configuration}" >/dev/null
-              kas shell "${configuration}" -c 'bitbake -p'
-          done
-      - name: Validate the image task graph
-        shell: bash
-        env:
-          KAS_WORK_DIR: ${{ runner.temp }}/kas-work
-          KAS_BUILD_DIR: ${{ runner.temp }}/kas-build-x86_64
-        run: |
-          set -euo pipefail
-          kas shell xCAT-genesis-builder/oe/kas/x86_64.yml -c \
-              'bitbake -n xcat-genesis-image xcat-genesis-extension-smoke'
-          export KAS_BUILD_DIR="${RUNNER_TEMP}/kas-build-s390x"
-          kas shell xCAT-genesis-builder/oe/kas/s390x.yml -c \
-              'bitbake -n xcat-genesis-image xcat-genesis-extension-smoke'

From e654de5c4a6bc826fb4f09095b43d4abe3dc8211 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?=
 <2031761+viniciusferrao@users.noreply.github.com>
Date: Wed, 23 Sep 2026 18:00:38 -0300
Subject: [PATCH 10/10] docs(developers): describe branches, backports and the
 release checklist

The project had no written release process, and the 2.18 and 2.19
releases missed steps: the release table in the documentation, the
docs version, signed tags, and the wiki and website index pages.

A new Releases section describes the branch and version model, the
label and milestone that each pull request needs, and a checklist for
release candidates, publishing, the signed tag, the GitHub release, the
release notes, the website and the announcement. The build hosts, the
signing key and the publish procedure stay in the private repository of
the maintainers.
---
 docs/source/developers/index.rst              |   1 +
 docs/source/developers/releases/checklist.rst | 321 ++++++++++++++++++
 docs/source/developers/releases/index.rst     |  68 ++++
 .../developers/releases/pull_requests.rst     |  64 ++++
 4 files changed, 454 insertions(+)
 create mode 100644 docs/source/developers/releases/checklist.rst
 create mode 100644 docs/source/developers/releases/index.rst
 create mode 100644 docs/source/developers/releases/pull_requests.rst

diff --git a/docs/source/developers/index.rst b/docs/source/developers/index.rst
index a56f0e63a..5bca03e07 100644
--- a/docs/source/developers/index.rst
+++ b/docs/source/developers/index.rst
@@ -10,3 +10,4 @@ This page is for developers interested in working with xCAT.
    license/index.rst
    github/index.rst
    guides/index.rst
+   releases/index.rst
diff --git a/docs/source/developers/releases/checklist.rst b/docs/source/developers/releases/checklist.rst
new file mode 100644
index 000000000..db884ad2d
--- /dev/null
+++ b/docs/source/developers/releases/checklist.rst
@@ -0,0 +1,321 @@
+Release Checklist
+=================
+
+A maintainer with publish access to the download server does these steps. The build hosts, the
+signing key and the publish procedure are in the private repository of the maintainers. This page
+does not repeat them.
+
+The steps use these names:
+
+* ``X.Y.Z``: the new release, for example ``2.19.1``.
+* ``P``: the tag of the previous release in the same series, for example ``2.19.0``. For X.Y.0,
+  use the tag of the previous minor release.
+* ``<commit>``: the full id of the release commit.
+* ``<page>``: the name of the release notes page on the wiki.
+
+Plan
+----
+
+#. Open the milestone of the release: ``X.Y.Z`` for a patch release, ``X.Y`` for X.Y.0. Make sure
+   that each closed pull request in it is merged on the ``X.Y`` branch. For a backported fix, the
+   ``[Backport X.Y]`` pull request must also be merged.
+
+#. Move each open item to the next milestone, or finish it before the release candidates start.
+
+Start a Minor Release
+---------------------
+
+Do these steps once for X.Y.0, when the release candidates start. You must have admin rights on
+``xcat2/xcat-core``.
+
+#. Create the ``X.Y`` branch from the head of ``master``: ::
+
+    $ gh api -X POST repos/xcat2/xcat-core/git/refs -f ref=refs/heads/X.Y -f sha=<commit>
+
+#. Add ``refs/heads/X.Y`` to the target branches of the ``release-branches`` ruleset, in
+   Settings, Rules, Rulesets.
+
+#. Create the backport label: ::
+
+    $ gh label create "backport X.Y" -R xcat2/xcat-core --color 0e8a16 \
+        --description "Backport this PR to the X.Y maintenance branch"
+
+#. Create the milestone ``X.Y.1`` and the milestone of the next minor release, if they do not
+   exist.
+
+#. Open a pull request against ``master`` that sets ``Version`` and the ``release`` value in
+   ``docs/source/conf.py`` to the next minor release, for example ``2.21.0`` after the ``2.20``
+   branch is created.
+
+After these steps, a fix for X.Y.0 goes to ``master`` with the label ``backport X.Y`` and the
+milestone ``X.Y``.
+
+When a series gets no more releases, delete its backport label.
+
+Prepare the Release Branch
+--------------------------
+
+#. Add a row to ``docs/source/overview/_files/X.Y.x.csv``. For X.Y.0, create the file and add a
+   section for it at the top of ``docs/source/overview/xcat2_release.rst``. A row has this form: ::
+
+    2.18.0,2026-06-22,"RHEL 10,AlmaLinux 10",`2.18.0 Release Notes <https://github.com/xcat2/xcat-core/wiki/XCAT_2.18_Release_Notes>`_
+
+   Open the change against ``master`` with the label ``backport X.Y``, so that the row is on both
+   branches before the tag.
+
+#. Make sure that ``Version`` and the ``release`` value in ``docs/source/conf.py`` on the ``X.Y``
+   branch are both ``X.Y.Z``.
+
+Release Candidates
+------------------
+
+A release candidate is a build of the head of the ``X.Y`` branch. Do not create a branch or a tag
+for a candidate, such as ``release/X.Y-rc1`` or ``X.Y.Z-rc1``.
+
+#. Build a candidate from the head of the ``X.Y`` branch, and sign the packages and the
+   repository metadata. Build xcat-dep too if its packages changed after the previous release.
+
+#. Publish the candidate to a staging location only. Do not publish a candidate to ``X.Y`` or to
+   ``latest``.
+
+#. Test the candidate on each operating system and architecture that the release notes will name.
+   Install a management node, and provision stateful and stateless compute nodes. Record what you
+   tested, because the release notes report it.
+
+#. If a test fails, fix the problem on ``master``, backport the fix, and build the next candidate
+   from the new head of ``X.Y``.
+
+The ``COMMIT_ID_LONG`` line in the ``buildinfo.txt`` file of a candidate identifies it.
+
+Build and Publish
+-----------------
+
+The release is the last candidate that passed the tests.
+
+#. Record the full id of the release commit. This is the ``COMMIT_ID_LONG`` of that candidate.
+
+#. Publish the packages of that candidate to ``repos/yum/X.Y/`` and ``repos/apt/X.Y/``. Do not
+   build them again. A new build from the same commit can give different packages, for example a
+   Genesis image with a newer kernel. If you must build a package again, test the new build as a
+   candidate before you publish it.
+
+#. Upload the offline bundles: ::
+
+    xcat-core/X.Y.x_Linux/xcat-core-X.Y.Z-linux.tar.bz2
+    xcat-core/X.Y.x_Ubuntu/xcat-core-X.Y.Z-ubuntu.tar.bz2
+    xcat-dep/2.x_Linux/xcat-dep-X.Y.Z-linux.tar.bz2
+    xcat-dep/2.x_Ubuntu/xcat-dep-X.Y.Z-ubuntu.tar.bz2
+
+#. Make sure that the ``.repo`` files in the published repositories point at the published
+   location, not at ``devel``.
+
+#. If X.Y is the newest series, point ``latest`` at ``X.Y`` for yum and for apt.
+
+#. Make sure that ``devel`` does not serve a build that is older than the release.
+
+Verify the Published Packages
+-----------------------------
+
+#. Check that ``buildinfo.txt`` shows ``VERSION=X.Y.Z`` and ``COMMIT_ID_LONG=<commit>``. If you
+   changed ``latest``, check it too: ::
+
+    $ curl -fsS https://xcat.org/files/xcat/repos/yum/X.Y/xcat-core/buildinfo.txt
+    $ curl -fsS https://xcat.org/files/xcat/repos/yum/latest/xcat-core/buildinfo.txt
+
+#. Check the signatures of the repository metadata against the key that ``xCAT-release`` ships.
+   Use a keyring that holds only ``xCAT-release/RPM-GPG-KEY-xCAT``, so that no other key can pass
+   the check: ::
+
+    $ gpg --dearmor <xCAT-release/RPM-GPG-KEY-xCAT >xcat-release.gpg
+    $ curl -fsSO https://xcat.org/files/xcat/repos/yum/X.Y/xcat-core/repodata/repomd.xml
+    $ curl -fsSO https://xcat.org/files/xcat/repos/yum/X.Y/xcat-core/repodata/repomd.xml.asc
+    $ gpgv --keyring ./xcat-release.gpg repomd.xml.asc repomd.xml
+
+   Do the same for ``xcat-dep/common`` and for each ``xcat-dep/rh<N>/<arch>`` directory. For apt,
+   check ``dists/<codename>/InRelease`` of xcat-core and xcat-dep for each codename: ::
+
+    $ curl -fsSO https://xcat.org/files/xcat/repos/apt/X.Y/xcat-core/dists/<codename>/InRelease
+    $ gpgv --keyring ./xcat-release.gpg InRelease
+
+#. Check that each offline bundle URL answers with HTTP 200: ::
+
+    $ curl -fsSI https://xcat.org/files/xcat/xcat-core/X.Y.x_Linux/xcat-core-X.Y.Z-linux.tar.bz2
+
+#. Install xCAT with signature checks on a new EL host and on a new Ubuntu host. ``go-xcat``
+   turns these checks off with ``dnf --nogpgcheck`` and ``apt-get --allow-unauthenticated``, so a
+   ``go-xcat`` installation does not prove the signatures.
+
+   On EL, use the three published ``.repo`` files, which set ``gpgcheck=1``. Enable the other
+   repositories that the installation guide requires first. The published files point at
+   ``latest``. For an older series, replace ``latest`` with ``X.Y`` in the three files. Install the
+   Genesis image for the host architecture by name, because ``xCAT`` only recommends it. ::
+
+    $ curl -fsSo /etc/yum.repos.d/xcat-core.repo \
+        https://xcat.org/files/xcat/repos/yum/X.Y/xcat-core/xcat-core.repo
+    $ curl -fsSo /etc/yum.repos.d/xcat-dep.repo \
+        https://xcat.org/files/xcat/repos/yum/X.Y/xcat-dep/rh<N>/<arch>/xcat-dep.repo
+    $ curl -fsSo /etc/yum.repos.d/xcat-dep-common.repo \
+        https://xcat.org/files/xcat/repos/yum/X.Y/xcat-dep/common/xcat-dep-common.repo
+    $ dnf install -y xCAT xCAT-genesis-openembedded-<arch>
+    $ lsxcatd -v
+
+   On Ubuntu, add the repositories with their signing keys. ``apt-get update`` must not report a
+   signature error. ::
+
+    $ url=https://xcat.org/files/xcat/repos/apt/X.Y
+    $ codename=$(. /etc/os-release && echo "$VERSION_CODENAME")
+    $ curl -fsSL $url/xcat-core/apt.key | gpg --dearmor -o /usr/share/keyrings/xcat-core.gpg
+    $ curl -fsSL $url/xcat-dep/apt.key | gpg --dearmor -o /usr/share/keyrings/xcat-dep.gpg
+    $ echo "deb [signed-by=/usr/share/keyrings/xcat-core.gpg] $url/xcat-core $codename main" \
+        >/etc/apt/sources.list.d/xcat.list
+    $ echo "deb [signed-by=/usr/share/keyrings/xcat-dep.gpg] $url/xcat-dep $codename main" \
+        >>/etc/apt/sources.list.d/xcat.list
+    $ apt-get update
+    $ apt-get install -y xcat
+    $ lsxcatd -v
+
+#. Install xCAT with ``go-xcat`` on another new EL host and on a new Ubuntu host, and check the
+   version: ::
+
+    $ ./go-xcat -x X.Y -y install
+    $ lsxcatd -v
+
+   If you changed ``latest``, also install once without ``-x X.Y``.
+
+Release Notes
+-------------
+
+The release notes are a page on the `xcat-core wiki <https://github.com/xcat2/xcat-core/wiki>`_.
+
+#. Name the page ``XCAT_X.Y_Release_Notes`` for X.Y.0, and ``XCAT_X.Y.Z_Release_Notes`` for a
+   patch release. Do not rename the page later, because other pages and https://xcat.org link to
+   it.
+
+#. Start from the notes of the previous release of the same kind, and keep the order of the
+   sections: ::
+
+    # xCAT X.Y.Z Release Notes (Month D, YYYY)
+    ## Operating System Support
+    ## Highlighted Changes
+    ## Download xCAT
+    ### Offline tarball bundles
+    ## Validation
+    ## Key Issues Resolved
+    ## Documentation
+
+#. Run each command in "Download xCAT" on a new host, and open each link.
+
+#. Report only the tests that were done, with the operating system, the architecture and the
+   node types.
+
+#. For a patch release, add a line at the top of the X.Y notes page that links to the new page.
+
+Tag
+---
+
+Create a signed annotated tag on the release commit, and push only the tag: ::
+
+    $ git fetch upstream
+    $ git tag -s X.Y.Z -m "xCAT X.Y.Z" <commit>
+    $ git tag -v X.Y.Z
+    $ git push upstream refs/tags/X.Y.Z
+
+Do not create the tag from the GitHub release page. That page creates a lightweight tag with no
+signature.
+
+GitHub Release
+--------------
+
+Write the release text to a file, for example ``notes.md``: ::
+
+    The release notes for X.Y.Z are available at https://github.com/xcat2/xcat-core/wiki/<page>
+
+    Full changelog: https://github.com/xcat2/xcat-core/compare/P...X.Y.Z
+
+Then create the release from the tag: ::
+
+    $ gh release create X.Y.Z -R xcat2/xcat-core --verify-tag --title X.Y.Z --notes-file notes.md
+
+For a patch release of a series that is not the newest, add ``--latest=false``.
+
+Read the Docs
+-------------
+
+Read the Docs builds a documentation version for each new tag, and ``stable`` follows the highest
+version tag.
+
+#. Open https://xcat-docs.readthedocs.io/en/X.Y.Z/, and check that the build passed and that the
+   title shows X.Y.Z.
+
+#. If X.Y.Z is the highest release, check that https://xcat-docs.readthedocs.io/en/stable/ shows
+   X.Y.Z. For a patch release of an older series, check that ``stable`` did not change.
+
+Wiki Index Pages
+----------------
+
+#. On the ``test_sidebar`` page, which holds the News list, add a line at the top: ::
+
+    * Mon DD, YYYY: [xCAT X.Y.Z](<page>) released.
+
+#. For X.Y.0, add a row for X.Y to the "General Release Information and Planning" table on the
+   ``Home`` page, and move the ``(stable)`` marker to it.
+
+Website
+-------
+
+The ``xcat2/xcat2.github.io`` repository holds the pages of https://xcat.org. The default
+downloads on these pages must stay on the same series as ``latest``.
+
+If X.Y is the newest series:
+
+#. In ``index.html``, change the release line and the release notes link.
+
+#. In ``download.html``, change the version and the offline bundle links.
+
+#. In ``footer.html``, change the release news link.
+
+For a patch release of an older series, do not change the default version on these pages. Add the
+older release to ``download.html`` as a separate entry.
+
+Commit the change in the repository, copy the changed files to the web server, and open each
+changed link on https://xcat.org.
+
+Announcement
+------------
+
+Send an email to xcat-user@lists.sourceforge.net with the subject
+``Announcement: xCAT X.Y.Z released``. Use the announcement of the previous release as the model: ::
+
+    Dear xCAT community,
+
+    We are pleased to announce the release of xCAT X.Y.Z.
+    <one or two sentences about the release>
+
+    Highlights:
+    * <area>
+      - <change>
+
+    Full release notes: https://github.com/xcat2/xcat-core/wiki/<page>
+    Downloads: https://github.com/xcat2/xcat-core/releases/tag/X.Y.Z
+
+    <thanks to contributors and reporters>
+
+    We welcome your feedback, bug reports, and contributions:
+    https://github.com/xcat2/xcat-core
+
+    Best regards,
+    <name>
+    on behalf of the xCAT Consortium
+
+Keep the announcement of a patch release short, and name the fixes it contains.
+
+After the Release
+-----------------
+
+#. Close the milestone of the release. Create the milestone ``X.Y.(Z+1)`` if it does not exist,
+   and move the open items to it.
+
+#. Open a pull request against ``X.Y`` that sets ``Version`` and the ``release`` value in
+   ``docs/source/conf.py`` to ``X.Y.(Z+1)``.
+
+#. For X.Y.0, make sure that the steps in "Start a Minor Release" are complete.
diff --git a/docs/source/developers/releases/index.rst b/docs/source/developers/releases/index.rst
new file mode 100644
index 000000000..56f082c9b
--- /dev/null
+++ b/docs/source/developers/releases/index.rst
@@ -0,0 +1,68 @@
+Releases
+========
+
+This section describes how xCAT versions are numbered, how a fix reaches a maintained release,
+and how a maintainer publishes a release.
+
+.. toctree::
+   :maxdepth: 2
+
+   pull_requests.rst
+   checklist.rst
+
+Branches and Versions
+---------------------
+
+xCAT has one development branch and one maintenance branch for each minor release.
+
+* ``master`` is the development line of the next minor release.
+* ``X.Y``, for example ``2.19``, is the maintenance branch of the X.Y series. It receives only
+  fixes that are backported from ``master``. The patch releases X.Y.1, X.Y.2 and later are tagged
+  on it.
+
+The ``Version`` file at the top of the tree is the only place that holds the version.
+``buildrpms.pl`` and ``builddebs.pl`` read it, and each build adds the release string
+``snapYYYYMMDDHHMM`` from the commit time. A release is the build of the tagged commit. There is no
+separate release build.
+
+.. list-table::
+   :header-rows: 1
+
+   * - Branch
+     - ``Version`` holds
+     - Example after 2.19.0 is released
+   * - ``master``
+     - the next minor release
+     - ``2.20.0``
+   * - ``X.Y``
+     - the next release of the series
+     - ``2.19.1``
+
+The ``release`` value in ``docs/source/conf.py`` must be the same as ``Version`` on the same
+branch, because Read the Docs shows it in the title of each page. Change the two files in the same
+pull request.
+
+A released version number is never used again. The :doc:`checklist` changes ``Version`` so that
+no build after a release carries the number of that release.
+
+Tags
+----
+
+Each release has a signed annotated tag named ``X.Y.Z``, for example ``2.19.1``. The tag points at
+the commit that the published packages were built from. Release candidates do not get tags,
+because Read the Docs builds a documentation version for each new tag.
+
+Package Channels
+----------------
+
+The download server has three channels under https://xcat.org/files/xcat/repos/yum/ and
+https://xcat.org/files/xcat/repos/apt/:
+
+* ``devel``: development snapshots from ``master``, in ``devel/core-snap`` and ``devel/xcat-dep``.
+* ``X.Y``: the most recent release of the X.Y series. A patch release replaces the contents of
+  its series directory.
+* ``latest``: the most recent release series. ``go-xcat`` installs from ``latest`` by default, so
+  when ``latest`` changes, new installations get the new release.
+
+The offline bundles are in https://xcat.org/files/xcat/xcat-core/ and
+https://xcat.org/files/xcat/xcat-dep/.
diff --git a/docs/source/developers/releases/pull_requests.rst b/docs/source/developers/releases/pull_requests.rst
new file mode 100644
index 000000000..51b3758ea
--- /dev/null
+++ b/docs/source/developers/releases/pull_requests.rst
@@ -0,0 +1,64 @@
+Pull Requests and Backports
+===========================
+
+Open every change against ``master``. A fix reaches a maintenance branch as a backport of the
+merged ``master`` pull request.
+
+Labels and Milestones
+---------------------
+
+Set a label and a milestone on each pull request against ``master``. The milestone is the first
+release that contains the change.
+
+.. list-table::
+   :header-rows: 1
+   :widths: 50 20 30
+
+   * - Change
+     - Label
+     - Milestone
+   * - A fix that users of the maintained series need: a regression, a failure on a supported
+       platform, a security fix, or a change that the maintenance branch needs to build or test
+     - ``backport X.Y``
+     - the next release of the series, for example ``2.19.1``
+   * - Any other change: a feature, a refactor, a change to tests only, documentation, or a fix
+       for an old or high-risk problem
+     - none
+     - the next minor release, for example ``2.20``
+
+Do not add a backport label to a change of the ``Version`` file.
+
+When a patch release ships, the next patch milestone replaces it. After 2.19.1, the milestone for
+backported fixes is ``2.19.2``.
+
+Automatic Backports
+-------------------
+
+The ``backport`` workflow in ``.github/workflows/backport.yml`` starts when a pull request with a
+``backport X.Y`` label is merged. It also starts when the label is added to a pull request that is
+already merged. The workflow:
+
+#. cherry-picks the commits of the pull request onto the ``X.Y`` branch,
+#. opens a pull request named ``[Backport X.Y] <original title>``,
+#. approves that pull request and turns on auto-merge.
+
+The backport pull request merges when ``xcat_pr_test`` passes. Do not set a milestone on it. The
+milestone stays on the original pull request.
+
+Manual Backports
+----------------
+
+If the cherry-pick has a conflict, the workflow fails and comments on the original pull request.
+Then do the backport by hand from your fork: ::
+
+    $ git fetch upstream
+    $ git switch -c backport-<number>-to-X.Y upstream/X.Y
+    $ git cherry-pick -x <commit> ...
+    $ git push origin backport-<number>-to-X.Y
+
+Resolve each conflict before you continue the cherry-pick. Open a pull request against ``X.Y``
+named ``[Backport X.Y] <original title>``, and set the patch release milestone on it.
+
+The ``release-branches`` ruleset protects each maintenance branch in the same way as ``master``.
+A change needs a pull request, an approval and a passing ``xcat_pr_test``, and the branch refuses
+direct pushes.