Merge pull request #7833 from VersatusHPC/fix/remove-xcat-genesis-builder

fix(xcat-core): Remove legacy xCAT-genesis-builder

.gitattributes

@@ -1,2 +1,2 @@
-xCAT-genesis-builder/oe/meta-xcat-genesis/recipes-kernel/linux/linux-yocto/0001-s390-use-stable-generator-name.patch whitespace=-space-before-tab
+xCAT-genesis-base/oe/meta-xcat-genesis/recipes-kernel/linux/linux-yocto/0001-s390-use-stable-generator-name.patch whitespace=-space-before-tab
 xCAT-test/fixtures/openeuler-media/*/.treeinfo whitespace=-blank-at-eol,-blank-at-eof

.github/workflows/genesis_openembedded.yml

@@ -2,7 +2,7 @@ name: genesis_openembedded
 on:
   pull_request:
     paths:
-      - 'xCAT-genesis-builder/oe/**'
+      - 'xCAT-genesis-base/oe/**'
       - 'xCAT-genesis-scripts/**'
       - '.github/workflows/genesis_openembedded.yml'
   workflow_dispatch:
@@ -17,7 +17,7 @@ jobs:
         with:
           python-version: '3.12'
           cache: pip
-          cache-dependency-path: xCAT-genesis-builder/oe/requirements.txt
+          cache-dependency-path: xCAT-genesis-base/oe/requirements.txt
       - name: Install OpenEmbedded host dependencies
         run: >-
           sudo env DEBIAN_FRONTEND=noninteractive apt-get install -y
@@ -25,7 +25,7 @@ jobs:
           build-essential chrpath cpio diffstat file gawk git locales
           lz4 openssl socat texinfo unzip wget xz-utils zstd
       - name: Install KAS
-        run: python -m pip install -r xCAT-genesis-builder/oe/requirements.txt
+        run: python -m pip install -r xCAT-genesis-base/oe/requirements.txt
       - name: Enable BitBake user namespaces
         run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
       - name: Validate every architecture configuration
@@ -36,7 +36,7 @@ jobs:
           set -euo pipefail
           for architecture in x86 x86_64 ppc64 ppc64le armv7hf aarch64 riscv64 s390x; do
               export KAS_BUILD_DIR="${RUNNER_TEMP}/kas-build-${architecture}"
-              configuration="xCAT-genesis-builder/oe/kas/${architecture}.yml"
+              configuration="xCAT-genesis-base/oe/kas/${architecture}.yml"
               kas dump "${configuration}" >/dev/null
               kas shell "${configuration}" -c 'bitbake -p'
           done
@@ -47,8 +47,8 @@ jobs:
           KAS_BUILD_DIR: ${{ runner.temp }}/kas-build-x86_64
         run: |
           set -euo pipefail
-          kas shell xCAT-genesis-builder/oe/kas/x86_64.yml -c \
+          kas shell xCAT-genesis-base/oe/kas/x86_64.yml -c \
               'bitbake -n xcat-genesis-image xcat-genesis-extension-smoke'
           export KAS_BUILD_DIR="${RUNNER_TEMP}/kas-build-s390x"
-          kas shell xCAT-genesis-builder/oe/kas/s390x.yml -c \
+          kas shell xCAT-genesis-base/oe/kas/s390x.yml -c \
               'bitbake -n xcat-genesis-image xcat-genesis-extension-smoke'

build-utils/lib/XCAT/BuildUtils.pm

@@ -24,7 +24,7 @@ use feature 'say';
 
 our @EXPORT_OK = qw(
     source_date_epoch snap_release deb_version
-    stage_probe_helpers XCAT_PROBE_HELPERS
+    stage_probe_helpers XCAT_PROBE_HELPERS stage_genesis_base_sources
     deb_package_arches dist_arches default_dists
     orig_tarball_name upstream_version resolve_dest
     pin_control_version rewrite_changelog_header
@@ -380,6 +380,56 @@ sub stage_probe_helpers {
     return @staged;
 }
 
+#-------------------------------------------------------------------------------
+
+=head3 stage_genesis_base_sources
+
+    Descriptions:
+        Write the build support tarball that xCAT-genesis-base.spec unpacks:
+        the dracut_105 modules, 80-net-name-slot.rules, and
+        verify-genesis-payload with its XCAT::GenesisPayload module, taken from
+        the xCAT-genesis-base directory of a checkout.
+    Arguments:
+        $checkout: the top of the xcat-core checkout
+        $tarball:  the path of the tarball to write
+        $epoch:    SOURCE_DATE_EPOCH, the mtime of every member
+    Returns:
+        $tarball. Dies when the checkout has no xCAT-genesis-base directory,
+        so a rename stops the build instead of producing an empty tarball.
+
+=cut
+
+#-------------------------------------------------------------------------------
+sub stage_genesis_base_sources {
+    my ($checkout, $tarball, $epoch) = @_;
+    my $source = "$checkout/xCAT-genesis-base";
+    die "Assertion failed! No directory xCAT-genesis-base in $checkout\n"
+        unless -d $source;
+
+    my $staging_parent = "/tmp/xcat-genesis-base-build-support.$$";
+    my $staging_root   = "$staging_parent/xCAT-genesis-base-build-support";
+    remove_tree($staging_parent) if -e $staging_parent;
+    make_path("$staging_root/dracut_105");
+
+    sh_or_die(qq(cp -a "$source/dracut_105" "$staging_root/"),
+        "Error copying dracut_105 sources");
+    copy("$source/80-net-name-slot.rules", "$staging_root/80-net-name-slot.rules")
+        or die "Unable to stage $source/80-net-name-slot.rules: $!\n";
+    # The spec runs the verifier against the extracted payload in %install.
+    copy("$source/verify-genesis-payload", "$staging_root/verify-genesis-payload")
+        or die "Unable to stage $source/verify-genesis-payload: $!\n";
+    make_path("$staging_root/lib/XCAT");
+    copy("$source/lib/XCAT/GenesisPayload.pm", "$staging_root/lib/XCAT/GenesisPayload.pm")
+        or die "Unable to stage $source/lib/XCAT/GenesisPayload.pm: $!\n";
+
+    unlink $tarball if -f $tarball;
+    sh_or_die(qq(tar --sort=name --owner=0 --group=0 --mtime="\@$epoch" -cjf "$tarball" -C "$staging_parent" xCAT-genesis-base-build-support),
+        "Error creating $tarball");
+
+    remove_tree($staging_parent);
+    return $tarball;
+}
+
 # deb_package_arches: the architectures to build a package for.
 # 'all' is a single arch-independent build; the three arch packages get one per arch.
 sub deb_package_arches {

builddebs.pl

@@ -366,11 +366,11 @@ sub build_one_genesis_deb {
         # pipeline builds from.
         my $stage = "$root$GENESIS_STAGE";
         sh_or_die("rm -rf " . sh_quote($stage) . " && mkdir -p "
-                . sh_quote("$stage/xCAT-genesis-builder"),
+                . sh_quote("$stage/xCAT-genesis-base"),
             "FATAL: cannot make the build directory in session:$id\n");
-        sh_or_die("cp -a " . sh_quote("$ROOT/xCAT-genesis-builder") . "/. "
-                . sh_quote("$stage/xCAT-genesis-builder") . "/",
-            "FATAL: cannot copy xCAT-genesis-builder into session:$id\n");
+        sh_or_die("cp -a " . sh_quote("$ROOT/xCAT-genesis-base") . "/. "
+                . sh_quote("$stage/xCAT-genesis-base") . "/",
+            "FATAL: cannot copy xCAT-genesis-base into session:$id\n");
         copy("$ROOT/Version", "$stage/Version")
             or die "FATAL: cannot copy Version into session:$id: $!\n";
         write_text("$stage/Release", "$RELEASE\n");
@@ -379,7 +379,7 @@ sub build_one_genesis_deb {
         # builder stops when the root it woke up in is not the release it was asked for.
         my $cmd = join ' ',
             'schroot', '--run-session', '-c', sh_quote("session:$id"), '-u', 'root', '-d', '/',
-            '--', '/bin/bash', "$GENESIS_STAGE/xCAT-genesis-builder/builddeb-genesis-base",
+            '--', '/bin/bash', "$GENESIS_STAGE/xCAT-genesis-base/builddeb-genesis-base",
             '--expect-codename', sh_quote($codename), '--outdir', "$GENESIS_STAGE/out";
         my $rc = sh("$cmd > " . sh_quote($logfile) . " 2>&1");
 

buildrpms.pl

@@ -55,6 +55,7 @@ use File::Temp qw(tempdir tempfile);
 use FindBin qw($Bin);
 use lib "$Bin/build-utils/lib";
 use XCAT::BuildUtils qw(git_revision source_date_epoch sh sh_or_die usage buildinfo_text
+                        stage_genesis_base_sources
                         write_script read_line targetarch_from_target
                         openeuler_build_target openeuler_repo_subdir);
 use POSIX ();                   # EEXIST, for the directory build lock (see main())
@@ -347,33 +348,8 @@ sub createmockconfig {
 }
 
 sub buildsources_genesis_base($) {
-    my ($target) = @_;
-
-    die "Assertion failed! No directory xCAT-genesis-builder in the current directory"
-        unless -d "./xCAT-genesis-builder";
-    my $staging_parent = "/tmp/xcat-genesis-base-build-support.$$";
-    my $staging_root = "$staging_parent/xCAT-genesis-base-build-support";
-    my $support_tarball = "$SOURCES/xCAT-genesis-base-build-support.tar.bz2";
-
-    remove_tree($staging_parent) if -e $staging_parent;
-    make_path("$staging_root/dracut_105");
-
-    sh_or_die(qq(cp -a "xCAT-genesis-builder/dracut_105" "$staging_root/"),
-        "Error copying dracut_105 sources");
-    cp "xCAT-genesis-builder/80-net-name-slot.rules",
-       "$staging_root/80-net-name-slot.rules";
-    # %install runs this against the extracted payload before it becomes an rpm.
-    cp "xCAT-genesis-builder/verify-genesis-payload",
-       "$staging_root/verify-genesis-payload";
-    make_path("$staging_root/lib/XCAT");
-    cp "xCAT-genesis-builder/lib/XCAT/GenesisPayload.pm",
-       "$staging_root/lib/XCAT/GenesisPayload.pm";
-
-    unlink $support_tarball if -f $support_tarball;
-    sh_or_die(qq(tar --sort=name --owner=0 --group=0 --mtime="\@$SOURCE_DATE_EPOCH" -cjf "$support_tarball" -C "$staging_parent" xCAT-genesis-base-build-support),
-        "Error creating $support_tarball");
-
-    remove_tree($staging_parent);
+    stage_genesis_base_sources(".", "$SOURCES/xCAT-genesis-base-build-support.tar.bz2",
+        $SOURCE_DATE_EPOCH);
 }
 
 sub prepare_xcat_probe_source_tar {
@@ -495,11 +471,8 @@ sub buildspkgs {
       : "dist/$target/rpms/SRPMS/$pkg-$VERSION-$RELEASE.src.rpm";
     return if -f $diskcache and not $opts{force};
 
-    my $dir = sub {
-        return "xCAT-genesis-builder"
-            if $pkg eq "xCAT-genesis-base";
-        $pkg;
-    }->();
+    # The source directory is the package name now that xCAT-genesis-builder is gone.
+    my $dir = $pkg;
 
     my @opts;
     push @opts, "--quiet" unless $opts{verbose};

docs/source/references/coral/index.rst

@@ -8,7 +8,6 @@ CORAL stands for Collaboration of Oak Ridge, Argonne, and Livermore and is solut
    :maxdepth: 2
 
    cluster_mgmt/index.rst
-   known_issues/index.rst
    ha/index.rst
 ..   mn/index.rst
 ..   discovery/index.rst

docs/source/references/coral/known_issues/genesis_base.rst

@@ -1,24 +0,0 @@
-xCAT Genesis Base
-=================
-
-xCAT ships a ``xCAT-genesis-base`` package as part of xcat-deps.  This is a light-weight diskless linux image based currently on Fedora28, that is used by xCAT to do hardware discovery.
-
-Follow the steps below to build your own version of the ``xCAT-genesis-base`` on-site.  You can include additional drivers or modules or you can build your own version of the ``xCAT-genesis-base`` on-site using a server running OS other than Fedora28, like Red Hat Enterprise Linux 8. Building ``xCAT-genesis-base`` on a server running Red Hat Enterprise Linux 7 or earlier, is no longer supported.
-
-1. Download the latest timestamp version of the ``xCAT-genesis-builder`` RPM provided here: http://xcat.org/files/xcat/xcat-dep/2.x_Linux/beta/
-
-2. Install the ``xCAT-genesis-builder`` RPM on a node installed with desired OS (currently verified with Fedora28 and Red Hat Enterprise Linux 8). For more details: https://github.com/xcat2/xcat-core/tree/master/xCAT-genesis-builder#readme 
-
-3. If additional drivers or modules need to be loaded when genesis kernel boots, edit ``xCAT-genesis-builder/xcat-cmdline.sh`` and add ``modprobe`` statements for each one.
-
-4. Build the new ``xCAT-genesis-base`` RPM: ::
-
-    /opt/xcat/share/xcat/netboot/genesis/builder/buildrpm
-
-To use the generated RPM from the step above for node discovery:
-
-1. Uninstall the old ``xCAT-genesis-base`` RPM and install the newly built ``xCAT-genesis-base`` RPM on the xCAT Management node 
-2. Execute: ``mknb ppc64``
-
-3. Follow :doc:`discover nodes </guides/admin-guides/manage_clusters/ppc64le/discovery/mtms/discovery>`
-

docs/source/references/coral/known_issues/index.rst

@@ -1,7 +0,0 @@
-Known Issues
-============
-
-.. toctree::
-   :maxdepth: 2
-
-   genesis_base.rst

makerpm-genesisbuilder

@@ -1,25 +0,0 @@
-#!/bin/sh
-VER=`cat Version`
-VER=${VER/-/.post}
-VER=${VER/-/.}
-rpmbuild --version > /dev/null
-if [ $? -gt 0 ]
-then
-	echo "Error: Is rpmbuild installed and working?"
-	exit 1
-fi
-RPMROOT=`rpmbuild --eval '%_topdir' xCATsn/xCATsn.spec`
-if [ $? -gt 0 ]
-then
-	echo "Could not determine rpmbuild's root"
-	exit 1
-fi
-echo "The location for rpm building is ${RPMROOT}"
-cd `dirname $0`/xCAT-genesis-builder
-tar --exclude .svn -cjvf $RPMROOT/SOURCES/xCAT-genesis-builder.tar.bz2 .
-#mv -f *.gz $RPMROOT/SOURCES
-cp LICENSE.html $RPMROOT/BUILD
-cp xCAT-genesis-builder.spec $RPMROOT/SOURCES
-cd -
-rm -f $RPMROOT/SRPMS/xCAT-genesis-builder-$VER*rpm $RPMROOT/RPMS/*/xCAT-genesis-builder-$VER*rpm
-rpmbuild -ba xCAT-genesis-builder/xCAT-genesis-builder.spec --define "version $VER" --define "release 1"

xCAT-genesis-base/README.md

@@ -0,0 +1,107 @@
+# xCAT Genesis
+
+## OpenEmbedded builder
+
+OpenEmbedded is the current Genesis build path. It builds a small, pinned
+system instead of copying files from the build host.
+
+Install the host packages required by OpenEmbedded, then install `kas` in a
+Python virtual environment:
+
+```sh
+python3 -m venv .venv
+. .venv/bin/activate
+pip install -r xCAT-genesis-base/oe/requirements.txt
+xCAT-genesis-base/oe/build x86_64
+```
+
+The build command accepts `x86`, `x86_64`, `ppc64`, `ppc64le`, `armv7hf`,
+`aarch64`, `riscv64`, and `s390x`. Multiple architectures are built in the
+order given.
+Run `xCAT-genesis-base/oe/build --list-architectures` to print this list.
+Artifacts are written below `xCAT-genesis-base/oe/.work/build/tmp/deploy/images`.
+The `x86` artifact uses an i686 CPU baseline. The build carries the reviewed
+Yocto release key in `oe/keys` and verifies its fingerprint locally.
+
+The serial console normally opens a read-only Newt status screen. `F1` shows
+help, `F2` shows diagnostics, and `F3` shows live logs. `F12` opens a confirmed
+root maintenance shell; exiting returns to the status screen. Add
+`xcat.console=plain` to the kernel command line for line output. In plain mode,
+type `shell` and press Enter to open the same confirmed maintenance shell.
+
+Export an image for xCAT after the build:
+
+```sh
+xCAT-genesis-base/oe/export x86_64 \
+    xCAT-genesis-base/oe/.work/build/tmp/deploy \
+    /tmp/xcat-genesis-x86_64
+```
+
+Copy the exported directory intact to the management node at
+`/opt/xcat/share/xcat/netboot/genesis/x86_64`, then publish it:
+
+```sh
+mknb x86_64
+```
+
+`mknb` verifies the kernel and initramfs checksums before replacing the files
+under the configured TFTP root.
+
+### Signed extensions
+
+Build an extension recipe with the same machine configuration as the Genesis
+image. Then export it with the site's Ed25519 release key:
+
+```sh
+xCAT-genesis-base/oe/export-extension x86_64 my-extension \
+    xCAT-genesis-base/oe/.work/build/tmp/deploy \
+    /secure/genesis-extension.key /secure/genesis-extension.pub \
+    /tmp/my-extension-bundle
+```
+
+The private key stays outside the source tree. The exported bundle contains
+the extension image, manifest, signature, public key, and checksums.
+
+A site layer can include that directory in its Genesis image with a small
+append file:
+
+```bitbake
+FILESEXTRAPATHS:prepend := "${THISDIR}/files:"
+SRC_URI += "file://my-extension-bundle"
+XCAT_GENESIS_EXTENSION_BUNDLE = "my-extension-bundle"
+```
+
+Place the exported directory below the append file's `files` directory.
+Genesis verifies every bundled extension before registration and stops the
+boot workflow if the image, manifest, signature, key, release, or architecture
+does not match.
+
+Genesis records registration time and memory use in `/run/xcat/metrics.env`.
+After copying that file from a test VM, create a report with image sizes and
+runtime measurements:
+
+```sh
+xCAT-genesis-base/oe/report --runtime /tmp/metrics.env \
+    x86_64 /tmp/xcat-genesis-x86_64 > /tmp/report.env
+```
+
+Compare a later build with the saved report:
+
+```sh
+xCAT-genesis-base/oe/report --runtime /tmp/metrics.env \
+    --baseline /tmp/report.env \
+    x86_64 /tmp/xcat-genesis-x86_64
+```
+
+Positive deltas mean a larger image, more memory use, or a slower registration.
+
+See the [architecture plan](../docs/source/developers/guides/code/genesis_openembedded_plan.rst)
+for the network, hardware, license, and signed-extension contracts.
+
+# Open Source License
+
+xCAT is made available under the EPL license: https://opensource.org/licenses/eclipse-1.0.php
+
+# Developers
+
+Want to help? Check out the [developers guide](http://xcat-docs.readthedocs.io/en/latest/developers)!

xCAT-genesis-base/oe/kas/aarch64.yml

@@ -1,6 +1,6 @@
 header:
   version: 19
   includes:
-    - xCAT-genesis-builder/oe/kas/common.yml
+    - xCAT-genesis-base/oe/kas/common.yml
 
 machine: xcat-genesis-aarch64

xCAT-genesis-base/oe/kas/armv7hf.yml

@@ -1,6 +1,6 @@
 header:
   version: 19
   includes:
-    - xCAT-genesis-builder/oe/kas/common.yml
+    - xCAT-genesis-base/oe/kas/common.yml
 
 machine: xcat-genesis-armv7hf

xCAT-genesis-base/oe/kas/common.yml

@@ -5,7 +5,7 @@ signers:
   yocto-release:
     fingerprint: 2AFB13F28FBBB0D1B9DAF63087EB3D32FB631AD9
     repo: xcat-core
-    path: xCAT-genesis-builder/oe/keys/yocto-release.asc
+    path: xCAT-genesis-base/oe/keys/yocto-release.asc
 
 distro: xcat-genesis
 target: xcat-genesis-image
@@ -40,7 +40,7 @@ repos:
 
   xcat-core:
     layers:
-      xCAT-genesis-builder/oe/meta-xcat-genesis:
+      xCAT-genesis-base/oe/meta-xcat-genesis:
 
 local_conf_header:
   xcat-genesis: |

xCAT-genesis-base/oe/kas/ppc64.yml

@@ -1,6 +1,6 @@
 header:
   version: 19
   includes:
-    - xCAT-genesis-builder/oe/kas/common.yml
+    - xCAT-genesis-base/oe/kas/common.yml
 
 machine: xcat-genesis-ppc64

xCAT-genesis-base/oe/kas/ppc64le.yml

@@ -1,6 +1,6 @@
 header:
   version: 19
   includes:
-    - xCAT-genesis-builder/oe/kas/common.yml
+    - xCAT-genesis-base/oe/kas/common.yml
 
 machine: xcat-genesis-ppc64le

xCAT-genesis-base/oe/kas/riscv64.yml

@@ -1,6 +1,6 @@
 header:
   version: 19
   includes:
-    - xCAT-genesis-builder/oe/kas/common.yml
+    - xCAT-genesis-base/oe/kas/common.yml
 
 machine: xcat-genesis-riscv64

xCAT-genesis-base/oe/kas/s390x.yml

@@ -1,6 +1,6 @@
 header:
   version: 19
   includes:
-    - xCAT-genesis-builder/oe/kas/common.yml
+    - xCAT-genesis-base/oe/kas/common.yml
 
 machine: xcat-genesis-s390x

xCAT-genesis-base/oe/kas/x86.yml

@@ -1,6 +1,6 @@
 header:
   version: 19
   includes:
-    - xCAT-genesis-builder/oe/kas/common.yml
+    - xCAT-genesis-base/oe/kas/common.yml
 
 machine: xcat-genesis-x86

xCAT-genesis-base/oe/kas/x86_64.yml

@@ -1,6 +1,6 @@
 header:
   version: 19
   includes:
-    - xCAT-genesis-builder/oe/kas/common.yml
+    - xCAT-genesis-base/oe/kas/common.yml
 
 machine: xcat-genesis-x86-64