diff --git a/xCAT-test/dhcptest/README.md b/xCAT-test/dhcptest/README.md index fd04f2d4e..f42fec6ff 100644 --- a/xCAT-test/dhcptest/README.md +++ b/xCAT-test/dhcptest/README.md @@ -215,7 +215,7 @@ option 53 either, so `msgtype` reads `BOOTREPLY`. a different reason from the rest: one asserts the boot file a *known* machine is handed, the other the boot file an *unknown* one is handed, and a run that only has a node defined can use the first without the second. Both hold on any -xCAT-served network — see `spec.md` S-56, which requires the subnet to answer +xCAT-served network — see the spec, S-56, which requires the subnet to answer an unknown machine with a loader on either backend. Splitting on that boundary rather than using `-s` is deliberate: `--set` values diff --git a/xCAT-test/dhcptest/conf/bootp-client.conf b/xCAT-test/dhcptest/conf/bootp-client.conf index 94ee67a97..1c87995ee 100644 --- a/xCAT-test/dhcptest/conf/bootp-client.conf +++ b/xCAT-test/dhcptest/conf/bootp-client.conf @@ -10,7 +10,7 @@ # and the request is simply never answered. # # Scenario names carry the review number of the specification scenario they -# assert -- see xCAT-test/dhcptest/spec.md. +# assert -- see the DHCP wire spec. # # dhcptest run -i eth1 --set bootp_mac=02:00:de:ad:b0:07 \ # --set pool=10.0.0.200-10.0.0.250 conf/bootp-client.conf diff --git a/xCAT-test/dhcptest/conf/discovery-bootfile.conf b/xCAT-test/dhcptest/conf/discovery-bootfile.conf index 3c1e92c54..f454ec7c3 100644 --- a/xCAT-test/dhcptest/conf/discovery-bootfile.conf +++ b/xCAT-test/dhcptest/conf/discovery-bootfile.conf @@ -3,7 +3,7 @@ # A machine being discovered has no reservation, by definition, so the answer # can only come from the subnet -- and it has to come, or the machine has no # way to reach the state where someone could define it. Both backends are -# required to answer it (spec.md S-56), which is why this holds on any +# required to answer it (spec S-56), which is why this holds on any # xCAT-served network and not only on some of them. # # It is a file of its own rather than a third scenario in diff --git a/xCAT-test/dhcptest/conf/dynamic-range-cidr.conf b/xCAT-test/dhcptest/conf/dynamic-range-cidr.conf index 4e14e6a11..1d436a092 100644 --- a/xCAT-test/dhcptest/conf/dynamic-range-cidr.conf +++ b/xCAT-test/dhcptest/conf/dynamic-range-cidr.conf @@ -9,7 +9,7 @@ # whose range was rewritten in the other notation. # # Scenario names carry the review number of the specification scenario they -# assert -- see xCAT-test/dhcptest/spec.md. +# assert -- see the DHCP wire spec. # # dhcptest run -i eth1 --set unknown_mac=02:00:de:ad:be:ef \ # --set pool=10.0.0.192/26 conf/dynamic-range-cidr.conf diff --git a/xCAT-test/dhcptest/conf/http-port.conf b/xCAT-test/dhcptest/conf/http-port.conf index a496377c1..19ad6e0de 100644 --- a/xCAT-test/dhcptest/conf/http-port.conf +++ b/xCAT-test/dhcptest/conf/http-port.conf @@ -12,7 +12,7 @@ # configuration regenerated. # # Scenario names carry the review number of the specification scenario they -# assert -- see xCAT-test/dhcptest/spec.md. +# assert -- see the DHCP wire spec. # # dhcptest run -i eth1 --set tftp=10.0.0.1 --set httpport=8080 \ # --set riscv64_loader=boot/grub2/grub2.riscv64 \ diff --git a/xCAT-test/dhcptest/conf/ipxe-userclass.conf b/xCAT-test/dhcptest/conf/ipxe-userclass.conf index 722aa3104..29b9c29bb 100644 --- a/xCAT-test/dhcptest/conf/ipxe-userclass.conf +++ b/xCAT-test/dhcptest/conf/ipxe-userclass.conf @@ -14,7 +14,7 @@ # gets is in netboot-methods.conf. # # Scenario names carry the review number of the specification scenario they -# assert -- see xCAT-test/dhcptest/spec.md. +# assert -- see the DHCP wire spec. # # dhcptest run -i eth1 --set user_class=xNBA \ # --set stage1_loader=xcat/xnba.kpxe \ diff --git a/xCAT-test/dhcptest/conf/iscsi-root-path.conf b/xCAT-test/dhcptest/conf/iscsi-root-path.conf index 33ca67135..792210daa 100644 --- a/xCAT-test/dhcptest/conf/iscsi-root-path.conf +++ b/xCAT-test/dhcptest/conf/iscsi-root-path.conf @@ -6,7 +6,7 @@ # one a machine reads depends on whose initiator it ships with. # # Scenario names carry the review number of the specification scenario they -# assert -- see xCAT-test/dhcptest/spec.md. +# assert -- see the DHCP wire spec. # # dhcptest run -i eth1 \ # --set iscsi_mac=02:00:dc:11:00:51 --set iscsi_ip=10.0.0.51 \ diff --git a/xCAT-test/dhcptest/conf/loader-absent.conf b/xCAT-test/dhcptest/conf/loader-absent.conf index 1aa5bb57e..9a84252cb 100644 --- a/xCAT-test/dhcptest/conf/loader-absent.conf +++ b/xCAT-test/dhcptest/conf/loader-absent.conf @@ -11,7 +11,7 @@ # decide which boot classes to write by looking at what is on disk. # # Scenario names carry the review number of the specification scenario they -# assert -- see xCAT-test/dhcptest/spec.md. +# assert -- see the DHCP wire spec. # # dhcptest run -i eth1 --set pool=10.0.0.200-10.0.0.250 \ # --set present_loader=xcat/xnba.efi \ diff --git a/xCAT-test/dhcptest/conf/localboot.conf b/xCAT-test/dhcptest/conf/localboot.conf index 7cca781f4..1cbb1a7a0 100644 --- a/xCAT-test/dhcptest/conf/localboot.conf +++ b/xCAT-test/dhcptest/conf/localboot.conf @@ -15,7 +15,7 @@ # sent because they are answered by different rules and a test that sends only # the first cannot see the second break. # -# What is asserted is spec.md S-31: no boot file at all, on either request. +# What is asserted is spec S-31: no boot file at all, on either request. # Allowing the stage-1 binary through would be a weaker statement that looks # equivalent and is not -- a node handed the stage-1 loader comes back as an # xNBA second stage and is answered with the *network's* script, so a server diff --git a/xCAT-test/dhcptest/conf/multi-mac-node.conf b/xCAT-test/dhcptest/conf/multi-mac-node.conf index 2af19cf4c..74b521564 100644 --- a/xCAT-test/dhcptest/conf/multi-mac-node.conf +++ b/xCAT-test/dhcptest/conf/multi-mac-node.conf @@ -11,7 +11,7 @@ # two interfaces of the same machine on one address the moment both are up. # # Scenario names carry the review number of the specification scenario they -# assert -- see xCAT-test/dhcptest/spec.md. +# assert -- see the DHCP wire spec. # # dhcptest run -i eth1 \ # --set first_mac=02:00:dc:11:00:41 --set first_ip=10.0.0.41 \ diff --git a/xCAT-test/dhcptest/conf/netboot-methods.conf b/xCAT-test/dhcptest/conf/netboot-methods.conf index bc88bae97..0139e6c77 100644 --- a/xCAT-test/dhcptest/conf/netboot-methods.conf +++ b/xCAT-test/dhcptest/conf/netboot-methods.conf @@ -8,7 +8,7 @@ # and the machine does not install. # # Scenario names carry the review number of the specification scenario they -# assert -- see xCAT-test/dhcptest/spec.md. +# assert -- see the DHCP wire spec. # # Every MAC, filename and URL comes in on the command line, so this file says # nothing about how the server was configured: diff --git a/xCAT-test/dhcptest/conf/next-server-source.conf b/xCAT-test/dhcptest/conf/next-server-source.conf index d801fcf83..14c3d9620 100644 --- a/xCAT-test/dhcptest/conf/next-server-source.conf +++ b/xCAT-test/dhcptest/conf/next-server-source.conf @@ -12,7 +12,7 @@ # that ignored the node attribute entirely would pass every scenario here. # # Scenario names carry the review number of the specification scenario they -# assert -- see xCAT-test/dhcptest/spec.md. +# assert -- see the DHCP wire spec. # # dhcptest run -i eth1 \ # --set tftp_mac=02:00:dc:11:00:31 --set tftp_ip=10.0.0.31 \ diff --git a/xCAT-test/dhcptest/conf/node-removal.conf b/xCAT-test/dhcptest/conf/node-removal.conf index 56e5d7f2c..82df0507e 100644 --- a/xCAT-test/dhcptest/conf/node-removal.conf +++ b/xCAT-test/dhcptest/conf/node-removal.conf @@ -11,7 +11,7 @@ # reservation straight back, since the node is still defined in the database. # # Scenario names carry the review number of the specification scenario they -# assert -- see xCAT-test/dhcptest/spec.md. +# assert -- see the DHCP wire spec. # # dhcptest run -i eth1 -s removed-node-keeps-its-reservation-until-it-is-withdrawn \ # --set removed_mac=02:00:dc:11:00:61 --set removed_ip=10.0.0.61 \ diff --git a/xCAT-test/dhcptest/conf/provision-vs-discovery.conf b/xCAT-test/dhcptest/conf/provision-vs-discovery.conf index 7a86b790e..ee606804a 100644 --- a/xCAT-test/dhcptest/conf/provision-vs-discovery.conf +++ b/xCAT-test/dhcptest/conf/provision-vs-discovery.conf @@ -13,7 +13,7 @@ # What an *unknown* machine is told to boot is asserted in # discovery-bootfile.conf rather than here, so that this file needs only a # defined node and that one needs only a pool. Both are required behaviour -- -# see spec.md S-56 -- not alternatives to pick between. +# see spec S-56 -- not alternatives to pick between. # # Nothing here says who configured the server, or with what. Every address and # filename is supplied on the command line: diff --git a/xCAT-test/dhcptest/conf/pxe-arch-matrix.conf b/xCAT-test/dhcptest/conf/pxe-arch-matrix.conf index a2e57331b..dbe60740a 100644 --- a/xCAT-test/dhcptest/conf/pxe-arch-matrix.conf +++ b/xCAT-test/dhcptest/conf/pxe-arch-matrix.conf @@ -23,7 +23,7 @@ # be defined at all. # # Scenario names carry the review number of the specification scenario they -# assert -- see xCAT-test/dhcptest/spec.md. +# assert -- see the DHCP wire spec. [defaults] # 12, 51, 114 and 209 are in the request list because ISC dhcpd sends an option diff --git a/xCAT-test/dhcptest/spec.md b/xCAT-test/dhcptest/spec.md deleted file mode 100644 index a3301d487..000000000 --- a/xCAT-test/dhcptest/spec.md +++ /dev/null @@ -1,867 +0,0 @@ -# What xCAT's DHCP server is supposed to do - -A specification of the behaviour a machine booting on an xCAT-provisioned -network actually observes, written as scenarios so it can be checked rather -than argued about. - -Every scenario is stated from the point of view of the client on the wire: what -it sent, and what came back. That is deliberate. `dhcptest` never reads the -xCAT database and never runs an xCAT command, so a scenario phrased in terms of -tables and plugins cannot be tested by it. Where a behaviour is genuinely not -observable from the wire -- a file's contents, a daemon's command line -- it is -marked **[config]** and belongs to the Perl unit tests under `xCAT-test/unit/` -instead. - -Each scenario cites the source it was read from, so a scenario that stops -matching xCAT is a bug in one of the two and it is clear where to look. - -Every scenario carries a review number, `@S-nn`, in the Gherkin tag above it. -The numbers are stable: the appendix, the wire scenarios in `conf/` and the -issues raised against this document all cite them, so "S-27 fails on Kea" names -one behaviour rather than a paragraph nobody can find twice. - -**No scenario in this document is conditional on the DHCP backend.** The -backend is chosen for the operator by the management node's distribution, so a -Then clause that holds on ISC and not on Kea describes a machine that boots on -RHEL 9 and hangs on RHEL 10. Where the two implementations disagreed, one -answer was chosen; *Appendix A* records each choice, its reason, and which side -has to move. - -Terms used throughout: - -| Term | Meaning | -| --- | --- | -| **known node** | a node in the xCAT database with a MAC in the `mac` table | -| **unknown machine** | any other MAC: never defined, or defined without a MAC | -| **pool** | `networks.dynamicrange` for the subnet | -| **reservation** | the fixed address a known node's MAC is bound to | -| **loader** | the boot program named in the BOOTP `file` header or option 67 | -| **next-server** | the BOOTP `siaddr` header: where the loader is fetched from | -| **backend** | `site.dhcpbackend`: `isc`, `kea`, or `auto` | - ---- - -## Feature: An address for every machine on a provisioning network - -Source: `xCAT-server/lib/xcat/plugins/dhcp.pm` (`addnode`, `kea_reservations_for_node`), -`perl-xCAT/xCAT/DHCP/Range.pm` - -```gherkin -@S-01 -Scenario: A known node is given the address it was defined with - Given a node defined with a MAC and an IP outside the dynamic range - And makedhcp has been run for that node - When the node sends a DHCPDISCOVER from that MAC - Then it is offered exactly that IP - And a DHCPREQUEST for that IP is acknowledged with the same IP - -@S-02 -Scenario: The same node comes back to the same address - Given a node that has already been offered its reserved address - When it discovers again after a reboot - Then it is offered the same address again - # A node's name-to-address mapping is what the rest of the deployment is - # built on: /etc/hosts, DNS, the installer's kickstart URL. - -@S-03 -Scenario: An unknown machine is given an address out of the pool - Given a subnet with a dynamic range - When a MAC with no reservation sends a DHCPDISCOVER - Then it is offered an address inside the dynamic range - And that address is not any node's reserved address - -@S-04 -Scenario: An unknown machine is ignored where there is no pool - Given a subnet with no dynamic range - When a MAC with no reservation sends a DHCPDISCOVER - Then no reply arrives - # makedhcp warns "No dynamic range specified for . If hardware - # discovery is being used, a dynamic range is required." -- dhcp.pm:4494 - -@S-05 -Scenario: A dynamic range may be written as a pair or as a CIDR - Given networks.dynamicrange is "10.0.0.200-10.0.0.250" - Or networks.dynamicrange is "10.0.0.192/26" - When an unknown MAC discovers - Then the offered address falls inside the range either way - # Range.pm parses both; several ranges may be given separated by ";" - -@S-06 -Scenario: A node whose IP falls inside the dynamic range is refused - Given a node whose defined IP is inside networks.dynamicrange - When makedhcp runs for that node - Then no reservation is created for it # [config] - And makedhcp reports the collision - # dhcp.pm:168 -- "Node has which is inside the DHCP dynamic range." - # On the wire the node is indistinguishable from an unknown machine. - -@S-07 -Scenario: A node interface deliberately without an address is denied - Given a mac table entry whose hostname field is the sentinel *NOIP* - When that MAC sends a DHCPDISCOVER - Then it is not offered an address - And it is not told what to boot - # An operator marks an interface *NOIP* precisely so nothing boots on it. - # ISC does it with "deny booting;" on the host statement; whatever mechanism - # a backend uses, the MAC must not be answered -- a subnet-wide boot class - # that still matches it defeats the marking. Decision 16. - -@S-08 -Scenario: A node with several MACs is reachable on any of them - Given a mac table entry of the form "mac1!host1|mac2!host2" - When either MAC discovers - Then each is offered the address of its own hostname - # dhcp.pm splits on "|" and on "!"; the hostname defaults to the node name. - -@S-09 -Scenario: An InfiniBand interface is reserved by its full hardware address - Given a node MAC that is an InfiniBand identity, not a 6-byte Ethernet MAC - When makedhcp runs for it - Then the reservation is created with hardware-type 37 and the twin address # [config] - # dhcp.pm _infiniband_identity_present / _infiniband_twin_update_commands - -@S-10 -Scenario: A malformed MAC is rejected outright - Given a mac table entry that is not 6 to 9 colon- or dash-separated octets - When makedhcp runs for that node - Then it reports "Invalid mac address for " and creates nothing # [config] -``` - ---- - -## Feature: Telling a machine what to boot, by client architecture - -xCAT signals the loader through **`siaddr` (next-server) and the BOOTP `file` -header**, deliberately not through options 66 and 67 -(`dhcp.pm:4446,4610`). Firmware reads the header; a server that answers only in -option 67 will not boot these clients. - -Source: `perl-xCAT/xCAT/DHCP/BootPolicy.pm` -(`isc_client_architecture_lines`, `kea_client_classes`, -`kea_httpboot_network_classes`, `kea_s390x_network_classes`) - -```gherkin -@S-11 -Scenario Outline: The loader follows the client architecture in option 93 - Given a subnet configured by makedhcp -n - When a client sends a DHCPDISCOVER with option 93 = - Then the reply names - And siaddr is the tftpserver for that subnet - - Examples: - | arch | client | loader | - | 0x0000 | x86 BIOS PXE | xcat/xnba.kpxe | - | 0x0002 | ia64 | elilo.efi | - | 0x0007 | x86-64 UEFI | xcat/xnba.efi | - | 0x0009 | x86-64 UEFI, alternate id | xcat/xnba.efi | - | 0x0010 | x86-64 UEFI HTTP boot | xcat/xnba.efi | - | 0x000b | aarch64 UEFI | boot/grub2/grub2.aarch64 | - | 0x000c | ppc64 UEFI | /boot/grub2/grub2.ppc | - | 0x001b | riscv64 UEFI, TFTP | boot/grub2/grub2.riscv64 | - # 0x000e and 0x001f are answered with a conf-file rather than a loader, and - # 0x001c with a URL; each has its own scenario below. Every row here holds on - # both backends -- see "Appendix: backend parity decisions". - -@S-12 -Scenario: A loader is named only when it is there to be fetched - Given /xcat/xnba.kpxe does not exist - When an x86 BIOS client discovers - Then it is offered an address - And it is not handed xcat/xnba.kpxe - And it is not handed some other loader in its place - # Naming a file the tftp server does not have costs the client a timeout it - # cannot diagnose; substituting a different loader hides the missing one and - # boots something nobody asked for. Both backends therefore gate every - # architecture branch on the loader existing, and neither substitutes. - # Decision 8/15 in the appendix. - -@S-13 -Scenario: An HTTP-boot client is given a URL and the tag its firmware demands - When a client sends option 93 = 0x001c (riscv64 HTTP boot) - Then the reply names a boot file beginning "http://" - And that URL ends in the same grub2 image the TFTP path serves - And the reply carries option 60 = "HTTPClient" - # UEFI HTTP boot firmware discards a reply that is not tagged HTTPClient. - # BootPolicy.pm:63-104 - -@S-14 -Scenario: The HTTP boot URL honours a non-default web port - Given site.httpport is not 80 - When an HTTP-boot client discovers - Then the URL carries that port - # BootPolicy.pm:76 -- port 80 is left out of the URL entirely. - -@S-15 -Scenario: An HTTP boot class is only offered where the image exists - Given the riscv64 grub2 image is absent from the tftp directory - When makedhcp -n runs - Then no HTTP boot class is written for that network # [config] - # BootPolicy.pm:85 loader_present. The same rule as the scenario above, at - # the granularity a class gives: what is not there is not offered. - -@S-16 -Scenario: A QEMU s390x client is given a conf-file rather than a loader - When a client sends option 93 = 0x001f - Then the reply carries option 209 (conf-file) = "s390x/_" - # BootPolicy.pm:106-131 and dhcp.pm:4446. The conf-file is per network. - -@S-17 -Scenario: An OPAL-v3 client is given a petitboot conf-file URL - When a client sends option 93 = 0x000e - Then the reply carries option 209 = "http:///tftpboot/pxelinux.cfg/p/_" - # BootPolicy.pm:168 - -@S-18 -Scenario: A client that identifies itself by vendor class alone still boots - When a client sends vendor class "Etherboot-5.4" and no usable option 93 - Then it is handed xcat/xnba.kpxe - # BootPolicy.pm:151 - -@S-19 -Scenario: An ONIE switch is offered an installer URL before it is a node - When a client sends a vendor class beginning "onie_vendor" - Then the reply carries option 114 (www-server) = the subnet's installer URL - # A switch announces onie_vendor on its first boot, which is by definition - # before anyone has defined it as a node, so the offer has to come from the - # subnet. A node definition refines it -- see the netboot=onie scenario. - -@S-20 -Scenario: A client that says nothing recognisable falls through to yaboot - When a client sends no client architecture and no known vendor class - And no filename has been set by an earlier rule - Then the reply names "/yaboot" - # BootPolicy.pm:172. A universal default of "/yaboot" is a poor one, but it - # is the one xCAT has always had, and changing what an unrecognised client is - # told to boot is a product decision rather than a parity fix. Decision 7. -``` - ---- - -## Feature: Chained network boot -- first stage, then second stage - -The loader that firmware runs comes straight back for a second DHCP exchange. -If it were handed the same loader again it would chainload itself forever, so -the second request must be answered differently. The loader announces itself in -the **user class, option 77**. - -Source: `BootPolicy.pm:133-257`, `dhcp.pm:1169-1188` - -```gherkin -@S-21 -Scenario: Firmware with no user class gets the loader binary - When a client with no option 77 discovers - Then it is handed a loader binary, not a script URL - -@S-22 -Scenario: The loader announcing itself gets a script instead - Given the first stage loader announces user class "xNBA" - When it discovers with option 93 = 0x0000 - Then it is handed "http:///tftpboot/xcat/xnba/nets/_" - And the reply is broadcast rather than unicast - # always-broadcast on -- BootPolicy.pm:143. The loader has no address yet. - -@S-23 -Scenario: A UEFI second stage gets the UEFI script - Given the loader announces user class "xNBA" - When it discovers with option 93 = 0x0007 or 0x0009 - Then it is handed the same URL with ".uefi" appended - # BootPolicy.pm:145-148 - -@S-24 -Scenario Outline: The user class is recognised however the client encodes it - Given the client announces user class "xNBA" encoded as - When it discovers - Then it is recognised as a second stage and handed the script URL - - Examples: - | encoding | - | a bare string, as most clients send it | - | RFC 3004 length-prefixed, as the RFC says | - # Kea accepts both: kea_xnba_user_class_test tests option[77].text, the raw - # hex, and the length-prefixed substring. ISC accepts both through - # isc_xnba_user_class_test, which compares the last four bytes: - # `suffix(option user-class-identifier, 4) = "xNBA"` is true of the bare - # string and of "\x04xNBA" alike. It has to be one expression rather than an - # alternation -- dhcpd's grammar has no parenthesised grouping, so `if (a or - # b) and c {` is a parse error and the daemon will not start. - -@S-25 -Scenario: A known node's second stage is addressed to that node - Given a node whose netboot method is xnba - And that node announces user class "xNBA" - When it discovers - Then it is handed "http:///tftpboot/xcat/xnba/nodes/" - And not the per-network script - # dhcp.pm:1183, BootPolicy.pm:178-212. Two machines chainloading at the same - # moment must not run the same script. - -@S-26 -Scenario: A node's first stage is unaffected by its second stage rule - Given the same node - When it discovers with no user class - Then it is handed xcat/xnba.kpxe -``` - ---- - -## Feature: The boot file a known node is given follows its netboot method - -Source: `dhcp.pm:1169-1232` (ISC), `kea_boot_for_node` (Kea) - -```gherkin -@S-27 -Scenario Outline: netboot decides the loader for a defined node - Given a node defined with netboot= - When that node's MAC discovers - Then the reply names - - Examples: - | method | loader | - | xnba | xcat/xnba.kpxe, or xcat/xnba.efi for x86-64 UEFI | - | pxe | pxelinux.0 | - | grub2 | /boot/grub2/grub2- | - | grub2-* | /boot/grub2/grub2- | - | yaboot | /yb/node/yaboot- | - | nimol | /vios/nodes/ | - # The node's own method decides, on either backend. A subnet-wide boot class - # answering in its place -- because the backend has no branch for that method - # -- is a bug, not a fallback: the operator set netboot for a reason, and the - # subnet class knows only the architecture. Decisions 5 and 6. - -@S-28 -Scenario: A petitboot node is given a conf-file, not a boot file - Given a node defined with netboot=petitboot - When it discovers - Then the reply carries option 209 = "http:///tftpboot/petitboot/" - And no boot file is named - # petitboot firmware acts on a filename if it sees one, so naming one as well - # as the conf-file starts a TFTP fetch the operator did not ask for. - # Decision 12. - -@S-29 -Scenario: An ONIE switch is given an installer URL keyed on its vendor class - Given a node defined with netboot=onie and an osimage whose pkgdir exists - When it discovers announcing a vendor class beginning "onie_vendor" - Then the reply carries option 114 (www-server) = the installer URL - # dhcp.pm:1214, the node's own image; the subnet-wide offer above is what an - # undefined switch gets. Both exist on both backends -- decision 11. - -@S-30 -Scenario: A ScaleMP client is given the ScaleMP loader - Given a node defined with netboot=pxe - When it discovers announcing vendor class "ScaleMP" - Then the reply names "vsmp/pxelinux.0" - # dhcp.pm:1194 - -@S-31 -Scenario: A node told to boot from disk is not handed a loader - Given a node whose chain.currstate is "boot" or "iscsiboot" - And it has no iSCSI target - When it discovers, with or without the xNBA user class - Then it is offered its address - And it is named no boot file at all, whatever its netboot method - # dhcp.pm:1139 -- otherwise a booted node netboots forever. "No boot file at - # all" rather than "not its own script": a node handed the stage-1 loader - # asks again as an xNBA second stage and is answered with the network's - # script, so stopping only the per-node script stops nothing. Both netboot - # methods that have a boot-from-disk rule, xnba and pxe, are covered by the - # same sentence for the same reason. - # - # An iSCSI node is the exception and keeps its loader: its root disk is on - # the network and gPXE is what attaches it. ISC gates that on $doiscsi; Kea - # leaves those MACs out of the xcat-localboot class. - -@S-32 -Scenario: A Windows UEFI install defers to the proxyDHCP daemon - Given a node in a Windows install or winshell state on UEFI firmware - And proxydhcp is enabled for it - When it discovers with option 93 = 0x0000, 0x0007 or 0x0009 - Then the reply names no boot file - And it carries option 60 = "PXEClient" - # dhcp.pm:1178 -- the tag tells firmware to ask the proxyDHCP daemon on 4011. - -@S-33 -Scenario: An iSCSI node is given a root path - Given a node with an iscsi server and target defined - When it discovers - Then the reply carries option 17 (root-path) = "iscsi::6:3260::" - -@S-34 -Scenario: An IBM iSCSI initiator is given the vendor form of the same thing - Given the same node, with an initiator name defined - When it discovers announcing vendor class "ISAN" - Then the reply carries the iSCSI IQN and root path as vendor options, not option 17 - # dhcp.pm:1153-1163 -- ISAN initiators do not read the standard option. - -@S-35 -Scenario: The node is told its own name - Given any known node - When it discovers - Then the reply carries option 12 (host-name) = the node name - # Genesis and every installer that takes its hostname from the lease depend - # on this. It has to be the option on the wire: ISC's `send host-name` is a - # parameter for the reply's sname/file handling and is not option 12, so a - # reservation that carries only that satisfies nothing here. Decision 21. -``` - ---- - -## Feature: Where the machine fetches its loader from -- hierarchy - -A service node serves the racks behind it. Which server a node is sent to is -what makes a hierarchical cluster work, and it is visible in one field. - -Source: `next_server_for_node`, which both backends read, and `dhcp.pm:3441` - -```gherkin -@S-36 -Scenario: next-server follows noderes.tftpserver - Given a node whose noderes.tftpserver names a service node - When it discovers - Then siaddr is that service node's address - -@S-37 -Scenario: next-server falls back to xcatmaster - Given a node with no tftpserver but with an xcatmaster - When it discovers - Then siaddr is the xcatmaster's address - -@S-38 -Scenario: next-server otherwise comes from the subnet - Given a node with neither tftpserver nor xcatmaster - When it discovers - Then siaddr is the tftpserver of the subnet it discovered on - # '${next-server}' is what a node that named no server of its own is - # given: ISC leaves the subnet statement to answer it, and Kea says - # nothing in the reservation, which comes to the same thing. - -@S-39 -Scenario: A node's URLs point at the same server as its next-server - Given a node sent to a service node - When it is handed an xNBA script URL or a petitboot conf-file - Then the host in that URL is the service node, not the management node - -@S-40 -Scenario: A pool delegated to another server is not served here - Given networks.dhcpserver names a machine that is not this one - When makedhcp -n runs on this machine - Then this machine serves no dynamic range for that subnet - And an unknown MAC on it draws no reply from this machine - # dhcp.pm:3441 -- two servers answering one pool would hand out conflicting - # addresses. - -@S-41 -Scenario: Reservations are still served for a delegated subnet - Given the same subnet, with its pool delegated - When a known node on it discovers - Then it is still offered its reserved address by this machine - -@S-42 -Scenario: A dynamic range without a dhcpserver is an error - Given a network with a dynamicrange and no dhcpserver in a hierarchy - When makedhcp runs - Then it reports the missing dhcpserver # [config] - # dhcp.pm:1593 - -@S-43 -Scenario: A service node serves only the interfaces it was given - Given a service node with servicenode.dhcpinterfaces set - When makedhcp -n runs on it - Then the daemon listens on exactly those interfaces # [config] - # dhcp.pm:1965-1978. The ":noboot" suffix is stripped and ignored here; it - # only matters to mknb. -``` - ---- - -## Feature: What else the reply has to carry for a deployment to complete - -An address alone does not deploy a node. The installer needs a route, a -resolver and a clock. - -Source: `dhcp.pm:4520-4600` - -```gherkin -@S-44 -Scenario: The reply carries the subnet's gateway - Given networks.gateway is set for the subnet - When any client discovers on it - Then the reply carries option 3 (routers) = that gateway - -@S-45 -Scenario: A gateway outside its own subnet is rejected - Given networks.gateway is not inside net/mask - When makedhcp -n runs - Then it fails with "Specified gateway is not valid for /" # [config] - -@S-46 -Scenario: The reply carries resolvers and the domain - Given nameservers are set for the subnet or in site - When any client discovers - Then the reply carries option 6 (domain-name-servers) - And option 15 (domain-name) - And option 119 (domain-search) listing every known domain - # dhcp.pm:4563-4589; domain-search is skipped on sles10 and rhel5. - -@S-47 -Scenario: The reply carries NTP servers when they are configured - Given ntpservers are set for the subnet or in site - Then the reply carries option 42 (ntp-servers) - -@S-48 -Scenario: The reply always carries a log server - When any client discovers - Then the reply carries option 7 (log-servers) - And it is the management node's own address when none was configured - # dhcp.pm:4560 -- genesis and the installer log to it during discovery. - -@S-49 -Scenario: The reply carries an MTU where the network defines one - Given networks.mtu is set - Then the reply carries option 26 (interface-mtu) - # A provisioning network on jumbo frames will not complete an install - # otherwise. - -@S-50 -Scenario: The lease is as long as site.dhcplease says - Given site.dhcplease is set - When a client that is not PXE firmware is acknowledged - Then option 51 (lease-time) is that value - And 43200 seconds when site.dhcplease is unset - # dhcp.pm:4524. "Not PXE firmware" because a client announcing a PXEClient - # vendor class is answered by S-51 instead, and the two are the same reply - # field. - -@S-51 -Scenario: A PXE client gets a short lease - When a client announcing a vendor class beginning "PXEClient" is acknowledged - Then option 51 (lease-time) is 600 seconds, not the cluster default - # dhcp.pm:4939 -- class "pxe" sets max-lease-time 600 so a pool address taken - # by firmware is returned quickly rather than being held for half a day - # during a discovery of a few thousand machines. 600 is the number both - # backends have to land on, and it is asserted on the wire because ISC also - # sets min-lease-time on the subnet and the two directives - # disagree. Decision 22. - -@S-52 -Scenario: Cumulus switches are told where to find their provisioning script - When any client discovers - Then the reply carries option 239 = "http:///install/postscripts/cumulusztp" - # dhcp.pm:4592 -- pushed on every subnet unconditionally. - -@S-53 -Scenario: The server answers authoritatively - When a client sends a DHCPREQUEST for an address that is not its own - Then it receives a DHCPNAK rather than silence - # dhcp.pm:4521 "authoritative;" -- a node that moved rack must be told to - # start over rather than waiting out a lease that will never be renewed. The - # server owns these networks on either backend. Decision 24. - -@S-54 -Scenario: A BOOTP-only client is served from the dynamic range - Given a subnet with a dynamic range - When a client that speaks BOOTP but not DHCP boots on it - Then it is given an address - # dhcp.pm:4648 -- "range dynamic-bootp ;". Decision 9: the hardware - # this is for is old enough that it will not be replaced, so dropping it is - # not on the table; a backend that cannot answer BOOTP has to be made to. -``` - ---- - -## Feature: Discovery -- a machine nobody has told the cluster about - -Discovery is the case where the server knows nothing about the client and still -has to get it far enough to identify itself. - -Source: `dhcp.pm:4481-4498`, `BootPolicy.pm`, xCAT discovery documentation - -```gherkin -@S-55 -Scenario: An undiscovered machine gets an address and a loader - Given a subnet with a dynamic range - When a machine with no reservation PXE boots on it - Then it is offered a pool address - And it is told what to boot, chosen by its option 93 - # This is what lets the genesis image run and report the machine's identity. - -@S-56 -Scenario: An unknown machine is told what to boot from the subnet, not a reservation - Given a subnet with a dynamic range - When a MAC with no reservation discovers on it - Then the boot class that answers is one written on the subnet - And it is answered whether or not any node holds that MAC - # Discovery is the case where there is no reservation to read a boot file - # from, so a backend that only names loaders per host cannot discover - # anything. kea_client_classes are subnet-wide and the ISC architecture - # chain is written into the subnet as well (dhcp.pm:4654). Decision 1. - -@S-57 -Scenario: A machine discovers on the architecture it actually is - When an aarch64 machine with no reservation PXE boots - Then it is handed the aarch64 loader, not the x86 one - # A discovery path that only works for x86 leaves every other architecture - # undiscoverable. - -@S-58 -Scenario: A machine adopted during discovery is served without a restart - Given a machine currently holding a pool address - When it is defined as a node and makedhcp is run for it - And it discovers again - Then it is offered its reserved address - And no DHCP daemon was restarted in between - # ISC reservations are injected over OMAPI into dhcpd.leases, not written to - # dhcpd.conf (dhcp.pm addnode). Discovery depends on this: restarting the - # daemon mid-discovery drops every machine being discovered. - -@S-59 -Scenario: A node removed from the cluster stops being served - Given a node with a reservation - When makedhcp -d is run for it - And its MAC discovers again - Then it is offered a pool address, or none, but not its old reservation -``` - ---- - -## Feature: Regenerating and updating the configuration - -Source: `dhcp.pm:87` (usage), `xCAT::DHCP::OmapiRunner`, `xCAT::DHCP::OmapiPolicy` - -```gherkin -@S-60 -Scenario: makedhcp -n rewrites the whole configuration - When makedhcp -n runs - Then every network in the networks table has a subnet block # [config] - And existing reservations are preserved or re-added - -@S-61 -Scenario: makedhcp adds those nodes only - When makedhcp is run for a node range - Then only those nodes' reservations change - -@S-62 -Scenario: makedhcp -a adds every node with a MAC - When makedhcp -a runs - Then every node with a MAC and a resolvable IP has a reservation - -@S-63 -Scenario: makedhcp -q reports what the server holds - When makedhcp -q is run for a node - Then it prints that node's reservation, or reports that there is none - -@S-64 -Scenario: An external DHCP server is left alone - Given site.externaldhcpservers is set - When makedhcp runs - Then no local configuration file is written # [config] - # dhcp.pm newconfig returns immediately. -``` - ---- - -## Feature: The daemon listens where xCAT was told to serve - -Wholly **[config]**: the wire cannot show which interfaces a daemon *did not* -bind. Covered by `xCAT-test/unit/dhcp_debian_interfaces.t`. - -Source: `dhcp.pm:1980-2030`, `dhcp.pm:2264-2310`, `debian_sysconfig_interface_keys` - -```gherkin -@S-65 -Scenario: site.dhcpinterfaces names the interfaces to serve - Given site.dhcpinterfaces = "eth1" - When makedhcp -n runs - Then dhcpd is started with eth1 and no other interface - -@S-66 -Scenario: site.dhcpinterfaces may be scoped per host - Given site.dhcpinterfaces = "mn|eth1;sn1,sn2|eth2" - When makedhcp -n runs on each host - Then each serves only the interfaces named against it - -@S-67 -Scenario Outline: The Debian default file names the variable the unit reads - Given isc-dhcp-server version - When makedhcp -n runs - Then /etc/default/isc-dhcp-server assigns - And dhcpd is launched with the interfaces xCAT serves - - Examples: - | version | variable | - | 4.3.3-5ubuntu12 | INTERFACES | - | 4.3.5-3ubuntu7 | INTERFACES | - | 4.4.1-2.1ubuntu5 | INTERFACES | - | 4.4.1-2.3ubuntu2 | INTERFACESv4 / INTERFACESv6 | - | 4.4.3-P1-4ubuntu2 | INTERFACESv4 / INTERFACESv6 | - # The split is a Debian packaging change, not an upstream ISC one: 20.04 and - # 22.04 both ship upstream 4.4.1. An unset variable expands to nothing and - # leaves dhcpd bound to every interface on the machine. - -@S-68 -Scenario: A remote interface is not passed to the local daemon - Given an interface entry marked !remote! - Then it does not appear on dhcpd's command line -``` - ---- - -## Feature: The two backends behave the same on the wire - -`xCAT::DHCP::Backend::default_backend` returns `kea` for Ubuntu >= 22.04 and -EL >= 10, `isc` below that; `auto` falls back to `isc` when `kea-dhcp4` is not -installed (`Backend.pm:36-48`). A cluster therefore runs either, and a node must -boot identically on both. - -```gherkin -@S-69 -Scenario: The same client gets the same answer from either backend - Given a network and a node defined once - When the cluster is served by ISC - And then by Kea, from the same definitions - Then a client's address, next-server and boot file are the same both times - And the same options - # Every scenario in this document is one of "either backend". There is no - # backend-conditional clause anywhere in it any more, and none may be added: - # a Then clause that holds on one backend is a bug report, not a - # specification. Where the two disagreed, the appendix records which answer - # was chosen and why. -``` - ---- - -## Feature: IPv6 - -Source: `dhcp.pm:568-583`, `addnode6`, `dhcp.pm:4293` - -```gherkin -@S-70 -Scenario: A node is reserved by DUID rather than by MAC - Given a node with a vpd.uuid - When makedhcp runs and the cluster uses IPv6 - Then a DHCPv6 reservation is created against DUID-UUID 00:04: - # dhcp.pm:731-745 - -@S-71 -Scenario: A node without a uuid is skipped - Given an IPv6 cluster and a node with no vpd.uuid - When makedhcp runs for it - Then it warns "Skipping DHCPv6 setup due to missing vpd.uuid information." - -@S-72 -Scenario: An IPv6 subnet serves a range6 - Given networks.dynamicrange holds an IPv6 CIDR - Then the DHCPv6 subnet declares range6 with that prefix - # dhcp.pm:4293; a subnet with no range warns that hosts without a static - # address will receive none. - -@S-73 -Scenario: The v6 daemon serves the same interfaces as the v4 one - Given a Debian management node serving IPv6 - Then INTERFACESv6 names the interfaces xCAT serves # [config] -``` - ---- - -## Notes on testing this specification - -- Scenarios not marked **[config]** are wire-observable and belong in - `xCAT-test/dhcptest/conf/`. They need a real DHCP server answering, which - `xCAT-test/autotest/testcase/dhcptest/dhcpfixture.sh` builds out of a veth - pair so a single-node management node can run them. -- Scenarios marked **[config]** assert on generated files or daemon command - lines and belong in `xCAT-test/unit/dhcp_*.t`, which need neither root nor a - network. -- A `.conf` file must never encode xCAT policy. Addresses, filenames and - architectures come in on the command line via `--set`; the fixture supplies - them from the node and network it defined. That is what keeps the same - scenarios usable against a server xCAT did not configure. -- Where two behaviours are both legitimate -- an unknown MAC being offered a - pool address versus being ignored -- they belong in separate `.conf` files, - chosen by whoever knows how the network under test is configured. Running - both against one network will always fail one of them. - -### What this specification does not cover on the wire - -Stated so that a green run is not read as more than it is: - -- **DHCPv6.** `dhcptest` speaks IPv4 only: it builds a BOOTP frame on UDP - 68→67. Everything under *Feature: IPv6* is therefore either asserted from the - generated configuration in `xCAT-test/unit/dhcp_*.t` or not asserted at all. -- **Relay agents.** `giaddr` and option 82 decide which subnet a reply is drawn - from, and no scenario here sends a relayed request: doing it honestly needs a - relay agent on a second network, not a forged `giaddr` from the same wire. - The hierarchy scenarios cover the part that is observable without one -- a - pool handed to another server stops being offered. -- **Service node deployment as a distinct case.** A service node boots exactly - as a compute node does; what differs is what it serves afterwards, which is - `servicenode.dhcpinterfaces` and the delegated pool. Both are covered, as - `[config]` and as the hierarchy scenarios respectively. -- **The architectures no loader exists for on the machine under test.** The - fixture skips those by name rather than asserting a filename that was never - configured; read the log for which ones actually ran. - ---- - -## Appendix A: backend parity decisions - -Every row is a place where ISC dhcpd and Kea answered the same frame -differently, or where only one of them answered it at all. They were found by -reading this document against `dhcp.pm` and `BootPolicy.pm` and are enumerated -in VersatusHPC/xcat-internal#175. Rows 26 and 27 are not in that issue: they -were found by the wire cases, which is what the wire cases are for. - -The decision column is now the specification: the scenarios above state it -unconditionally, and the wire cases assert it against both backends. "Already -parity" means the source had moved on by the time the finding was checked -- -the drift was in the spec text, not in xCAT. - -| # | Drift | Decision | Why | Scenario | Has to change | -| --- | --- | --- | --- | --- | --- | -| 1 | Unknown MAC told what to boot: subnet class (Kea) vs per-host block (ISC) | Subnet-wide, both | Discovery has no reservation to read a boot file from; per-host only cannot discover anything | S-55, S-56 | already parity -- the ISC chain is written into the subnet too (dhcp.pm:4654) | -| 2 | 0x000c ppc64: `/boot/grub2/grub2.ppc` (Kea) vs `/yaboot` (ISC) | grub2.ppc | yaboot is not a UEFI loader; ISC's answer is the absence of a branch, not a decision | S-11 | ISC: add the 0x000c branch | -| 3 | 0x0010 UEFI HTTP x86-64: matched (Kea) vs unmatched (ISC) | `xcat/xnba.efi` | 0x0010 is a real client architecture; falling through to `/yaboot` cannot be right for any of them | S-11 | ISC: add the 0x0010 branch | -| 4 | 0x000e OPAL-v3 petitboot conf-file: ISC only | conf-file URL, both | Stated unconditionally; a POWER machine has nowhere to fetch its petitboot config otherwise | S-17 | already parity -- `kea_opal_client_class` | -| 5 | `netboot=xnba`: per-node branch on ISC, none in `kea_boot_for_node` | Per-node, both | The operator set `netboot` deliberately; the subnet class knows only the architecture | S-27 | Kea: add the xnba branch | -| 6 | `netboot=nimol`: `/vios/nodes/` on ISC only | `/vios/nodes/`, both | Same reason; on Kea a NIMOL node is handed the subnet's loader and does not install | S-27 | Kea: add the nimol branch | -| 7 | Unmatched client: `/yaboot` (ISC) vs no boot file (Kea) | `/yaboot`, both | A poor default, but xCAT's long-standing one; changing what an unrecognised client boots is a product decision, not a parity fix | S-20 | Kea: add the fallback | -| 8 | Loader file missing: silence (ISC per-host) vs `pxelinux.0` (Kea) | Name nothing; never substitute | Naming an absent file costs a timeout the client cannot diagnose; substituting boots something nobody asked for | S-12 | Kea: drop the pxelinux.0 fallback. ISC: gate the subnet branches on the loader existing | -| 9 | BOOTP-only client: served by ISC, no Kea counterpart | Served, both | The hardware this is for will not be replaced | S-54 | Kea: serve BOOTP | -| 10 | Vendor class `Etherboot-5.4`: ISC only | `xcat/xnba.kpxe`, both | Stated unconditionally; on Kea such a client then matches nothing at all | S-18 | Kea: add the vendor-class match | -| 11 | `onie_vendor`: per subnet (ISC) vs per node only (Kea) | Both forms, both backends | A switch announces `onie_vendor` on its first boot, before anyone has defined it as a node | S-19, S-29 | Kea: add the subnet class | -| 12 | `netboot=petitboot`: Kea sets `boot-file-name` as well as the conf-file | conf-file only | petitboot acts on a filename if it sees one -- a TFTP fetch that does not happen on ISC | S-28 | Kea: drop `boot-file-name` | -| 13 | 0x001f QEMU s390x: thought to be Kea-only | conf-file `s390x/_`, both | Stated unconditionally | S-16 | already parity -- ISC has the 00:1f branch | -| 14 | 0x001c riscv64 HTTP boot: thought to be Kea-only | URL plus option 60 `HTTPClient`, both | Firmware discards a reply that is not tagged | S-13 | already parity -- ISC has the 00:1c branch | -| 15 | HTTP boot class gated on the image existing: Kea only | Gate on both | Same rule as 8, at the granularity a class gives | S-12, S-15 | ISC: gate the branch | -| 16 | `*NOIP*` interface: denied by an ISC statement, no Kea counterpart | Not answered at all, both | The marking exists so nothing boots on that interface; a subnet-wide class that still matches it defeats the marking | S-07 | Kea: drop the packet for that MAC | -| 17 | Boot-from-disk suppression: per-host on ISC, subnet-wide classes on Kea | Suppressed, both | Otherwise an installed node netboots forever instead of booting its disk | S-31 | Kea: suppress per node over the subnet class | -| 18 | Windows UEFI proxyDHCP deferral: ISC only | No boot file plus option 60 `PXEClient`, both | The tag is what hands the client to the proxyDHCP daemon on 4011; a boot file from the subnet class pre-empts it | S-32 | Kea: add the deferral | -| 19 | Vendor class `ScaleMP`: ISC only | `vsmp/pxelinux.0`, both | A different binary; on Kea the reservation's `pxelinux.0` wins and the machine boots the wrong loader | S-30 | Kea: add the vendor-class match | -| 20 | iSCSI root-path and the ISAN vendor form: ISC only | Both forms, both backends | ISAN initiators do not read option 17, so emitting only the standard form does not serve them | S-33, S-34 | Kea: add the ISAN vendor form, including the empty option 43 that carries the space -- Kea sends an encapsulated space only when the option encapsulating it is configured too | -| 21 | option 12 host-name: sent by Kea, `send host-name` on ISC | option 12 = the node name, both | `send host-name` is a parameter for sname/file handling, not option 12; genesis and installers read the option | S-35 | ISC: emit `option host-name`. Kea: write the reservation's hostname fully qualified (trailing dot), or `ddns-qualifying-suffix` is appended to it and the node is told an FQDN. Kea builds option 12 and the DDNS name from that one field, so the suffix still qualifies the dynamic clients that have no reservation | -| 22 | Short PXE lease: ISC class `pxe` against the subnet's `min-lease-time` | option 51 = 600 for a `PXEClient` vendor class | A pool address taken by firmware must come back quickly during a large discovery | S-51 | both: land on 600 and assert it on the wire | -| 23 | Adoption without a daemon restart: an ISC/OMAPI property | No restart, both | Restarting mid-discovery drops every other machine being discovered | S-58 | already asserted on both by `run-adoption` | -| 24 | `authoritative`: an ISC directive, nothing cited for Kea | DHCPNAK rather than silence, both | A node that moved rack must be told to start over | S-53 | Kea: `authoritative` on | -| 25 | The whole common-option block sourced only from the ISC generator | Parity, asserted not assumed | An installer that loses its resolver, route, clock or MTU fails late and obscurely | S-44 to S-52 | neither, so far -- now asserted on the wire on both | -| 26 | `noderes.xcatmaster`: read for every node (Kea) vs only for `petitboot` and `onie` (ISC) | Read for every node, both | An operator sets it per node and deliberately; a hierarchical cluster's compute nodes were being sent to the management node on ISC | S-37 | ISC: honour it whatever the netboot method, and put it in siaddr as well as in the URLs built from it | -| 27 | No server named at all: subnet value (ISC) vs `my_ip_facing` (Kea) | The subnet's value, both | The two agree only while the subnet's tftpserver is this machine; `networks.tftpserver` exists precisely to say otherwise | S-38 | Kea: leave `next-server` out of the reservation and let the subnet answer | - -## Appendix B: spec review - -What this revision changed, and why. The review numbers are the ones tagged -above. - -| Change | Scenarios | Rationale | -| --- | --- | --- | -| Every scenario given a stable review number `@S-nn` | all | So a finding, a `.conf` scenario and a CI failure can name the same behaviour | -| "Known asymmetries between the backends" deleted | was under S-69 | A list of drifts is a bug report; the specification has to say which answer is right. Replaced by Appendix A | -| Backend-conditional discovery scenario rewritten | S-56 | It said the answer was "backend policy". It is not: the operator does not choose the backend | -| Architecture Examples merged into one table, 0x0010 and 0x000c added | S-11 | Two tables, one of them annotated "(Kea only)" per row, is not executable as a scenario outline | -| "told nothing" widened to "and not some other loader in its place" | S-12 | The old Then clause was true on both backends and hid Kea substituting `pxelinux.0` | -| Loader-presence gating stated as one rule for every branch | S-12, S-15 | The two backends gated different things at different granularities | -| Subnet-wide ONIE offer given its own scenario | S-19 | It was only visible as a comment on the per-node one, which hid that Kea has only the per-node form | -| `no boot file is named` justified rather than merely stated | S-28 | It is the clause Kea contradicts, so the reason it exists belongs next to it | -| option 12 stated as the option on the wire | S-35 | `send host-name` satisfied a reading of the old text without sending option 12 | -| PXE lease pinned to 600 seconds | S-51 | "short, not the cluster default" cannot be asserted, and the ISC directives disagree with each other | -| BOOTP and `authoritative` stated for both backends | S-53, S-54 | Both were sourced from the ISC generator alone | -| `*NOIP*`, boot-from-disk and proxyDHCP restated as requirements on the reply | S-07, S-31, S-32 | Each was phrased around the ISC mechanism that implements it, so no Kea gap was visible | diff --git a/xCAT-test/provtest/README.md b/xCAT-test/provtest/README.md index a5b0562fa..b27d587e7 100644 --- a/xCAT-test/provtest/README.md +++ b/xCAT-test/provtest/README.md @@ -8,8 +8,9 @@ Scenarios are `.conf` files, not Perl. The tool knows nothing about xCAT's database and never runs an xCAT command to decide what to expect: everything it expects arrives on the command line, from the fixture that set the cluster up. -The specification the scenarios implement is [spec.md](spec.md); every scenario -carries the `@P-nn` tag of the clause it covers. +The specification the scenarios implement is `specs/provision-chain.md` in the +internal repository; every scenario carries the `@P-nn` tag of the clause it +covers, so a failure names the clause without the document being at hand. ## Why @@ -197,7 +198,7 @@ something those programs will do. ## Shipped scenarios -| File | Stage | Scenarios | spec.md | +| File | Stage | Scenarios | Clauses | | --- | --- | --- | --- | | `dns.conf` | DNS | `node-forward`, `node-reverse`, `node-alias`, `forwarded-name`, `local-nxdomain`, `master-resolves` | P-01..P-07 | | `dns-removal.conf` | DNS | `removed-node` | P-08 | @@ -299,7 +300,6 @@ on a cluster that could not boot a single node. provtest/ README.md this file - spec.md the specification, @P-01 to @P-75 src/provtest entry point; runs from a checkout, no install src/provtest_lib/ cli.py argument parsing and the three subcommands diff --git a/xCAT-test/provtest/conf/discovery-artefacts.conf b/xCAT-test/provtest/conf/discovery-artefacts.conf index b50f664b2..c11798636 100644 --- a/xCAT-test/provtest/conf/discovery-artefacts.conf +++ b/xCAT-test/provtest/conf/discovery-artefacts.conf @@ -1,4 +1,4 @@ -# Stage 5: the genesis / discovery artefacts. spec.md P-25 to P-31. +# Stage 5: the genesis / discovery artefacts. Spec P-25 to P-31. # # A machine nobody has defined has no node name to look a file up by, so # everything here is keyed on the *network* it booted on -- the only thing the diff --git a/xCAT-test/provtest/conf/dns-removal.conf b/xCAT-test/provtest/conf/dns-removal.conf index 2ec010f59..e8fa297c0 100644 --- a/xCAT-test/provtest/conf/dns-removal.conf +++ b/xCAT-test/provtest/conf/dns-removal.conf @@ -1,4 +1,4 @@ -# Stage 1: DNS. spec.md P-08. +# Stage 1: DNS. Spec P-08. # # Run this only *after* the node has been removed and makedns re-run. It is a # separate file rather than a scenario in dns.conf because the two make diff --git a/xCAT-test/provtest/conf/dns.conf b/xCAT-test/provtest/conf/dns.conf index 05679c17c..cfde6444a 100644 --- a/xCAT-test/provtest/conf/dns.conf +++ b/xCAT-test/provtest/conf/dns.conf @@ -1,4 +1,4 @@ -# Stage 1: DNS. spec.md P-01 through P-08. +# Stage 1: DNS. Spec P-01 through P-08. # # Everything a node does after DHCP is done by name, and xcatd decides which # node it is talking to by resolving the client's address backwards. So the diff --git a/xCAT-test/provtest/conf/findme.conf b/xCAT-test/provtest/conf/findme.conf index 9dc577d04..a84bb5a9b 100644 --- a/xCAT-test/provtest/conf/findme.conf +++ b/xCAT-test/provtest/conf/findme.conf @@ -1,4 +1,4 @@ -# Stage 7: findme. spec.md P-42 to P-46, P-48, P-73. +# Stage 7: findme. Spec P-42 to P-46, P-48, P-73. # # Read the gates before reading the scenarios, because they are not where they # look. xcatd's UDP listener sends `processing` back to the client's TCP 3001 diff --git a/xCAT-test/provtest/conf/flowcontrol.conf b/xCAT-test/provtest/conf/flowcontrol.conf index ba923cfa1..61da86aca 100644 --- a/xCAT-test/provtest/conf/flowcontrol.conf +++ b/xCAT-test/provtest/conf/flowcontrol.conf @@ -1,4 +1,4 @@ -# Stage 6: flow control on UDP 3001. spec.md P-40, P-41. +# Stage 6: flow control on UDP 3001. Spec P-40, P-41. # # A large discovery has hundreds of machines asking for the same few xcatd # slots at once, so genesis asks before it connects: `xcatflowrequest` sends diff --git a/xCAT-test/provtest/conf/http.conf b/xCAT-test/provtest/conf/http.conf index 171014153..060b388bd 100644 --- a/xCAT-test/provtest/conf/http.conf +++ b/xCAT-test/provtest/conf/http.conf @@ -1,4 +1,4 @@ -# Stage 4: HTTP. spec.md P-32 to P-39, and P-35/P-36 for the cross-stage half. +# Stage 4: HTTP. Spec P-32 to P-39, and P-35/P-36 for the cross-stage half. # # Two Apache aliases carry the whole of a provision: /install for the repository # and the autoinst file, /tftpboot for the kernel and initrd that grub2-http and diff --git a/xCAT-test/provtest/conf/monitor.conf b/xCAT-test/provtest/conf/monitor.conf index 225ff145d..69c874600 100644 --- a/xCAT-test/provtest/conf/monitor.conf +++ b/xCAT-test/provtest/conf/monitor.conf @@ -1,4 +1,4 @@ -# Stage 13: the install monitor on TCP 3002. spec.md P-64 to P-66, P-68, P-69. +# Stage 13: the install monitor on TCP 3002. Spec P-64 to P-66, P-68, P-69. # # The installed system reports its progress here with `updateflag.awk`, over a # plain socket with no TLS and no certificate of any kind. The only thing that diff --git a/xCAT-test/provtest/conf/ordering.conf b/xCAT-test/provtest/conf/ordering.conf index 216707e4e..f9fc65c28 100644 --- a/xCAT-test/provtest/conf/ordering.conf +++ b/xCAT-test/provtest/conf/ordering.conf @@ -1,4 +1,4 @@ -# Failing at exactly one place. spec.md P-72, P-74, P-75. +# Failing at exactly one place. Spec P-72, P-74, P-75. # # Every scenario in this file arranges for each stage before the interesting # one to be correct, so the run goes red at one identifiable point. That is the diff --git a/xCAT-test/provtest/conf/tftp-grub2.conf b/xCAT-test/provtest/conf/tftp-grub2.conf index 58ede8031..f677def3d 100644 --- a/xCAT-test/provtest/conf/tftp-grub2.conf +++ b/xCAT-test/provtest/conf/tftp-grub2.conf @@ -1,4 +1,4 @@ -# Stage 3: TFTP, grub2. spec.md P-09, P-11 to P-17, P-24, P-70, P-71. +# Stage 3: TFTP, grub2. Spec P-09, P-11 to P-17, P-24, P-70, P-71. # # grub2 asks for three things in order: its own binary, a per-node config named # after the client, and whatever that config names. Each fetch is by exact diff --git a/xCAT-test/provtest/conf/tftp-petitboot.conf b/xCAT-test/provtest/conf/tftp-petitboot.conf index b6efdc2f9..e5d1ee8e6 100644 --- a/xCAT-test/provtest/conf/tftp-petitboot.conf +++ b/xCAT-test/provtest/conf/tftp-petitboot.conf @@ -1,4 +1,4 @@ -# Stage 3: TFTP, petitboot. spec.md P-23. +# Stage 3: TFTP, petitboot. Spec P-23. # # POWER firmware fetches a config named by the uppercase hex of its own # address, from the tftp root rather than from a subdirectory. xCAT writes diff --git a/xCAT-test/provtest/conf/tftp-pxelinux.conf b/xCAT-test/provtest/conf/tftp-pxelinux.conf index 554c15a62..a62675fa7 100644 --- a/xCAT-test/provtest/conf/tftp-pxelinux.conf +++ b/xCAT-test/provtest/conf/tftp-pxelinux.conf @@ -1,4 +1,4 @@ -# Stage 3: TFTP, pxelinux. spec.md P-18, P-19, P-20. +# Stage 3: TFTP, pxelinux. Spec P-18, P-19, P-20. # # pxelinux looks its config up under a series of names and takes the first that # answers: the MAC form, then progressively shorter hex-IP prefixes, then diff --git a/xCAT-test/provtest/conf/tftp-xnba.conf b/xCAT-test/provtest/conf/tftp-xnba.conf index 55dade754..084df78a2 100644 --- a/xCAT-test/provtest/conf/tftp-xnba.conf +++ b/xCAT-test/provtest/conf/tftp-xnba.conf @@ -1,4 +1,4 @@ -# Stage 3/4: TFTP then HTTP, xnba. spec.md P-21, P-22. +# Stage 3/4: TFTP then HTTP, xnba. Spec P-21, P-22. # # xnba is the one netboot method that crosses transports on its own: the gpxe # script arrives over TFTP and then fetches the kernel over HTTP. So a script diff --git a/xCAT-test/provtest/conf/xcatd-credentials.conf b/xCAT-test/provtest/conf/xcatd-credentials.conf index 24df9cdfd..eb3dd265a 100644 --- a/xCAT-test/provtest/conf/xcatd-credentials.conf +++ b/xCAT-test/provtest/conf/xcatd-credentials.conf @@ -1,4 +1,4 @@ -# Stage 11: getcredentials. spec.md P-61, P-62, P-63. +# Stage 11: getcredentials. Spec P-61, P-62, P-63. # # This is the one request a forged client cannot simply ask for. Before xcatd # signs anything it connects back to the client on TCP 300 and sends diff --git a/xCAT-test/provtest/conf/xcatd-destiny.conf b/xCAT-test/provtest/conf/xcatd-destiny.conf index 76b16cae0..df370d50a 100644 --- a/xCAT-test/provtest/conf/xcatd-destiny.conf +++ b/xCAT-test/provtest/conf/xcatd-destiny.conf @@ -1,4 +1,4 @@ -# Stage 9/10: getdestiny and nextdestiny over TLS 3001. spec.md P-49 to P-56. +# Stage 9/10: getdestiny and nextdestiny over TLS 3001. Spec P-49 to P-56. # # This is the request a genesis image makes to find out what it is for. The # genesis script pipes three lines of XML into `openssl s_client` with no client diff --git a/xCAT-test/provtest/conf/xcatd-policy.conf b/xCAT-test/provtest/conf/xcatd-policy.conf index 4713d1d22..037a1dbab 100644 --- a/xCAT-test/provtest/conf/xcatd-policy.conf +++ b/xCAT-test/provtest/conf/xcatd-policy.conf @@ -1,4 +1,4 @@ -# Stage 9: what a certless client may and may not ask for. spec.md P-59, P-60. +# Stage 9: what a certless client may and may not ask for. Spec P-59, P-60. # # The default policy grants a certless client exactly the commands a booting # node needs: getdestiny, nextdestiny, getpostscript, getcredentials, lsxcatd, diff --git a/xCAT-test/provtest/conf/xcatd-postscript.conf b/xCAT-test/provtest/conf/xcatd-postscript.conf index 3982043f2..947315e9e 100644 --- a/xCAT-test/provtest/conf/xcatd-postscript.conf +++ b/xCAT-test/provtest/conf/xcatd-postscript.conf @@ -1,4 +1,4 @@ -# Stage 12: getpostscript, on both transports. spec.md P-57, P-58, P-67. +# Stage 12: getpostscript, on both transports. Spec P-57, P-58, P-67. # # The node fetches the script it is to run after the installer finishes, and it # fetches it over whichever transport it has: TLS 3001 during genesis, diff --git a/xCAT-test/provtest/spec.md b/xCAT-test/provtest/spec.md deleted file mode 100644 index 88fa9bb19..000000000 --- a/xCAT-test/provtest/spec.md +++ /dev/null @@ -1,774 +0,0 @@ -# What xCAT is supposed to answer while a node provisions - -A specification of the traffic a machine actually produces between power-on and -a running installer, and of what xCAT must send back, written as scenarios so -it can be checked rather than argued about. - -Every scenario is stated from the point of view of the client on the wire: what -it asked for, and what came back. That is deliberate. `provtest` never reads the -xCAT database and never runs an xCAT command, so a scenario phrased in terms of -tables and plugins cannot be tested by it. Where a behaviour is genuinely not -observable from the wire -- a row that changed, a plugin that won -- it is -marked **[config]** and belongs to the Perl unit tests under `xCAT-test/unit/` -instead. - -Each scenario cites the source it was read from, so a scenario that stops -matching xCAT is a bug in one of the two and it is clear where to look. - -Every scenario carries a review number, `@P-nn`, in the Gherkin tag above it. -The numbers are stable and are the ones in VersatusHPC/xcat-internal#176: the -wire scenarios in `conf/`, the appendices here and any issue raised against this -document all cite them, so "P-42 was wrong" names one behaviour rather than a -paragraph nobody can find twice. They do not collide with `dhcptest`'s `@S-nn`. - -**DHCP is not specified here.** It is stage 2 of the chain and it has its own -document, `xCAT-test/dhcptest/spec.md`, because it is the one stage that needs a -raw L2 client and that has two backends to reconcile. This document starts where -a DHCP acknowledgement leaves off and ends where a real installer would take -over. - -Terms used throughout: - -| Term | Meaning | -| --- | --- | -| **the node** | a node in the xCAT database, with an address on a managed network and a PTR for it | -| **the client** | the forged machine on the wire: an address, and whatever it chooses to send | -| **a known client** | a client whose address reverse-resolves to a name in the node list | -| **an unknown client** | any other client: no PTR, or a PTR naming nothing defined | -| **HEXIP** | the node's IPv4 address as eight uppercase hex digits, e.g. `0A63000B` | -| **HEXNET** | the same encoding applied to a network address | -| **the master** | the address the node is told to talk xCAT to: `site.master`, or `noderes.xcatmaster` where it is set | -| **destiny** | `chain.currstate`: what the node is to do next -- `install`, `netboot`, `boot`, `discover`, `shell`, `runcmd` | -| **the chain** | `chain.currchain`: the states queued after this one | - ---- - -## Why forging a client is enough - -This is the load-bearing fact for the whole suite, so it is stated with -citations rather than assumed. Everything a booting node sends can be produced -by a script on a veth peer, with no VM, no genesis image and no client -certificate: - -- **TLS 3001 accepts a client with no certificate.** `SSL_verify_mode => 1` is - set without `SSL_VERIFY_FAIL_IF_NO_PEER_CERT` (`xCAT-server/sbin/xcatd` - around the SSL listener setup), so a certless client completes the handshake - and simply leaves the peer name undefined. -- **Identity otherwise comes from the reverse DNS of the peer address**, with - `-eth\d*`, `-myri\d*` and `-ib\d*` suffixes stripped. A client that owns an - address with a matching PTR *is* that node as far as xcatd is concerned. -- **The default policy grants the node-boot command set to certless clients.** - `xcatconfig` installs policy rows for `getdestiny`, `nextdestiny`, - `getpostscript`, `getcredentials`, `lsxcatd`, `syncfiles`, `litefile`, - `litetree`, `getadapter`, `getbmcconfig` and `remoteimmsetup`, none of them - carrying a `name` field -- i.e. none of them requiring a client certificate. -- **TCP 3002 has no TLS at all** and rests entirely on the reverse lookup. -- **findme is not signed for authentication.** `xcatd` gates it on the command - being `findme`, the UDP source port being below 1000, and the source address - being on a network xCAT manages (`xcatd:708-711`). Nothing verifies a - signature. See *Appendix B, row 3*. - -The one exception is `getcredentials`, which additionally requires the client to -answer a callback on its own TCP port 300. - ---- - -## Feature: DNS answers for a node and for the cluster - -Source: `xCAT-server/lib/xcat/plugins/ddns.pm` (`"$name IN A $ip"` at 1758, -`"$_ IN CNAME $name"` at 1758ff, `"$rname IN PTR $name"` at 1766, reverse zone -name built at 1663-1669, `site.domain` required at 369-377, forwarders at -672-676) - -```gherkin -@P-01 -Scenario: The node's own name resolves forward - Given a node defined with an address on a managed network - And makedns has been run - When a resolver queries the node's short name in the cluster domain - Then an A record is returned - And it holds the address the node was defined with - -@P-02 -Scenario: The node's address resolves in reverse - Given a node defined with an address on a managed network - When a resolver queries the IN-ADDR.ARPA name for that address - Then a PTR record is returned - And it holds the node's fully qualified name - -@P-03 -Scenario: Reverse resolution is what xcatd will use to name the client - Given a PTR exists for the node's address - When the cluster domain is stripped from the name it returns - Then the result is the node's name - # Not a DNS scenario at heart: this is the identity xcatd will act on, so a - # PTR that resolves to something else is an authentication bug, not a - # cosmetic one. P-74 asserts the consequence. - -@P-04 -Scenario: Each configured alias resolves to the node - Given a node with one or more aliases - When a resolver queries an alias - Then a CNAME to the node's fully qualified name is returned - -@P-05 -Scenario: A name outside the cluster domain is forwarded - Given forwarders are configured - When a resolver queries a name in no local zone - Then the query is answered from a forwarder, not refused - # Skipped where the fixture's network has no route to a forwarder: an - # unreachable forwarder is the operator's problem, not xCAT's. - -@P-06 -Scenario: A name inside the cluster domain that has no record is refused locally - When a resolver queries an undefined short name in the cluster domain - Then NXDOMAIN is returned - And the answer is authoritative - -@P-07 -Scenario: The master's name resolves - When a resolver queries the name the node will be handed as its xCAT master - Then it resolves to the address on the node's own network - # The node has one route. A master that resolves to the management node's - # other address is a name that answers and an address that does not. - -@P-08 -Scenario: Removing a node removes both directions - Given a node that resolves forward and in reverse - When the node is removed and makedns is re-run - Then neither the A record nor the PTR is returned -``` - ---- - -## Feature: TFTP serves what DHCP named - -Source: `xCAT-server/lib/xcat/plugins/grub2.pm` (`boot/grub2/grub2.` at -179 and 194, `grub.cfg-` and `grub.cfg-01-` at 379-382), -`pxe.pm` (`pxelinux.cfg/` plus the HEXIP link, in `setstate`), -`xnba.pm:254` (`xcat/xnba/nodes/`), `petitboot.pm:149,196-203`, -`perl-xCAT/xCAT/DHCP/BootPolicy.pm:429,433` (s390x and OPAL conf-file URLs), -`xCAT-server/lib/xcat/plugins/AAsn.pm:1145` (`in.tftpd` is the daemon) - -```gherkin -@P-09 -Scenario: The architecture loader is present before any node is defined - Given the tftp root has been populated - When a client fetches the grub2 binary for the node's architecture - Then the transfer succeeds and the file is non-empty - -@P-10 -Scenario: A missing architecture loader is a hard error at nodeset time - Given the grub2 binary for the node's architecture is absent - When nodeset is run for the node - Then nodeset reports an error - And no per-node configuration file is written # [config] - # The wire half of this is P-70: whatever DHCP named must be fetchable. - -@P-11 -Scenario: The per-node grub2 config is fetchable by hex-IP name - Given a node set to install and using the grub2 netboot method - When a client fetches boot/grub2/grub.cfg- - Then the transfer succeeds - -@P-12 -Scenario: The per-node grub2 config is also fetchable by MAC name - Given the same node - When a client fetches boot/grub2/grub.cfg-01- - Then the transfer succeeds - And the content is byte-identical to the hex-IP name - # grub2 tries the MAC form first and the hex-IP form second. Two names, one - # file: a node whose MAC changed must not get a stale config. - -@P-13 -Scenario: The grub2 config names a kernel that is itself fetchable - When a client fetches the per-node grub2 config - And extracts the kernel path from the linux line - Then that path is fetchable over the transport the config names - -@P-14 -Scenario: The grub2 config names an initrd that is itself fetchable - When a client fetches the per-node grub2 config - And extracts the initrd path from the initrd line - Then that path is fetchable over the transport the config names - -@P-15 -Scenario: The grub2 config carries the node's MAC in BOOTIF - When a client fetches the per-node grub2 config - Then the linux line contains BOOTIF set to the node's MAC - # The installer picks its interface from BOOTIF. A node with two NICs that - # is given the wrong one installs onto the wrong network. - -@P-16 -Scenario: The kernel command line carries the xCAT master and port - When a client fetches the per-node config for a node in a boot state - Then the kernel command line contains xcatd=: - -@P-17 -Scenario: The kernel command line carries the destiny the node was set to - When a client fetches the per-node config - Then the kernel command line contains destiny= - -@P-18 -Scenario: A pxelinux node gets a config under its node name - Given a node whose netboot method is pxe - When a client fetches pxelinux.cfg/ - Then the transfer succeeds - And the content begins with a DEFAULT stanza - -@P-19 -Scenario: The pxelinux config is reachable by hex-IP as well - When a client fetches pxelinux.cfg/ - Then the transfer succeeds - And the content is identical to the by-name fetch - -@P-20 -Scenario: A node set to boot from local disk is told to do so - Given a node set to boot - When a client fetches its boot config - Then it directs the client to the local disk - And it names no kernel - # The DHCP half of this is S-31: a node that has finished installing must - # stop being handed a boot file at all. This half asserts the other - # direction -- that if a config is served, it does not netboot. - -@P-21 -Scenario: An xnba node gets a gpxe script - Given a node whose netboot method is xnba - When a client fetches xcat/xnba/nodes/ - Then the first line is the gpxe shebang - -@P-22 -Scenario: The xnba script fetches its kernel over HTTP, not TFTP - When a client fetches the xnba script - Then it contains an imgfetch whose URL scheme is http - And the URL host is the address DHCP gave as next-server - # xnba.pm:339,347 - -@P-23 -Scenario: A petitboot node gets a config hardlinked under its hex IP - Given a node whose netboot method is petitboot - When a client fetches at the tftp root - Then the transfer succeeds - And it is byte-identical to petitboot/ - -@P-24 -Scenario: Fetching a path outside the tftp root fails - When a client fetches a path containing parent-directory components - Then the transfer is refused -``` - ---- - -## Feature: The discovery artefacts are per network, not per node - -Source: `xCAT-server/lib/xcat/plugins/mknb.pm` (`xcat/xnba/nets/` at 826, -`pxelinux.cfg/` and the `p/` and `s390x/` forms at 918, -`boot/grub2/grub.cfg-` and `set fallback=1` at 1018) - -A machine nobody has defined has no node name to look a file up by. Everything -in this section is keyed on the network it booted on, which is the only thing -the server knows about it before discovery. - -```gherkin -@P-25 -Scenario: A discovery config exists per network, not per node - Given mknb has been run for the management node architecture - When a client fetches boot/grub2/grub.cfg- for the managed network - Then the transfer succeeds - -@P-26 -Scenario: The discovery config falls back from HTTP to TFTP - When a client fetches the grub2 discovery config - Then it contains two menu entries naming the same kernel - And the first uses http and the second does not - And fallback is set to the second - -@P-27 -Scenario: The discovery kernel command line asks for the discover destiny - When a client fetches the grub2 discovery config - Then its kernel command line contains destiny=discover - -@P-28 -Scenario: The discovery kernel command line names the xCAT master - When a client fetches the grub2 discovery config - Then its kernel command line contains xcatd=: - -@P-29 -Scenario: The pxelinux discovery config is reachable by hex network address - When a client fetches pxelinux.cfg/ - Then the transfer succeeds - -@P-30 -Scenario: The genesis kernel named by the discovery config is fetchable - When the kernel path is extracted from the discovery config - Then it is fetchable over TFTP - -@P-31 -Scenario: The genesis initrd named by the discovery config is fetchable - When the initrd path is extracted from the discovery config - Then it is fetchable over TFTP - # The initrd is tens of megabytes and is the artefact most likely to be - # half-written by an interrupted mknb. A HEAD is not enough; fetch it. -``` - ---- - -## Feature: HTTP serves the install tree and the boot tree - -Source: `xCAT/xcat.conf` and `xCAT/xcat.conf.apach24` (`AliasMatch -^/install/(.*)`, `^/tftpboot/(.*)`), `site.httpport` read at -`anaconda.pm:268`, `debian.pm:1236-1239`, `xnba.pm:164`; `grub2.pm:267-268` -(`set root=http,$serverip:$httpport`); autoinst served from -`/install/autoinst/` (`anaconda.pm:1477`, `debian.pm:771,1260`) - -```gherkin -@P-32 -Scenario: The install tree is served on the configured port - Given a site http port - When a client requests a known path under /install on that port - Then the response status is 200 - -@P-33 -Scenario: The tftp tree is also served over HTTP - When a client requests a known path under /tftpboot over HTTP - Then the response status is 200 - And the body is byte-identical to the same file fetched over TFTP - # Two daemons, one tree. grub2-http and xnba fetch the kernel over HTTP - # having been told its name by a TFTP fetch, so a divergence here is a node - # that boots a different kernel than the one it was told about. - -@P-34 -Scenario: The autoinst file for a node is served - Given a node set to install - When a client requests /install/autoinst/ - Then the response status is 200 - -@P-35 -Scenario: The autoinst URL in the kernel command line is the one that is served - When a client fetches the node's boot config - And extracts the kickstart or preseed URL from the kernel command line - Then a request to that exact URL returns 200 - # Not the same assertion as P-34. P-34 says the file is where this document - # says it is; P-35 says the node was told where it is. - -@P-36 -Scenario: The repository URL in the kernel command line is served - When the install repository URL is extracted from the kernel command line - Then a request for the repository metadata under it returns 200 - -@P-37 -Scenario: A path outside the served aliases is not reachable - When a client requests a path under neither /install nor /tftpboot - Then the response status is not 200 - -@P-38 -Scenario: Directory listing is available where postscripts live - When a client requests the postscripts directory - Then a listing is returned - -@P-39 -Scenario: The HTTP port the node is told about is the port that answers - When the port is extracted from the node's boot config - Then a request to that port succeeds -``` - ---- - -## Feature: Flow control on UDP 3001 - -Source: `xCAT-server/sbin/xcatd:647-672` (the requestor table and -`resourcerequest: ok`), `xcatd:877-880` (`ackresourcerequest`), -`xCAT-genesis-scripts/usr/bin/udpcat.awk` - -A large discovery has hundreds of machines asking for the same slots. The -protocol is two datagrams: an acknowledgement that the request was heard, and -later a grant. - -```gherkin -@P-40 -Scenario: A flow-control request is acknowledged immediately - When a client sends "resourcerequest: xcatd" to UDP 3001 - Then an acknowledgement datagram is returned - -@P-41 -Scenario: A flow-control request is eventually granted - When a client sends a resource request and waits - Then a grant datagram is returned within the timeout - # An acknowledgement without a grant is exactly the failure a node cannot - # diagnose: it waits forever and reports nothing. -``` - ---- - -## Feature: findme, and the callback it produces - -Source: `xcatd:708-711` (the command, the source port below 1000, and -`nodeonmynet`), `xcatd:861-876` (the `processing` callback, sent on receipt), -`xCAT-server/lib/xcat/plugins/zzzdiscovery.pm:34-41` (the `processed` -callback), `xCAT-genesis-scripts/usr/bin/dodiscovery`, -`xCAT-genesis-scripts/usr/bin/udpcat.awk` (`/inet/udp/301/`) - -Read the gates carefully: they are not where the issue that proposed these -scenarios placed them. The `processing` callback is sent by the UDP listener as -soon as a datagram arrives that starts with the gzip magic or with ` because we either have no discovery - # plugins or the client address does not match an IP network that xCAT is - # managing" -- xcatd:721 - -@P-44 -Scenario: A well-formed findme produces a callback on the client's TCP 3001 - Given a client listening on TCP 3001 - When a findme packet is sent from a privileged source port on a managed network - Then the server connects back to the client on TCP 3001 - -@P-45 -Scenario: The callback says the request is being processed - When the callback connection is accepted - Then the first message on it is the processing token - -@P-46 -Scenario: A findme that no discovery method claims ends in a failure callback - Given no discovery method is configured to match the client - When a findme is sent from a privileged source port on a managed network - Then a second callback carrying the processed token is received - # zzzdiscovery runs last and exists to say "nobody claimed this". A node - # that gets no such callback cannot tell failure from slowness. - -@P-47 -Scenario: A findme declaring a virtual node type is not claimed by switch or sequential discovery - When a findme carrying a virtual node type is sent - Then no node is claimed by those methods # [config] - -@P-48 -Scenario: Both the gzipped and the plain XML findme encodings are accepted - When the same findme payload is sent gzipped, and again as plain XML - Then both produce a callback - # xcatd:861 tests for the RFC 1952 magic and xcatd:869 for a "300` element) - -```gherkin -@P-49 -Scenario: A client with no certificate can open the TLS port - When a client connects to 3001 with no client certificate - Then the handshake completes - # If this ever stops being true, every scenario below it is untestable and - # genesis stops booting. It is asserted first for that reason. - -@P-50 -Scenario: An unknown client is told to discover itself - Given a client whose address has no reverse mapping to a defined node - When it sends a getdestiny request - Then the response destiny is discover - -@P-51 -Scenario: A known node is told the destiny it was set to - Given a node set to install, and a client owning that node's address and PTR - When it sends getdestiny - Then the response destiny matches the state nodeset was given - -@P-52 -Scenario: An install destiny carries a kernel and an initrd - When a node set to install sends getdestiny - Then the response contains both a kernel and an initrd element - -@P-53 -Scenario: An install destiny carries a kernel command line - When a node set to install sends getdestiny - Then the response contains a kcmdline element - And it names the xCAT master and the destiny - -@P-54 -Scenario: The image server in the response answers - When a node set to install sends getdestiny - Then the response names an image server - And a request to that address on the xCAT port is answered - -@P-55 -Scenario: The image server falls back through the documented chain - Given the node has neither a tftpserver nor an xcatmaster attribute - When it sends getdestiny - Then the image server in the response is the site master - # destiny.pm:975-987 tries noderes.tftpserver, then noderes.xcatmaster, - # then the network's tftpserver, then site.master. Four sources, no error - # if the wrong one wins -- the node boots and then fetches from an address - # that does not answer. - -@P-56 -Scenario: nextdestiny advances the chain - Given a node with more than one state in its chain - When the client sends nextdestiny and then getdestiny - Then the second destiny differs from the first - -@P-57 -Scenario: getpostscript returns a script terminated by the end marker - When a known client sends getpostscript - Then the response body ends with the end-of-script marker - # The client reads until the marker. A truncated script with no marker is - # read as a hang, not as an error. - -@P-58 -Scenario: getpostscript from an unknown client does not return another node's script - Given a client whose address maps to no node - When it sends getpostscript - Then no node-specific script is returned - -@P-59 -Scenario: A command outside the default policy is refused to a certless client - When a certless client sends a command that has no policy row granting it - Then the response is a refusal, not a result - -@P-60 -Scenario: A malformed request does not kill the listener - When a client sends bytes that are not well-formed XML - And a second client then sends a valid getdestiny - Then the second client is answered - -@P-61 -Scenario: getcredentials requires the node callback to agree - Given a client with no listener on the credential callback port - When it requests credentials - Then no signed certificate is returned - -@P-62 -Scenario: getcredentials succeeds when the node callback agrees - Given a client listening on the credential callback port - And that listener answers the challenge affirmatively - When it requests credentials - Then a signed certificate is returned - # credentials.pm:611 sends "CREDOKBYYOU?\n" and requires "CREDOKBYME". - # The callback is what stops an address that merely has a PTR from - # collecting a signed certificate. - -@P-63 -Scenario: The credential callback is made to the port the protocol specifies - When a client requests credentials naming a callback port - Then the server's callback connection arrives on that port - # credentials.pm:130-136 -``` - ---- - -## Feature: The install monitor on TCP 3002 - -Source: `xcatd:330-400` (the listener, and `site.xcatiport`), `xcatd:404-520` -(the `ready` / `done` framing and the verbs), -`xCAT-server/lib/xcat/plugins/xcatdsklspost:1058-1078` -(`updateflag.awk $MASTER 3002 "installstatus ..."`) - -```gherkin -@P-64 -Scenario: The install monitor port answers with a readiness token - When a client connects to TCP 3002 - Then a readiness token is received before any request is sent - -@P-65 -Scenario: An install status report is accepted and terminated - When a known client sends an install status line - Then a completion token is received - -@P-66 -Scenario: A report from a client that maps to no node is not accepted - Given a client whose address has no reverse mapping - When it connects to the monitor port - Then it is given no readiness token - # xcatd closes the connection on a peer it cannot name, so the absence of - # a greeting is the observable. It is not a transport error and provtest - # records it as a result. - -@P-67 -Scenario: getpostscript over the monitor port returns the same body as over TLS - When the same node requests its postscript on 3002 and on 3001 - Then the two bodies are identical - -@P-68 -Scenario: An unknown verb on the monitor port is rejected without hanging - When a client sends a verb the service does not implement - Then the connection is closed or an error returned within the timeout - -@P-69 -Scenario: The monitor port is plain text, not TLS - When a client sends a TLS client hello to 3002 - Then no TLS handshake completes -``` - ---- - -## Feature: Failing at exactly one place - -Each of these is the interesting case: every stage before it is correct, so the -node gets far enough to fail visibly at one identifiable point. They exist -because the whole argument for a wire suite is that these failures are currently -indistinguishable from each other -- all of them look like "the node timed out". - -```gherkin -@P-70 -Scenario: Correct DHCP with a missing boot file is visible as a TFTP failure - Given an acknowledgement naming a boot file - When a client fetches that exact file name over TFTP - And the file was never written - Then the fetch returns file-not-found - -@P-71 -Scenario: A boot file present but naming an absent kernel fails one fetch later - Given the per-node config fetches successfully - When the kernel path it names is fetched - Then the failure is at the kernel fetch, not at the config fetch - -@P-72 -Scenario: A boot config naming an unreachable master is detectable without booting - When the master address is extracted from the kernel command line - And a TLS connection is attempted to it on the xCAT port - Then a connection failure is distinguishable from a protocol failure - -@P-73 -Scenario: A node whose findme is unanswered receives only the failure callback - When a findme is sent and no discovery method claims it - Then the client's listener records the processing token - And then the processed token - And nothing else - -@P-74 -Scenario: A missing PTR turns a known node into an unknown client - Given a node set to install whose reverse record has been removed - When it sends getdestiny - Then the response destiny is discover, not the state it was set to - # The whole authentication story in one scenario. A PTR that makedns did - # not write costs a node its identity and nothing anywhere reports it. - -@P-75 -Scenario: nodeset for a new state leaves no trace of the previous one - Given a node set to install, then set to boot - When the per-node config is fetched - Then it reflects the second state only -``` - ---- - -## Notes on testing this specification - -- Scenarios not marked **[config]** are wire-observable and belong in - `xCAT-test/provtest/conf/`. They need real daemons answering, which - `xCAT-test/autotest/testcase/provtest/provfixture.sh` builds out of a veth - pair so a single-node management node can run them. -- Scenarios marked **[config]** assert on database rows, generated files or - plugin dispatch and belong in `xCAT-test/unit/`, which needs neither root nor - a network. -- **A `.conf` file must never encode xCAT policy.** The hex-IP file name, the - kernel path, the master address, the destiny string, the HTTP port: every - expected value comes in on the command line via `--set`, and the fixture - supplies it from the node and network it defined. A test that asks xCAT what - it wrote and then checks that xCAT wrote it proves nothing. -- **No scenario may be conditional on a netboot method.** Where the artefact - names genuinely differ between grub2, pxelinux, xnba and petitboot, that - belongs in the `.conf` file as a parameter, not in the scenario as a branch. - The four TFTP groups are four files, chosen by whoever knows how the node - under test is configured; running all four against one node will always fail - three of them. -- The client end of the veth pair carries **the node's** address, not a spare - one. That is not a convenience: xcatd names a client by the reverse lookup of - the address it connected from, so the address is the credential and binding to - it is what makes P-50 and P-51 different scenarios. - -### What this specification does not cover on the wire - -Stated so that a green run is not read as more than it is: - -- **DHCP.** Stage 2 has its own document and its own suite. -- **IPv6.** Every scenario here is IPv4. -- **A real installer installing.** The suite asserts that the artefacts and the - answers are correct, not that a distribution installs from them. Postscripts - running, `updatenode` against a live node and console access are all out. -- **Service-node hierarchy.** Every scenario assumes one management node serving - directly. A service node boots exactly as a compute node does; what differs is - what it serves afterwards. -- **Which discovery plugin claimed a findme.** Only the callbacks are asserted. - `switch.pm`, `typemtms.pm`, `seqdiscovery.pm`, `blade.pm` and `hpblade.pm` are - reached through state changes that belong to the unit suite. - ---- - -## Appendix A: corrections to the proposal these scenarios came from - -VersatusHPC/xcat-internal#176 stated seventy-five scenarios from a reading of -the source. Three of them did not survive being read against it again. The -numbers are kept -- an issue that cites P-42 should still find P-42 -- and what -changed is recorded here rather than silently rewritten. - -| # | Scenario | As proposed | As specified | Why | -| --- | --- | --- | --- | --- | -| 1 | P-42 | "no callback connection is made to the client" | the processing callback still arrives; no second callback follows | `xcatd:863` sends `processing` from the UDP listener on the gzip magic alone. The source-port test is at `xcatd:708`, in the worker, after the callback has gone out | -| 2 | P-43 | "no callback connection is made" | no discovery is attempted | Same reason. `nodeonmynet` is checked at `xcatd:711`, also after the callback | -| 3 | P-47, and open question 3 | whether an unsigned findme is rejected was "not confirmed" | nothing verifies a signature | `xcatd:706-711` parses the XML and tests the command name, the source port and the network. There is no signature check on the path, so forging findme needs no key | -| 4 | P-66 | "no node's state is claimed to have changed" | no readiness token is given | The proposed Then clause is a database assertion, which this suite may not make. xcatd closes the connection on an unnameable peer, and the missing greeting is the wire-observable form of the same fact | -| 5 | P-72 | "the failure is a connection failure, not a protocol failure" | a connection failure is distinguishable from a protocol failure | The original asserts which failure occurs, on a machine where neither may. What is under test is that the two are told apart | - -## Appendix B: what the fixture deviates from, and why - -| Decision | The issue proposed | Here | Why | -| --- | --- | --- | --- | -| Network | reuse `dhcptest0`/`dhcptest1`, `10.99.0.0/24`, `dhcptest.cluster` | `provtest0`/`provtest1`, `10.99.1.0/24`, `provtest.cluster` | The two suites run back to back in the same CI job. Sharing constants means a leaked `site.dhcpinterfaces` or an undeleted node from the DHCP run silently changes what the provision run is testing. Adjacent networks cost nothing and cannot collide | -| Client address | the client end carries the node's address | unchanged | This is the credential. See *Notes* above | -| Daemons | stand up `named`, `httpd`, `in.tftpd`, `xcatd` | per stage, each skipped independently | A management node that cannot free port 53 -- `dnsmasq`, `systemd-resolved`, libvirt -- must still be able to run the TFTP, HTTP and xcatd stages. One unusable daemon skips its own scenarios and nothing else | -| Port proof | `ss` before running anything | unchanged, per stage | A suite that passes because nothing was listening is worse than one that fails | - -## Appendix C: open questions - -Carried from the issue, with what has since been settled. - -1. **Can `xcatd` run against a scratch database?** Still open, and still the - largest unknown. Until it is settled, the 3001 and 3002 scenarios mutate the - real cluster database, and the fixture saves and restores the node - definitions and the `site` rows it touches. The suite is labelled - `prov_wire` and is not part of `ci_test` for this reason. -2. **`getbootparams` and `getinstallpkgs`** do not exist in this tree. No - scenario depends on them. -3. **The findme signature.** Settled: there is no verification. *Appendix A, - row 3*. -4. **`nodestat` state transitions.** The mapping from an `installstatus` report - on 3002 to an observable `nodelist.status` value has not been traced. P-66 - is written conservatively because of it. -5. **`in.tftpd` on a non-standard port.** The fixture takes port 69 and restores - it, as `dhcpfixture.sh` does with 67. `tftpflags` (`Schema.pm:1303`) may - offer a cleaner path; it has not been tried. -6. **ONIE and HTTPClient boot URLs.** The HTTP boot paths above are confirmed - for grub2-http and xnba. ONIE's URL shape is not traced and no scenario - asserts it. -7. **Callback timing.** P-44 asserts a callback arrives, not how soon. - `dodiscovery` retries with a 180-second cap, and no bound on the server side - is confirmed. diff --git a/xCAT-test/provtest/src/provtest_lib/__pycache__/__init__.cpython-312.pyc b/xCAT-test/provtest/src/provtest_lib/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 000000000..6a1dac872 Binary files /dev/null and b/xCAT-test/provtest/src/provtest_lib/__pycache__/__init__.cpython-312.pyc differ diff --git a/xCAT-test/provtest/src/provtest_lib/__pycache__/assertions.cpython-312.pyc b/xCAT-test/provtest/src/provtest_lib/__pycache__/assertions.cpython-312.pyc new file mode 100644 index 000000000..6a5487c6b Binary files /dev/null and b/xCAT-test/provtest/src/provtest_lib/__pycache__/assertions.cpython-312.pyc differ diff --git a/xCAT-test/provtest/src/provtest_lib/__pycache__/cli.cpython-312.pyc b/xCAT-test/provtest/src/provtest_lib/__pycache__/cli.cpython-312.pyc new file mode 100644 index 000000000..00abce4bc Binary files /dev/null and b/xCAT-test/provtest/src/provtest_lib/__pycache__/cli.cpython-312.pyc differ diff --git a/xCAT-test/provtest/src/provtest_lib/__pycache__/config.cpython-312.pyc b/xCAT-test/provtest/src/provtest_lib/__pycache__/config.cpython-312.pyc new file mode 100644 index 000000000..28453db80 Binary files /dev/null and b/xCAT-test/provtest/src/provtest_lib/__pycache__/config.cpython-312.pyc differ diff --git a/xCAT-test/provtest/src/provtest_lib/__pycache__/dnsc.cpython-312.pyc b/xCAT-test/provtest/src/provtest_lib/__pycache__/dnsc.cpython-312.pyc new file mode 100644 index 000000000..5b60b1d4e Binary files /dev/null and b/xCAT-test/provtest/src/provtest_lib/__pycache__/dnsc.cpython-312.pyc differ diff --git a/xCAT-test/provtest/src/provtest_lib/__pycache__/errors.cpython-312.pyc b/xCAT-test/provtest/src/provtest_lib/__pycache__/errors.cpython-312.pyc new file mode 100644 index 000000000..3b754f606 Binary files /dev/null and b/xCAT-test/provtest/src/provtest_lib/__pycache__/errors.cpython-312.pyc differ diff --git a/xCAT-test/provtest/src/provtest_lib/__pycache__/httpc.cpython-312.pyc b/xCAT-test/provtest/src/provtest_lib/__pycache__/httpc.cpython-312.pyc new file mode 100644 index 000000000..a5e2b52f6 Binary files /dev/null and b/xCAT-test/provtest/src/provtest_lib/__pycache__/httpc.cpython-312.pyc differ diff --git a/xCAT-test/provtest/src/provtest_lib/__pycache__/machine.cpython-312.pyc b/xCAT-test/provtest/src/provtest_lib/__pycache__/machine.cpython-312.pyc new file mode 100644 index 000000000..6e9257d2b Binary files /dev/null and b/xCAT-test/provtest/src/provtest_lib/__pycache__/machine.cpython-312.pyc differ diff --git a/xCAT-test/provtest/src/provtest_lib/__pycache__/model.cpython-312.pyc b/xCAT-test/provtest/src/provtest_lib/__pycache__/model.cpython-312.pyc new file mode 100644 index 000000000..7fe299385 Binary files /dev/null and b/xCAT-test/provtest/src/provtest_lib/__pycache__/model.cpython-312.pyc differ diff --git a/xCAT-test/provtest/src/provtest_lib/__pycache__/netutil.cpython-312.pyc b/xCAT-test/provtest/src/provtest_lib/__pycache__/netutil.cpython-312.pyc new file mode 100644 index 000000000..345c5c9d5 Binary files /dev/null and b/xCAT-test/provtest/src/provtest_lib/__pycache__/netutil.cpython-312.pyc differ diff --git a/xCAT-test/provtest/src/provtest_lib/__pycache__/proc.cpython-312.pyc b/xCAT-test/provtest/src/provtest_lib/__pycache__/proc.cpython-312.pyc new file mode 100644 index 000000000..b9afc28a2 Binary files /dev/null and b/xCAT-test/provtest/src/provtest_lib/__pycache__/proc.cpython-312.pyc differ diff --git a/xCAT-test/provtest/src/provtest_lib/__pycache__/report.cpython-312.pyc b/xCAT-test/provtest/src/provtest_lib/__pycache__/report.cpython-312.pyc new file mode 100644 index 000000000..9fd98582f Binary files /dev/null and b/xCAT-test/provtest/src/provtest_lib/__pycache__/report.cpython-312.pyc differ diff --git a/xCAT-test/provtest/src/provtest_lib/__pycache__/subst.cpython-312.pyc b/xCAT-test/provtest/src/provtest_lib/__pycache__/subst.cpython-312.pyc new file mode 100644 index 000000000..085fed7b7 Binary files /dev/null and b/xCAT-test/provtest/src/provtest_lib/__pycache__/subst.cpython-312.pyc differ diff --git a/xCAT-test/provtest/src/provtest_lib/__pycache__/tftpc.cpython-312.pyc b/xCAT-test/provtest/src/provtest_lib/__pycache__/tftpc.cpython-312.pyc new file mode 100644 index 000000000..3bb66a0f7 Binary files /dev/null and b/xCAT-test/provtest/src/provtest_lib/__pycache__/tftpc.cpython-312.pyc differ diff --git a/xCAT-test/provtest/tests/__pycache__/context.cpython-312.pyc b/xCAT-test/provtest/tests/__pycache__/context.cpython-312.pyc new file mode 100644 index 000000000..e71c896b3 Binary files /dev/null and b/xCAT-test/provtest/tests/__pycache__/context.cpython-312.pyc differ diff --git a/xCAT-test/provtest/tests/__pycache__/test_assertions.cpython-312.pyc b/xCAT-test/provtest/tests/__pycache__/test_assertions.cpython-312.pyc new file mode 100644 index 000000000..d4f0ed5fe Binary files /dev/null and b/xCAT-test/provtest/tests/__pycache__/test_assertions.cpython-312.pyc differ diff --git a/xCAT-test/provtest/tests/__pycache__/test_boundaries.cpython-312.pyc b/xCAT-test/provtest/tests/__pycache__/test_boundaries.cpython-312.pyc new file mode 100644 index 000000000..677ee471c Binary files /dev/null and b/xCAT-test/provtest/tests/__pycache__/test_boundaries.cpython-312.pyc differ diff --git a/xCAT-test/provtest/tests/__pycache__/test_config.cpython-312.pyc b/xCAT-test/provtest/tests/__pycache__/test_config.cpython-312.pyc new file mode 100644 index 000000000..1e9903223 Binary files /dev/null and b/xCAT-test/provtest/tests/__pycache__/test_config.cpython-312.pyc differ diff --git a/xCAT-test/provtest/tests/__pycache__/test_machine.cpython-312.pyc b/xCAT-test/provtest/tests/__pycache__/test_machine.cpython-312.pyc new file mode 100644 index 000000000..74c2ab403 Binary files /dev/null and b/xCAT-test/provtest/tests/__pycache__/test_machine.cpython-312.pyc differ diff --git a/xCAT-test/provtest/tests/__pycache__/test_parsers.cpython-312.pyc b/xCAT-test/provtest/tests/__pycache__/test_parsers.cpython-312.pyc new file mode 100644 index 000000000..f46b0bc3f Binary files /dev/null and b/xCAT-test/provtest/tests/__pycache__/test_parsers.cpython-312.pyc differ diff --git a/xCAT-test/provtest/tests/__pycache__/test_report.cpython-312.pyc b/xCAT-test/provtest/tests/__pycache__/test_report.cpython-312.pyc new file mode 100644 index 000000000..b6ab35718 Binary files /dev/null and b/xCAT-test/provtest/tests/__pycache__/test_report.cpython-312.pyc differ diff --git a/xCAT-test/provtest/tests/__pycache__/test_subst.cpython-312.pyc b/xCAT-test/provtest/tests/__pycache__/test_subst.cpython-312.pyc new file mode 100644 index 000000000..53b7cd336 Binary files /dev/null and b/xCAT-test/provtest/tests/__pycache__/test_subst.cpython-312.pyc differ