test(provtest): assert the 75 provision clauses as wire scenarios

Sixteen files, fifty scenarios, one per behaviour the specification names. The cross-stage ones are the point: a config is fetched over TFTP, the kernel path is read out of it, and that path is fetched. No file says what the kernel will be called, which is what separates testing xCAT from comparing xCAT's output against a copy of it. The four tftp-*.conf files are alternatives, not a set -- a node has one netboot method -- and the fixture selects the one it was defined with. conf/tftp-grub2.conf @P-70 meets dhcptest at the only shared assertion: the boot file DHCP named is fetchable under exactly that name.
2026-10-07 10:06:39 +00:00 · 2026-09-11 07:55:59 -03:00
parent 7ea4bbdfb4
commit 1f06d22883
16 changed files with 1420 additions and 0 deletions
@@ -0,0 +1,110 @@
+# Stage 5: the genesis / discovery artefacts.  spec.md P-25 to P-31.
+#
+# A machine nobody has defined has no node name to look a file up by, so
+# everything here is keyed on the *network* it booted on -- the only thing the
+# server knows about it before discovery has happened.  That makes these files
+# the ones that are wrong the longest: a cluster whose nodes are all defined
+# never fetches them, and the day someone racks a new machine, nothing works and
+# nothing says why.
+#
+#   provtest run \
+#       --set server=10.99.1.1 --set client=10.99.1.11 \
+#       --set hexnet=0A630100 --set net=10.99.1.0 \
+#       --set master=10.99.1.1 --set xcatport=3001 \
+#       conf/discovery-artefacts.conf
+
+[defaults]
+server  = %(server)s
+bind    = %(client)s
+timeout = 10
+retries = 2
+
+# --- P-25, P-26, P-27, P-28 -------------------------------------------------
+
+[scenario grub2-discovery-config]
+description = The per-network grub2 discovery config exists and asks for discovery
+
+[step config]
+type = tftp
+path = boot/grub2/grub.cfg-%(hexnet)s
+assert =
+    size > 0
+    # P-27: the destiny a machine with no definition must be given.
+    text contains destiny=discover
+    # P-28: and where to ask for the next one.
+    text contains xcatd=%(master)s:%(xcatport)s
+
+[step httpentry]
+type    = extract
+from    = $config.text
+pattern = set root=(http)
+# P-26, first half: the fast path.  Loading a 60 MB initrd over TFTP takes
+# minutes per machine, which is why the HTTP entry exists and why it is first.
+assert =
+    matched == yes
+
+[step fallback]
+type    = extract
+from    = $config.text
+pattern = set fallback=(\d+)
+# P-26, second half: and the reason the slow path is still written.  A machine
+# whose firmware has no HTTP stack must still discover itself, so the entry
+# order and the fallback index are load-bearing, not decoration.
+assert =
+    matched == yes
+    value   == 1
+
+# --- P-29 -------------------------------------------------------------------
+
+[scenario pxelinux-discovery-config]
+description = The pxelinux discovery config answers under the hex network address
+
+[step pxeconfig]
+type = tftp
+path = pxelinux.cfg/%(hexnet)s
+assert =
+    size > 0
+    text contains destiny=discover
+
+# --- P-30, P-31 -------------------------------------------------------------
+
+[scenario genesis-kernel-and-initrd]
+description = The genesis kernel and initrd named by the discovery config are fetchable
+
+[step gconfig]
+type = tftp
+path = boot/grub2/grub.cfg-%(hexnet)s
+assert =
+    size > 0
+
+[step kernelpath]
+type    = extract
+from    = $gconfig.text
+pattern = ^\s*linux(?:efi|16)?\s+/?(?:tftpboot/)?(\S+)
+assert =
+    matched == yes
+
+[step kernel]
+type = tftp
+path = $kernelpath.value
+assert =
+    size > 0
+
+[step initrdpath]
+type    = extract
+from    = $gconfig.text
+pattern = ^\s*initrd(?:efi|16)?\s+/?(?:tftpboot/)?(\S+)
+assert =
+    matched == yes
+
+[step initrd]
+type    = tftp
+path    = $initrdpath.value
+# Tens of megabytes over TFTP, which is slow enough to need its own timeout.
+# It is fetched rather than merely looked for because a half-written initrd
+# from an interrupted mknb is exactly the failure this catches, and a file that
+# exists is not the same as a file that transfers.
+timeout = 120
+retries = 1
+assert =
+    size > 0
@@ -0,0 +1,39 @@
+# Stage 1: DNS.  spec.md P-08.
+#
+# Run this only *after* the node has been removed and makedns re-run.  It is a
+# separate file rather than a scenario in dns.conf because the two make
+# opposite assertions about the same names, and a suite that ran both against
+# one state would always fail one of them.
+#
+#   provtest run \
+#       --set server=10.99.1.1 --set client=10.99.1.11 \
+#       --set node=provtestcn --set domain=provtest.cluster \
+#       --set revname=11.1.99.10.in-addr.arpa \
+#       conf/dns-removal.conf
+
+[defaults]
+server  = %(server)s
+bind    = %(client)s
+timeout = 5
+
+[scenario removed-node]
+description = A withdrawn node resolves in neither direction
+
+[step forward]
+type   = dns
+name   = %(node)s.%(domain)s
+rrtype = A
+assert =
+    status == NXDOMAIN
+    count  == 0
+
+[step reverse]
+type   = dns
+name   = %(revname)s
+rrtype = PTR
+# A PTR left behind by a removal is the worse half of the two: the name is
+# gone, so nothing resolves the node forward, but the address still claims to
+# be it, and xcatd will act on that claim.
+assert =
+    status == NXDOMAIN
+    count  == 0
@@ -0,0 +1,127 @@
+# Stage 1: DNS.  spec.md P-01 through P-08.
+#
+# Everything a node does after DHCP is done by name, and xcatd decides which
+# node it is talking to by resolving the client's address backwards.  So the
+# forward record is what the node needs and the reverse record is what the
+# server needs, and a cluster missing either one fails in a way that reports
+# nothing.
+#
+# Every expected value is supplied with --set.  This file states that a record
+# exists and holds a given value; it states nothing about how named was
+# configured, or by what.
+#
+#   provtest run \
+#       --set server=10.99.1.1 --set client=10.99.1.11 \
+#       --set node=provtestcn --set domain=provtest.cluster \
+#       --set nodeip=10.99.1.11 --set revname=11.1.99.10.in-addr.arpa \
+#       --set alias=provtestcn-eth0 --set master=10.99.1.1 \
+#       --set missing=nosuchnode --set forwarded=example.com \
+#       --set net=10.99.1.0/24 \
+#       conf/dns.conf
+
+[defaults]
+server  = %(server)s
+bind    = %(client)s
+timeout = 5
+retries = 2
+
+# --- P-01 -------------------------------------------------------------------
+
+[scenario node-forward]
+description = The node's own name resolves to the address it was defined with
+
+[step a]
+type   = dns
+name   = %(node)s.%(domain)s
+rrtype = A
+assert =
+    status == NOERROR
+    count  >= 1
+    type   == A
+    data   == %(nodeip)s
+
+# --- P-02, P-03 -------------------------------------------------------------
+
+[scenario node-reverse]
+description = The node's address resolves to its fully qualified name
+
+[step ptr]
+type   = dns
+name   = %(revname)s
+rrtype = PTR
+assert =
+    status == NOERROR
+    count  >= 1
+    type   == PTR
+    data   == %(node)s.%(domain)s
+    # P-03: the same record read as an identity.  xcatd strips the domain and
+    # the -eth/-ib/-myri suffix from this and looks the rest up in the node
+    # list, so a PTR naming anything else costs the node its identity.  P-74
+    # asserts the consequence at the xcatd end.
+    data   starts-with %(node)s.
+
+# --- P-04 -------------------------------------------------------------------
+
+[scenario node-alias]
+description = A configured alias is a CNAME to the node
+
+[step cname]
+type   = dns
+name   = %(alias)s.%(domain)s
+rrtype = A
+assert =
+    status == NOERROR
+    # dig follows the CNAME, so both records come back in one answer section.
+    type   == CNAME
+    data   == %(node)s.%(domain)s
+    data   == %(nodeip)s
+
+# --- P-05 -------------------------------------------------------------------
+
+[scenario forwarded-name]
+description = A name in no local zone is answered from a forwarder
+
+[step outside]
+type      = dns
+name      = %(forwarded)s
+rrtype    = A
+recursion = yes
+timeout   = 10
+assert =
+    status == NOERROR
+    count  >= 1
+
+# --- P-06 -------------------------------------------------------------------
+
+[scenario local-nxdomain]
+description = An undefined name in the cluster domain is refused, not forwarded
+
+[step undefined]
+type   = dns
+name   = %(missing)s.%(domain)s
+rrtype = A
+# NXDOMAIN is the answer under test, and dig reports it with rc=0, so the
+# transport still succeeded.  `expect` stays at its default.
+assert =
+    status    == NXDOMAIN
+    count     == 0
+    # aa: the refusal came from the zone itself rather than from a forwarder
+    # that happened not to know the name either.
+    flags     contains aa
+    authority >= 1
+
+# --- P-07 -------------------------------------------------------------------
+
+[scenario master-resolves]
+description = The name the node is handed as its master resolves to the master
+
+[step master]
+type   = dns
+name   = %(master)s
+rrtype = A
+assert =
+    status == NOERROR
+    count  >= 1
+    # The node has one route.  A master resolving to the management node's
+    # other address is a name that answers and an address that does not.
+    data   in %(net)s
@@ -0,0 +1,104 @@
+# Stage 7: findme.  spec.md P-42 to P-46, P-48, P-73.
+#
+# Read the gates before reading the scenarios, because they are not where they
+# look.  xcatd's UDP listener sends `processing` back to the client's TCP 3001
+# as soon as a datagram arrives that begins with the gzip magic or with `<xcat`
+# (xcatd:861-876).  The source-port and managed-network checks happen later, in
+# the discovery worker (xcatd:708-711), and what they gate is the plugin
+# dispatch -- so an ignored findme still produces the first callback and never
+# produces a second.  That asymmetry is the whole reason these scenarios assert
+# on the number of callbacks rather than on their presence.
+#
+# Nothing here signs the packet.  Nothing in xcatd checks a signature.
+#
+#   provtest run \
+#       --set server=10.99.1.1 --set client=10.99.1.11 \
+#       --set foreign=10.98.1.11 \
+#       conf/findme.conf
+
+[defaults]
+server  = %(server)s
+bind    = %(client)s
+timeout = 10
+retries = 1
+
+# --- P-44, P-45, P-46, P-73 -------------------------------------------------
+
+[scenario findme-callbacks]
+description = A well-formed findme is acknowledged and then resolved
+
+[step discover]
+type            = findme
+source_port     = 301
+encoding        = gzip
+callback_listen = 3001
+# zzzdiscovery runs last and exists to say "nobody claimed this".  The second
+# callback can take as long as the slowest discovery plugin, which is the ARP
+# table lookup in seqdiscovery.
+callback_wait   = 20
+assert =
+    # P-44, P-45
+    count       >= 1
+    callbacks.0 == processing
+    # P-46, P-73: the failure callback is what lets a node tell "not yet" from
+    # "never".  Without it, dodiscovery retries for 180 seconds and the console
+    # says nothing either way.
+    count       == 2
+    callbacks.1 == processed
+
+# --- P-48 -------------------------------------------------------------------
+
+[scenario findme-plain-xml]
+description = The uncompressed encoding is accepted as well as the gzipped one
+
+[step plain]
+type            = findme
+encoding        = plain
+source_port     = 301
+callback_listen = 3001
+callback_wait   = 20
+# xcatd tests for the RFC 1952 magic and then for a `<xcat` prefix.  Anything
+# matching neither falls through to the flow-control branch and is discarded in
+# silence, so an encoding that stopped being recognised would look exactly like
+# a network that dropped the packet.
+assert =
+    count       >= 1
+    callbacks.0 == processing
+
+# --- P-42 -------------------------------------------------------------------
+
+[scenario findme-unprivileged-port]
+description = A findme from an unprivileged port is acknowledged and then dropped
+
+[step unprivileged]
+type            = findme
+source_port     = 32768
+encoding        = gzip
+callback_listen = 3001
+callback_wait   = 20
+assert =
+    # The acknowledgement is sent before the port is looked at, so it arrives.
+    count       == 1
+    callbacks.0 == processing
+    # And nothing follows it.  The node is told its request is being handled
+    # and then never hears again: this scenario pins that behaviour so a change
+    # to it is visible, not because it is a good one.
+
+# --- P-43 -------------------------------------------------------------------
+
+[scenario findme-foreign-address]
+description = A findme from an address on no managed network is not dispatched
+
+[step foreign]
+type            = findme
+bind            = %(foreign)s
+source_port     = 301
+encoding        = gzip
+callback_listen = 3001
+callback_wait   = 20
+# "xcatd: Skipping discovery from <ip> because ... the client address does not
+# match an IP network that xCAT is managing" -- xcatd:721.  The acknowledgement
+# still goes out, to the foreign address, which is why the listener is bound
+# there too.
+assert =
+    count == 1
@@ -0,0 +1,49 @@
+# Stage 6: flow control on UDP 3001.  spec.md P-40, P-41.
+#
+# A large discovery has hundreds of machines asking for the same few xcatd
+# slots at once, so genesis asks before it connects: `xcatflowrequest` sends
+# `resourcerequest: xcatd` and loops until a grant comes back.  Two datagrams,
+# and they are not the same datagram -- the first says the request was heard,
+# the second says there is room.  A server that sends the first and never the
+# second leaves every node in the cluster waiting silently.
+#
+#   provtest run --set server=10.99.1.1 --set client=10.99.1.11 \
+#       conf/flowcontrol.conf
+
+[defaults]
+server  = %(server)s
+bind    = %(client)s
+timeout = 10
+retries = 2
+
+# --- P-40 -------------------------------------------------------------------
+
+[scenario acknowledged]
+description = A resource request is acknowledged immediately
+
+[step request]
+type    = flowrequest
+message = resourcerequest: xcatd
+replies = 1
+assert =
+    count    >= 1
+    replies  == ackresourcerequest
+
+# --- P-41 -------------------------------------------------------------------
+
+[scenario granted]
+description = A resource request is granted, not merely acknowledged
+
+[step grant]
+type    = flowrequest
+message = resourcerequest: xcatd
+replies = 2
+# The grant is sent on the next pass of the requestor table rather than in
+# reply to the datagram, so it arrives seconds later on an idle server and
+# much later on a busy one.  A single retry, a long window: a retransmit here
+# would put a second entry in the table and be answered twice.
+timeout = 30
+retries = 1
+assert =
+    count   >= 2
+    replies == resourcerequest: ok
@@ -0,0 +1,167 @@
+# Stage 4: HTTP.  spec.md P-32 to P-39, and P-35/P-36 for the cross-stage half.
+#
+# Two Apache aliases carry the whole of a provision: /install for the repository
+# and the autoinst file, /tftpboot for the kernel and initrd that grub2-http and
+# xnba fetch over HTTP rather than TFTP.  Both trees are also served by another
+# daemon over another protocol, which is where the interesting failure lives --
+# the node is told a name over TFTP and fetches it over HTTP, so the two views
+# of one tree have to agree.
+#
+#   provtest run \
+#       --set server=10.99.1.1 --set client=10.99.1.11 \
+#       --set node=provtestcn --set httpport=80 --set otherport=8080 \
+#       --set hexip=0A63010B --set knownfile=boot/grub2/grub.cfg-0A63010B \
+#       --set installpath=autoinst/provtestcn --set repofile=repodata/repomd.xml \
+#       conf/http.conf
+
+[defaults]
+server  = %(server)s
+bind    = %(client)s
+port    = %(httpport)s
+timeout = 10
+retries = 2
+
+# --- P-32, P-34 -------------------------------------------------------------
+
+[scenario install-tree]
+description = The install tree is served on the configured port
+
+[step autoinst]
+type = http
+path = /install/%(installpath)s
+assert =
+    status == 200
+    size   > 0
+
+# --- P-33 -------------------------------------------------------------------
+
+[scenario tftp-tree-over-http]
+description = The tftp tree is served over HTTP, byte for byte
+
+[step overtftp]
+type    = tftp
+path    = %(knownfile)s
+port    = 69
+assert =
+    size > 0
+
+[step overhttp]
+type = http
+path = /tftpboot/%(knownfile)s
+# One tree, two daemons.  grub2-http is told a file name by a TFTP fetch and
+# then fetches it over HTTP; a divergence here is a node booting a different
+# kernel from the one it was told about, and neither daemon logs anything odd.
+assert =
+    status == 200
+    sha256 == $overtftp.sha256
+
+# --- P-35, P-36 -------------------------------------------------------------
+
+[scenario urls-the-node-was-given]
+description = The autoinst and repository URLs in the kernel command line are served
+
+[step urlconfig]
+type = tftp
+path = %(knownfile)s
+port = 69
+assert =
+    size > 0
+
+[step autoinsturl]
+type    = extract
+from    = $urlconfig.text
+# anaconda writes inst.ks=, debian writes url=/preseed/url=; one pattern for
+# the three because what is under test is the URL, not which installer it is
+# addressed to.
+pattern = (?:inst\.ks|\bks|preseed/url|\burl)=(https?://\S+)
+assert =
+    matched == yes
+
+[step autoinsturlfetch]
+type = http
+url  = $autoinsturl.value
+# Not the same assertion as the install-tree scenario above.  That one says the
+# file is where this document says it is; this one says the node was told where
+# it is.  A cluster can pass either one and fail the other, and only the second
+# failure stops an install.
+assert =
+    status == 200
+    size   > 0
+
+[step repourl]
+type    = extract
+from    = $urlconfig.text
+pattern = (?:inst\.repo|method|mirror/http/hostname)=(https?://\S+)
+assert =
+    matched == yes
+
+[step repo]
+type = http
+url  = $repourl.value/%(repofile)s
+assert =
+    status == 200
+
+# --- P-37 -------------------------------------------------------------------
+
+[scenario outside-the-aliases]
+description = A path under neither alias is not served
+
+[step outside]
+type   = http
+path   = /etc/xcat/cfgloc
+expect = any
+# `expect = any`: a 404 is a transport that worked perfectly and returned the
+# right answer, so the assertion is on the status, not on the fetch.
+assert =
+    status != 200
+
+# --- P-38 -------------------------------------------------------------------
+
+[scenario postscripts-listing]
+description = The postscripts directory is listable
+
+[step listing]
+type = http
+path = /install/postscripts/
+# xcatdsklspost fetches postscripts by name, but an operator debugging a failed
+# one needs to see what is there; xcat.conf turns Indexes on for this tree
+# deliberately.
+assert =
+    status == 200
+    size   > 0
+
+# --- P-39 -------------------------------------------------------------------
+
+[scenario port-the-node-was-told]
+description = The port named in the node's boot config is the port that answers
+
+[step portconfig]
+type = tftp
+path = %(knownfile)s
+port = 69
+assert =
+    size > 0
+
+[step configport]
+type    = extract
+from    = $portconfig.text
+pattern = set root=http,[^:\s]+:(\d+)
+# site.httpport is read by four plugins that each build their own URL from it.
+# Extracting the port the node was actually given, rather than asserting the
+# one this file was told, is what makes this a test of xCAT and not of --set.
+assert =
+    matched == yes
+
+[step served]
+type = http
+url  = http://%(server)s:$configport.value/tftpboot/%(knownfile)s
+assert =
+    status == 200
+
+[step wrongport]
+type   = http
+url    = http://%(server)s:%(otherport)s/tftpboot/%(knownfile)s
+expect = fail
+# The control: if every port answered, the step above would prove nothing.
+assert =
+    status == 0
@@ -0,0 +1,90 @@
+# Stage 13: the install monitor on TCP 3002.  spec.md P-64 to P-66, P-68, P-69.
+#
+# The installed system reports its progress here with `updateflag.awk`, over a
+# plain socket with no TLS and no certificate of any kind.  The only thing that
+# decides whose report it is, is the reverse lookup of the peer address: xcatd
+# resolves it, strips the domain, and looks the result up in the node list
+# (xcatd:428-470).  A peer it cannot name has its connection closed with no
+# greeting at all, which is what makes the greeting itself an assertable thing.
+#
+#   provtest run \
+#       --set server=10.99.1.1 --set client=10.99.1.11 \
+#       --set unknown=10.99.1.201 --set monitorport=3002 --set xcatport=3001 \
+#       conf/monitor.conf
+
+[defaults]
+server  = %(server)s
+port    = %(monitorport)s
+bind    = %(client)s
+timeout = 20
+retries = 2
+
+# --- P-64, P-65 -------------------------------------------------------------
+
+[scenario monitor-accepts-status]
+description = A known node is greeted and its status report is acknowledged
+
+[step report]
+type = monitor
+send =
+    installstatus installing
+assert =
+    # P-64: the greeting arrives before anything is sent.
+    greeting == ready
+    # P-65: and each line is acknowledged.  xcatd answers `done` before it
+    # acts, so this says the line was read, not that the state changed --
+    # which is a database assertion and belongs to the unit suite.
+    lines.0  == done
+
+# --- P-66 -------------------------------------------------------------------
+
+[scenario monitor-unknown-client]
+description = A client that maps to no node is not greeted at all
+
+[step unknownreport]
+type   = monitor
+bind   = %(unknown)s
+send =
+    installstatus installing
+expect = fail
+# The connection is accepted and then closed in silence.  From the node's side
+# this is indistinguishable from a server that is up and ignoring it, which is
+# exactly why a missing PTR is such an expensive mistake -- see P-74.
+assert =
+    greeting absent
+    closed   == yes
+
+# --- P-68 -------------------------------------------------------------------
+
+[scenario monitor-unknown-verb]
+description = A verb the service does not implement does not hang the client
+
+[step nonsense]
+type = monitor
+send =
+    thisisnotaverb
+# The service answers `done` to anything and then falls through its chain of
+# verbs without matching one.  What must not happen is the connection being
+# held open until the client's own timeout: xcatd's alarm(2) closes it.
+timeout = 10
+assert =
+    greeting == ready
+    lines.0  == done
+    closed   == yes
+
+# --- P-69 -------------------------------------------------------------------
+
+[scenario monitor-is-not-tls]
+description = A TLS client hello gets no handshake on the monitor port
+
+[step handshake]
+type    = xcatreq
+port    = %(monitorport)s
+command = lsxcatd
+expect  = fail
+# Sent as an xcatreq deliberately: that step type opens TLS, and this port does
+# not speak it.  Worth asserting rather than assuming, because it is the reason
+# every scenario above can be sent in the clear -- and the reason nothing on
+# this port may be trusted with anything a certificate would protect.
+assert =
+    handshake == no
@@ -0,0 +1,93 @@
+# Failing at exactly one place.  spec.md P-72, P-74, P-75.
+#
+# Every scenario in this file arranges for each stage before the interesting
+# one to be correct, so the run goes red at one identifiable point.  That is the
+# whole argument for a wire suite: today all of these failures look the same
+# from the outside -- the node times out -- and telling them apart means
+# watching a console during a reboot.
+#
+# P-74 needs the node's PTR removed first, and P-75 needs a second nodeset, so
+# the fixture drives them in that order and the scenarios are selected with -s
+# rather than all run at once.
+#
+#   provtest run \
+#       --set server=10.99.1.1 --set client=10.99.1.11 \
+#       --set node=provtestcn --set hexip=0A63010B \
+#       --set xcatport=3001 --set unreachable=10.99.1.250 \
+#       --set destiny=install --set second=boot \
+#       conf/ordering.conf
+
+[defaults]
+server  = %(server)s
+bind    = %(client)s
+timeout = 15
+retries = 1
+
+# --- P-72 -------------------------------------------------------------------
+
+[scenario unreachable-master]
+description = A master that does not answer is distinguishable from one that refuses
+
+[step reachable]
+type    = xcatreq
+port    = %(xcatport)s
+command = lsxcatd
+element =
+    arg = -v
+assert =
+    handshake == yes
+
+[step unreachable]
+type    = xcatreq
+server  = %(unreachable)s
+port    = %(xcatport)s
+command = lsxcatd
+expect  = fail
+# The two steps differ in one thing: the address.  A node whose kernel command
+# line names an address nothing answers on boots perfectly and then stops, and
+# from the console that looks identical to a protocol fault at a server that
+# is up.  Here the two are told apart without booting anything.
+assert =
+    handshake == no
+    error     present
+
+# --- P-74 -------------------------------------------------------------------
+# Run this only after the node's PTR has been removed.
+
+[scenario missing-ptr]
+description = A node whose reverse record is gone is treated as an unknown client
+
+[step getdestiny]
+type    = xcatreq
+port    = %(xcatport)s
+command = getdestiny
+# Nothing about the node changed: it is still defined, still set to install,
+# still on the same address.  One DNS record decides whether xcatd recognises
+# it, and its absence is reported nowhere -- the node is simply told to
+# discover itself, forever.
+assert =
+    destiny == discover
+    destiny != %(destiny)s
+
+# --- P-75 -------------------------------------------------------------------
+# Run this only after a second nodeset has set the node to another state.
+
+[scenario state-replaced]
+description = A second nodeset leaves no trace of the first state
+
+[step config]
+type = tftp
+path = boot/grub2/grub.cfg-%(hexip)s
+assert =
+    size > 0
+    text contains destiny=%(second)s
+
+[step stale]
+type    = extract
+from    = $config.text
+pattern = destiny=%(destiny)s
+# The previous state's artefacts are the ones nobody looks for.  A config
+# rewritten in place is fine; a config left behind under another of its names
+# is a node that boots last week's decision.
+assert =
+    matched == no
@@ -0,0 +1,155 @@
+# Stage 3: TFTP, grub2.  spec.md P-09, P-11 to P-17, P-24, P-70, P-71.
+#
+# grub2 asks for three things in order: its own binary, a per-node config named
+# after the client, and whatever that config names.  Each fetch is by exact
+# name, and the names are the thing under test -- a hex encoding that is wrong
+# by one digit produces no error anywhere, just a node that sits at the loader
+# until someone reboots it.
+#
+# The cross-stage steps are the point of the file: the config is fetched, the
+# kernel path is read out of it, and *that* path is fetched.  Nothing in this
+# file says what the kernel will be called, which is what stops it from
+# asserting xCAT's output against a copy of xCAT's output.
+#
+#   provtest run \
+#       --set server=10.99.1.1 --set client=10.99.1.11 \
+#       --set node=provtestcn --set hexip=0A63010B \
+#       --set macdashes=52-54-00-dc-11-01 --set mac=52:54:00:dc:11:01 \
+#       --set loader=boot/grub2/grub2.x86_64 \
+#       --set master=10.99.1.1 --set xcatport=3001 --set destiny=install \
+#       --set bootfile=boot/grub2/grub2.x86_64 \
+#       conf/tftp-grub2.conf
+
+[defaults]
+server  = %(server)s
+bind    = %(client)s
+timeout = 8
+retries = 2
+
+# --- P-09 -------------------------------------------------------------------
+
+[scenario grub2-binary]
+description = The architecture's grub2 binary is present and not empty
+
+[step loader]
+type = tftp
+path = %(loader)s
+assert =
+    # Not merely "the fetch succeeded": tftp-hpa creates the local file before
+    # it knows whether the server will answer, so a zero-length result is what
+    # a missing file looks like.  Size is the assertion that means something.
+    size  > 0
+    error absent
+
+# --- P-11, P-15, P-16, P-17 -------------------------------------------------
+
+[scenario grub2-node-config]
+description = The per-node config is fetchable by hex-IP name and names the node's boot
+
+[step config]
+type = tftp
+path = boot/grub2/grub.cfg-%(hexip)s
+assert =
+    size > 0
+    # P-15: the installer picks its interface from BOOTIF.  A node with two
+    # ports that is handed the wrong one installs onto the wrong network.
+    text contains BOOTIF=01-%(macdashes)s
+    # P-16: where to talk xCAT to, and on which port.
+    text contains xcatd=%(master)s:%(xcatport)s
+    # P-17: what nodeset was told to make it do.
+    text contains destiny=%(destiny)s
+
+# --- P-12 -------------------------------------------------------------------
+
+[scenario grub2-config-by-mac]
+description = The MAC-named config is the same file as the hex-IP one
+
+[step byhex]
+type = tftp
+path = boot/grub2/grub.cfg-%(hexip)s
+assert =
+    size > 0
+
+[step bymac]
+type = tftp
+path = boot/grub2/grub.cfg-01-%(macdashes)s
+# grub2 tries the MAC form first and the hex-IP form second.  Two names for one
+# file: a node whose MAC changed must not be served a stale config under the
+# other name.
+assert =
+    size   > 0
+    sha256 == $byhex.sha256
+
+# --- P-13, P-14, P-71 -------------------------------------------------------
+
+[scenario grub2-kernel-and-initrd]
+description = The kernel and initrd the config names are themselves fetchable
+
+[step kconfig]
+type = tftp
+path = boot/grub2/grub.cfg-%(hexip)s
+assert =
+    size > 0
+
+[step kernelpath]
+type    = extract
+from    = $kconfig.text
+# The path as grub2 wrote it, with the tftp root's prefix removed if it is
+# there: the same file is reachable as /tftpboot/x over HTTP and as x over
+# TFTP, and which form the config uses depends on whether root= is http.
+pattern = ^\s*linux(?:efi|16)?\s+/?(?:tftpboot/)?(\S+)
+assert =
+    matched == yes
+
+[step kernel]
+type = tftp
+path = $kernelpath.value
+# P-71: if this is where the run goes red, the failure is at the kernel fetch
+# and not at the config fetch, and the log says which file was missing.
+assert =
+    size  > 0
+    error absent
+
+[step initrdpath]
+type    = extract
+from    = $kconfig.text
+pattern = ^\s*initrd(?:efi|16)?\s+/?(?:tftpboot/)?(\S+)
+assert =
+    matched == yes
+
+[step initrd]
+type = tftp
+path = $initrdpath.value
+assert =
+    size > 0
+
+# --- P-24 -------------------------------------------------------------------
+
+[scenario tftp-escape]
+description = A path climbing out of the tftp root is refused
+
+[step escape]
+type   = tftp
+path   = ../../../../etc/passwd
+expect = fail
+# in.tftpd refuses this; the assertion is that it stays refused.  A server that
+# served it would be handing the contents of the management node to anything
+# that can reach port 69, which on a provisioning network is everything.
+assert =
+    size == 0
+
+# --- P-70 -------------------------------------------------------------------
+
+[scenario dhcp-named-file-must-exist]
+description = The boot file DHCP names is fetchable under exactly that name
+
+[step named]
+type = tftp
+# Supplied by the caller from the DHCP acknowledgement it observed, not from
+# anything this file knows.  dhcptest asserts that the right name was sent;
+# this asserts that the name sent can be fetched.  Neither one alone is enough
+# to boot a node.
+path = %(bootfile)s
+assert =
+    size  > 0
+    error absent
@@ -0,0 +1,37 @@
+# Stage 3: TFTP, petitboot.  spec.md P-23.
+#
+# POWER firmware fetches a config named by the uppercase hex of its own
+# address, from the tftp root rather than from a subdirectory.  xCAT writes
+# petitboot/<node> and hardlinks the hex name to it, so the two names are one
+# file and must stay one file -- a copy that drifts is a node that boots what it
+# was set to three nodesets ago.
+#
+#   provtest run \
+#       --set server=10.99.1.1 --set client=10.99.1.11 \
+#       --set node=provtestcn --set hexip=0A63010B \
+#       --set master=10.99.1.1 --set xcatport=3001 --set destiny=install \
+#       conf/tftp-petitboot.conf
+
+[defaults]
+server  = %(server)s
+bind    = %(client)s
+timeout = 8
+retries = 2
+
+[scenario petitboot-config]
+description = The petitboot config answers under the node name and under the hex IP
+
+[step byname]
+type = tftp
+path = petitboot/%(node)s
+assert =
+    size > 0
+    text contains xcatd=%(master)s:%(xcatport)s
+    text contains destiny=%(destiny)s
+
+[step byhex]
+type = tftp
+path = %(hexip)s
+assert =
+    size   > 0
+    sha256 == $byname.sha256
@@ -0,0 +1,70 @@
+# Stage 3: TFTP, pxelinux.  spec.md P-18, P-19, P-20.
+#
+# pxelinux looks its config up under a series of names and takes the first that
+# answers: the MAC form, then progressively shorter hex-IP prefixes, then
+# `default`.  xCAT writes the node-named file and links the hex-IP name to it,
+# so both must answer and both must answer the same thing.
+#
+# A separate file from tftp-grub2.conf, not a branch inside it: a node has one
+# netboot method, and running both sets against one node would always fail one.
+#
+#   provtest run \
+#       --set server=10.99.1.1 --set client=10.99.1.11 \
+#       --set node=provtestcn --set hexip=0A63010B \
+#       --set master=10.99.1.1 --set xcatport=3001 --set destiny=install \
+#       --set bootnode=provtestbn \
+#       conf/tftp-pxelinux.conf
+
+[defaults]
+server  = %(server)s
+bind    = %(client)s
+timeout = 8
+retries = 2
+
+# --- P-18, P-19 -------------------------------------------------------------
+
+[scenario pxelinux-node-config]
+description = The pxelinux config answers under the node name and under the hex IP
+
+[step byname]
+type = tftp
+path = pxelinux.cfg/%(node)s
+assert =
+    size > 0
+    text contains DEFAULT
+    text contains xcatd=%(master)s:%(xcatport)s
+    text contains destiny=%(destiny)s
+
+[step byhex]
+type = tftp
+path = pxelinux.cfg/%(hexip)s
+# The hex-IP name is the one the firmware actually asks for; the node name is
+# the one an operator reads.  They are the same file, and a link that was not
+# made is a node that falls through to `default` and discovers itself instead
+# of installing.
+assert =
+    size   > 0
+    sha256 == $byname.sha256
+
+# --- P-20 -------------------------------------------------------------------
+
+[scenario pxelinux-boot-from-disk]
+description = A node set to boot is told to use its disk and is given no kernel
+
+[step config]
+type = tftp
+path = pxelinux.cfg/%(bootnode)s
+assert =
+    size > 0
+    text contains LOCALBOOT
+
+[step kernelline]
+type    = extract
+from    = $config.text
+pattern = ^\s*KERNEL\s+\S
+# The DHCP half of this is dhcptest S-31: an installed node must stop being
+# handed a boot file at all.  This half says that if a config is served anyway,
+# it does not netboot.  Written as an extract that must find nothing, because
+# "contains" has no negative form and an absent line is not an absent field.
+assert =
+    matched == no
@@ -0,0 +1,53 @@
+# Stage 3/4: TFTP then HTTP, xnba.  spec.md P-21, P-22.
+#
+# xnba is the one netboot method that crosses transports on its own: the gpxe
+# script arrives over TFTP and then fetches the kernel over HTTP.  So a script
+# that is correct and a web server that is not produce a node that gets all the
+# way to `imgfetch` and stops.  Both halves are asserted here, in that order.
+#
+#   provtest run \
+#       --set server=10.99.1.1 --set client=10.99.1.11 \
+#       --set node=provtestcn --set nextserver=10.99.1.1 \
+#       conf/tftp-xnba.conf
+
+[defaults]
+server  = %(server)s
+bind    = %(client)s
+timeout = 8
+retries = 2
+
+# --- P-21, P-22 -------------------------------------------------------------
+
+[scenario xnba-script]
+description = The gpxe script fetches its kernel over HTTP from the next-server
+
+[step script]
+type = tftp
+path = xcat/xnba/nodes/%(node)s
+assert =
+    size          > 0
+    # gpxe refuses a script without this first line, and refuses it silently.
+    text starts-with #!gpxe
+    text contains imgfetch
+
+[step kernelurl]
+type    = extract
+from    = $script.text
+pattern = imgfetch\s+(?:-n\s+\S+\s+)?(\S+)
+assert =
+    matched == yes
+    # The host in the URL is the address DHCP handed out as next-server.  xnba
+    # builds it from the node's own attributes, so a node in a hierarchical
+    # cluster that is pointed at the management node instead of its service
+    # node fetches across a link that may not carry it.
+    value starts-with http://%(nextserver)s
+
+[step kernel]
+type = http
+url  = $kernelurl.value
+# The assertion that matters: the URL the node was told to use answers.  Not a
+# URL this file built out of the same parts, which would only prove that two
+# copies of the same assumption agree.
+assert =
+    status == 200
+    size   > 0
@@ -0,0 +1,65 @@
+# Stage 11: getcredentials.  spec.md P-61, P-62, P-63.
+#
+# This is the one request a forged client cannot simply ask for.  Before xcatd
+# signs anything it connects back to the client on TCP 300 and sends
+# `CREDOKBYYOU?`; it signs only if the client answers `CREDOKBYME`
+# (credentials.pm:611, callback port checked at credentials.pm:130-136).
+#
+# That callback is the only thing standing between "an address with a PTR" and
+# "a signed cluster certificate", so both halves are asserted: that it is made,
+# and that a client which cannot answer it gets nothing.
+#
+#   provtest run \
+#       --set server=10.99.1.1 --set client=10.99.1.11 \
+#       --set xcatport=3001 --set credtype=xcat \
+#       conf/xcatd-credentials.conf
+
+[defaults]
+server        = %(server)s
+port          = %(xcatport)s
+bind          = %(client)s
+callback_port = 300
+timeout       = 30
+retries       = 1
+
+# --- P-62, P-63 -------------------------------------------------------------
+
+[scenario credentials-granted]
+description = A client that answers the callback is given a signed certificate
+
+[step getcredentials]
+type            = xcatreq
+command         = getcredentials
+element =
+    credentials = %(credtype)s
+# The listener goes up on the client's port 300 before the request is sent,
+# because the server connects back while the request is still open: a listener
+# started afterwards would be started too late.
+callback_listen = 300
+callback_reply  = CREDOKBYME
+callback_wait   = 20
+assert =
+    # P-63: the callback arrived, and on the port the request named.
+    callback_seen == yes
+    callback_data contains CREDOKBYYOU
+    # P-62: and the answer carries something signed.
+    data          present
+    error         absent
+
+# --- P-61 -------------------------------------------------------------------
+
+[scenario credentials-refused]
+description = A client with no listener on the callback port is given nothing
+
+[step norefused]
+type    = xcatreq
+command = getcredentials
+element =
+    credentials = %(credtype)s
+# No callback_listen: the port is closed, the server's connection is refused,
+# and the challenge goes unanswered.  Anything signed arriving here would mean
+# an address alone is enough to collect a cluster certificate.
+expect  = any
+assert =
+    callback_seen == no
+    data          absent
@@ -0,0 +1,115 @@
+# Stage 9/10: getdestiny and nextdestiny over TLS 3001.  spec.md P-49 to P-56.
+#
+# This is the request a genesis image makes to find out what it is for.  The
+# genesis script pipes three lines of XML into `openssl s_client` with no client
+# certificate and reads the answer back with sed; provtest does the same over a
+# socket, from a chosen source address, because the source address is the
+# identity: xcatd names the client by the reverse lookup of the address the
+# connection arrived from.
+#
+# So `known` and `unknown` below differ in exactly one thing -- which address
+# the request left by -- and that is the whole authentication model on the wire.
+#
+#   provtest run \
+#       --set server=10.99.1.1 --set client=10.99.1.11 \
+#       --set unknown=10.99.1.201 --set node=provtestcn \
+#       --set destiny=install --set master=10.99.1.1 --set xcatport=3001 \
+#       conf/xcatd-destiny.conf
+
+[defaults]
+server        = %(server)s
+port          = %(xcatport)s
+bind          = %(client)s
+callback_port = 300
+timeout       = 20
+retries       = 2
+
+# --- P-49 -------------------------------------------------------------------
+
+[scenario certless-handshake]
+description = A client with no certificate can open the TLS port
+
+[step hello]
+type    = xcatreq
+command = lsxcatd
+element =
+    arg = -v
+# If this ever stops holding, every scenario below it is untestable and no
+# genesis image boots anywhere.  It is asserted first for that reason.
+assert =
+    handshake  == yes
+    serverdone == yes
+
+# --- P-50 -------------------------------------------------------------------
+
+[scenario unknown-client]
+description = A client whose address maps to no node is told to discover itself
+
+[step unknown]
+type    = xcatreq
+command = getdestiny
+bind    = %(unknown)s
+assert =
+    destiny == discover
+
+# --- P-51, P-52, P-53, P-54, P-55 -------------------------------------------
+
+[scenario known-node]
+description = A known node is told its state, and everything it needs to reach it
+
+[step known]
+type    = xcatreq
+command = getdestiny
+assert =
+    # P-51
+    destiny   == %(destiny)s
+    # P-52: an install destiny that names no kernel is a node that boots
+    # nothing and reports success.
+    kernel    present
+    initrd    present
+    # P-53
+    kcmdline  contains xcatd=%(master)s:%(xcatport)s
+    kcmdline  contains destiny=%(destiny)s
+    # P-55: destiny.pm:975-987 tries noderes.tftpserver, then
+    # noderes.xcatmaster, then the network's tftpserver, then site.master.
+    # Four sources, no error if the wrong one wins -- the node boots and then
+    # fetches from an address that does not answer.
+    imgserver == %(master)s
+
+[step imgserver]
+type    = xcatreq
+command = lsxcatd
+# P-54: the address in the answer is asked a question of its own.  Asserting
+# that the field holds the expected string proves the field; connecting to it
+# proves the node can get there from where it is.
+server  = $known.imgserver
+element =
+    arg = -v
+assert =
+    handshake == yes
+
+# --- P-56 -------------------------------------------------------------------
+
+[scenario chain-advances]
+description = nextdestiny moves the node on to the next state in its chain
+
+[step first]
+type    = xcatreq
+command = getdestiny
+assert =
+    destiny present
+
+[step advance]
+type    = xcatreq
+command = nextdestiny
+assert =
+    serverdone == yes
+
+[step second]
+type    = xcatreq
+command = getdestiny
+# The database effect belongs to the unit suite; what is observable here is
+# that two identical requests either side of a nextdestiny do not answer the
+# same.  A chain that does not advance is a node that reinstalls forever.
+assert =
+    destiny !=  $first.destiny
@@ -0,0 +1,60 @@
+# Stage 9: what a certless client may and may not ask for.  spec.md P-59, P-60.
+#
+# The default policy grants a certless client exactly the commands a booting
+# node needs: getdestiny, nextdestiny, getpostscript, getcredentials, lsxcatd,
+# syncfiles, litefile, litetree, getadapter, getbmcconfig, remoteimmsetup.  That
+# list is what makes this whole suite possible, and it is also the boundary --
+# anything outside it must be refused to a client that has shown no
+# certificate, or a machine on the provisioning network owns the cluster.
+#
+#   provtest run \
+#       --set server=10.99.1.1 --set client=10.99.1.11 \
+#       --set xcatport=3001 --set refused=rpower --set node=provtestcn \
+#       conf/xcatd-policy.conf
+
+[defaults]
+server  = %(server)s
+port    = %(xcatport)s
+bind    = %(client)s
+timeout = 20
+retries = 1
+
+# --- P-59 -------------------------------------------------------------------
+
+[scenario command-outside-policy]
+description = A command with no policy row is refused to a certless client
+
+[step refused]
+type    = xcatreq
+command = %(refused)s
+element =
+    noderange = %(node)s
+    arg       = stat
+expect  = any
+# A refusal is the passing result, so the transport expectation is waived and
+# the assertion is on what came back.  What must not come back is a result.
+assert =
+    handshake  == yes
+    error      present
+    serverdone == yes
+
+# --- P-60 -------------------------------------------------------------------
+
+[scenario malformed-request]
+description = Garbage on the wire does not take the listener down
+
+[step garbage]
+type    = xcatreq
+command = getdestiny
+# `raw` replaces the whole request body, so this is not XML at all.  A forked
+# child dying on it is fine; the parent accepting no further connections is
+# not, and the difference is only visible from a second client.
+raw     = this is not xml, and it is not meant to be
+expect  = any
+
+[step afterwards]
+type    = xcatreq
+command = getdestiny
+assert =
+    handshake == yes
+    destiny   present
@@ -0,0 +1,86 @@
+# Stage 12: getpostscript, on both transports.  spec.md P-57, P-58, P-67.
+#
+# The node fetches the script it is to run after the installer finishes, and it
+# fetches it over whichever transport it has: TLS 3001 during genesis,
+# plain 3002 from `xcatdsklspost` inside the installed system.  Two paths, one
+# script, and the client reads until `#END OF SCRIPT` -- so a body that is
+# truncated without the marker is read as a hang, not as an error.
+#
+#   provtest run \
+#       --set server=10.99.1.1 --set client=10.99.1.11 \
+#       --set unknown=10.99.1.201 --set node=provtestcn \
+#       --set xcatport=3001 --set monitorport=3002 \
+#       conf/xcatd-postscript.conf
+
+[defaults]
+server  = %(server)s
+bind    = %(client)s
+timeout = 30
+retries = 2
+
+# --- P-57 -------------------------------------------------------------------
+
+[scenario postscript-terminated]
+description = A known node's postscript arrives complete, with its end marker
+
+[step getpostscript]
+type    = xcatreq
+port    = %(xcatport)s
+command = getpostscript
+assert =
+    text contains #END OF SCRIPT
+    text contains %(node)s
+
+# --- P-58 -------------------------------------------------------------------
+
+[scenario postscript-unknown-client]
+description = A client that maps to no node is given no node's script
+
+[step leaked]
+type    = xcatreq
+port    = %(xcatport)s
+command = getpostscript
+bind    = %(unknown)s
+expect  = any
+# Whether xcatd answers this at all is a policy question; what must not happen
+# is that it answers with somebody else's script.  A postscript carries the
+# node's name, its master and the credentials it is to install.
+
+[step leak]
+type    = extract
+from    = $leaked.text
+pattern = %(node)s
+assert =
+    matched == no
+
+# --- P-67 -------------------------------------------------------------------
+
+[scenario postscript-both-transports]
+description = The same node gets the same script on 3001 and on 3002
+
+[step overtls]
+type    = xcatreq
+port    = %(xcatport)s
+command = getpostscript
+assert =
+    text contains #END OF SCRIPT
+
+[step marker]
+type    = extract
+from    = $overtls.text
+# A line from the body that is specific to this node and to this run, used
+# below as the thing the other transport has to agree about.  The two framings
+# differ -- XML elements on one side, bare lines on the other -- so a byte
+# comparison of the responses would compare the envelopes, not the script.
+pattern = ^\s*(NODE=\S+|MASTER=\S+)\s*$
+assert =
+    matched == yes
+
+[step overplain]
+type = monitor
+port = %(monitorport)s
+send =
+    getpostscript
+assert =
+    text contains #END OF SCRIPT
+    text contains $marker.value