diff --git a/github_action_xcat_test.pl b/github_action_xcat_test.pl index 848f672fd..a7f3348ee 100644 --- a/github_action_xcat_test.pl +++ b/github_action_xcat_test.pl @@ -503,81 +503,40 @@ sub run_bats_tests{ } #-------------------------------------------------------- -# Fuction name: run_dhcptest_unit_tests -# Description: Run the dhcptest offline checks: its Python unit tests and -# `dhcptest validate` over every shipped .conf. Both are +# Fuction name: run_blackboxtest_unit_tests +# Description: Run the blackboxtest offline checks: its Python unit tests and +# `blackboxtest validate` over every shipped .conf. Both are # unprivileged and touch no network. Skipped when python3 is # absent rather than failing the build. # Attributes: # Return code: 0 all checks passed, 1 otherwise #-------------------------------------------------------- -sub run_dhcptest_unit_tests{ - my $testdir = "$unitsrc/xCAT-test/dhcptest"; +sub run_blackboxtest_unit_tests{ + my $testdir = "$unitsrc/xCAT-test/blackboxtest"; if (! -d $testdir) { - print "[run_dhcptest_unit_tests] no dhcptest found under $testdir\n"; + print "[run_blackboxtest_unit_tests] no blackboxtest found under $testdir\n"; return 0; } runcmd("which python3 2>/dev/null"); if($::RUNCMD_RC){ - print "[run_dhcptest_unit_tests] python3 is not installed, skipping\n"; + print "[run_blackboxtest_unit_tests] python3 is not installed, skipping\n"; return 0; } my $cmd = "cd $testdir && python3 -m unittest discover -s tests" - . " && python3 src/dhcptest validate conf/*.conf"; - print "[run_dhcptest_unit_tests] running $cmd\n"; + . " && python3 src/blackboxtest validate conf/*/*.conf"; + print "[run_blackboxtest_unit_tests] running $cmd\n"; my @output = runcmd("$cmd"); print Dumper \@output; if($::RUNCMD_RC){ - print RED "[run_dhcptest_unit_tests] $cmd ....[Failed]\n"; - $check_result_str .= "> **DHCPTEST TESTS Failed** : Please click ``Details`` label in ``Merge pull request`` box for detailed information\n"; + print RED "[run_blackboxtest_unit_tests] $cmd ....[Failed]\n"; + $check_result_str .= "> **BLACKBOXTEST TESTS Failed** : Please click ``Details`` label in ``Merge pull request`` box for detailed information\n"; print $check_result_str; return 1; } - print "[run_dhcptest_unit_tests] $cmd ....[Pass]\n"; - $check_result_str .= "> **DHCPTEST TESTS Successful**\n"; - print $check_result_str; - return 0; -} - -#-------------------------------------------------------- -# Fuction name: run_provtest_unit_tests -# Description: Run the provtest offline checks: its Python unit tests and -# `provtest validate` over every shipped .conf. Both are -# unprivileged and touch no network. Skipped when python3 is -# absent rather than failing the build. -# Attributes: -# Return code: 0 all checks passed, 1 otherwise -#-------------------------------------------------------- -sub run_provtest_unit_tests{ - my $testdir = "$unitsrc/xCAT-test/provtest"; - if (! -d $testdir) { - print "[run_provtest_unit_tests] no provtest found under $testdir\n"; - return 0; - } - runcmd("which python3 2>/dev/null"); - if($::RUNCMD_RC){ - print "[run_provtest_unit_tests] python3 is not installed, skipping\n"; - return 0; - } - - # Discovered from inside tests/ because that is where their own context - # module puts the package on the path. - my $cmd = "cd $testdir/tests && python3 -m unittest discover -s . -p 'test_*.py'" - . " && cd $testdir && python3 src/provtest validate conf/*.conf"; - print "[run_provtest_unit_tests] running $cmd\n"; - my @output = runcmd("$cmd"); - print Dumper \@output; - if($::RUNCMD_RC){ - print RED "[run_provtest_unit_tests] $cmd ....[Failed]\n"; - $check_result_str .= "> **PROVTEST TESTS Failed** : Please click ``Details`` label in ``Merge pull request`` box for detailed information\n"; - print $check_result_str; - return 1; - } - - print "[run_provtest_unit_tests] $cmd ....[Pass]\n"; - $check_result_str .= "> **PROVTEST TESTS Successful**\n"; + print "[run_blackboxtest_unit_tests] $cmd ....[Pass]\n"; + $check_result_str .= "> **BLACKBOXTEST TESTS Successful**\n"; print $check_result_str; return 0; } @@ -759,7 +718,7 @@ sub run_dhcp_wire_test{ } my $conf_file = write_regression_conf("dhcp-wire.conf", 0); - my $fixture = "/opt/xcat/share/xcat/tools/autotest/testcase/dhcptest/dhcpfixture.sh"; + my $fixture = "/opt/xcat/share/xcat/tools/autotest/testcase/blackboxtest/dhcpfixture.sh"; unless(-f $fixture){ print RED "[run_dhcp_wire_test] $fixture is missing, so the wire cases cannot be run per backend\n"; $check_result_str .= "> **DHCP WIRE TEST Failed**: $fixture is not installed\n"; @@ -861,7 +820,7 @@ sub run_prov_wire_test{ return 0; } - my $fixture = "/opt/xcat/share/xcat/tools/autotest/testcase/provtest/provfixture.sh"; + my $fixture = "/opt/xcat/share/xcat/tools/autotest/testcase/blackboxtest/provfixture.sh"; unless(-f $fixture){ print RED "[run_prov_wire_test] $fixture is missing, so the wire cases cannot be run\n"; $check_result_str .= "> **PROVISION WIRE TEST Failed**: $fixture is not installed\n"; @@ -1078,15 +1037,15 @@ if($rst){ mark_time("run_fast_regression_test"); #The provisioning wire cases come next: after the ci_test set, and before the -#DHCP phases, which is the order a booting node meets them in. Their offline +#DHCP phase, which is the order a booting node meets them in. The offline #checks run first, so a scenario file that does not parse is reported as that. -print GREEN "\n------Running xCAT-test provtest checks ------\n"; -$rst = run_provtest_unit_tests(); +print GREEN "\n------Running xCAT-test blackboxtest checks ------\n"; +$rst = run_blackboxtest_unit_tests(); if($rst){ - print RED "Run of xCAT-test provtest checks failed\n"; + print RED "Run of xCAT-test blackboxtest checks failed\n"; exit $rst; } -mark_time("run_provtest_unit_tests"); +mark_time("run_blackboxtest_unit_tests"); print GREEN "\n------Running provisioning on-wire test ------\n"; $rst = run_prov_wire_test(); @@ -1096,16 +1055,6 @@ if($rst){ } mark_time("run_prov_wire_test"); -#The dhcptest offline checks run last: they exercise the source tree rather than -#the installed copy, and nothing else in the run depends on them. -print GREEN "\n------Running xCAT-test dhcptest checks ------\n"; -$rst = run_dhcptest_unit_tests(); -if($rst){ - print RED "Run of xCAT-test dhcptest checks failed\n"; - exit $rst; -} -mark_time("run_dhcptest_unit_tests"); - #The DHCP wire cases go last, once per backend installed. They are the only #phase that reconfigures the DHCP server and restarts the daemon, so nothing #following them could be trusted to run against the machine it was given. diff --git a/xCAT-test/README.md b/xCAT-test/README.md index 86e46917c..af2f031b1 100644 --- a/xCAT-test/README.md +++ b/xCAT-test/README.md @@ -7,7 +7,7 @@ language: | --------- | -------- | ------ | | Perl unit tests | `xCAT-test/unit/*.t` | `prove -r xCAT-test/unit` | | Shell unit tests | `xCAT-test/bats/*.bats` | `bats -r xCAT-test/bats` | -| DHCP wire tests | `xCAT-test/dhcptest/` | `python3 -m unittest discover -s tests`, then `dhcptest run -i ` | +| Black-box service tests | `xCAT-test/blackboxtest/` | `python3 -m unittest discover -s tests`, then `blackboxtest run ` on a management node | | CLI functional tests | `xCAT-test/autotest/testcase/` and `xCAT-test/autotest/bundle/` | `xcattest -f -t ` or `xcattest -f -b ` | Use Perl `.t` tests for Perl modules, Perl scripts, templates, and repository @@ -18,11 +18,11 @@ commands. Shell behavior should not be tested by Perl tests that grep shell source. Put those tests under `xCAT-test/bats` instead. -`dhcptest` is the odd one out: its Python unit tests and `dhcptest validate` -run offline like the others, but its point is the third column -- driving real -DHCP transactions on a provisioning NIC and asserting on what came back. It -talks to the server only over the wire, so it is agnostic to which DHCP -implementation is answering and to xCAT itself. +`blackboxtest` is the odd one out: its Python unit tests and `blackboxtest +validate` run offline like the others, but its point is the third column -- +asking a management node what a booting node asks (DHCP, DNS, TFTP, HTTP, +xcatd) and asserting on the answers. It talks to the services only as a client, +so it does not depend on which DHCP server answers or on the xCAT database. -See `unit/README.md`, `bats/README.md` and `dhcptest/README.md` for the +See `unit/README.md`, `bats/README.md` and `blackboxtest/README.md` for the detailed rules for each suite. diff --git a/xCAT-test/autotest/testcase/blackboxtest/cases0 b/xCAT-test/autotest/testcase/blackboxtest/cases0 new file mode 100644 index 000000000..f70815dde --- /dev/null +++ b/xCAT-test/autotest/testcase/blackboxtest/cases0 @@ -0,0 +1,20 @@ +# blackboxtest's offline checks: its unit tests, and every shipped scenario +# parsed and walked. No network, no root, no scapy. The wire cases that use +# the tool are in dhcp_cases and prov_cases. + +start:blackboxtest_unit_tests +description:blackboxtest's own unit tests: no network and no root needed +label:mn_only,ci_test,dhcp,prov +os:Linux +cmd:cd /opt/xcat/share/xcat/tools/autotest/blackboxtest && python3 -m unittest discover -s tests +check:rc==0 +end + +start:blackboxtest_validate_conf +description:Every shipped scenario parses, and every assertion names a field its step can carry +label:mn_only,ci_test,dhcp,prov +os:Linux +cmd:cd /opt/xcat/share/xcat/tools/autotest/blackboxtest && python3 src/blackboxtest validate conf/*/*.conf +check:rc==0 +check:output=~scenario +end diff --git a/xCAT-test/autotest/testcase/dhcptest/cases0 b/xCAT-test/autotest/testcase/blackboxtest/dhcp_cases similarity index 76% rename from xCAT-test/autotest/testcase/dhcptest/cases0 rename to xCAT-test/autotest/testcase/blackboxtest/dhcp_cases index 5baf1e293..0ee565848 100644 --- a/xCAT-test/autotest/testcase/dhcptest/cases0 +++ b/xCAT-test/autotest/testcase/blackboxtest/dhcp_cases @@ -1,11 +1,10 @@ -# dhcptest drives real DHCP transactions on the wire and asserts on the replies. -# It never reads the xCAT database and never runs an xCAT command, so it tests -# what a booting node would see rather than what the generated config says. +# The DHCP wire cases: blackboxtest drives real DHCP transactions and asserts on +# the replies. It never reads the xCAT database and never runs an xCAT command, +# so it tests what a booting node sees rather than what the generated config says. # -# The first two cases are offline and safe anywhere. The wire cases build their -# own provisioning network out of a veth pair via dhcpfixture.sh, which is where -# all the xCAT knowledge lives: it defines a network and a node and runs makedhcp -# so the configuration comes from xCAT rather than by hand. +# Each case builds its own provisioning network out of a veth pair via +# dhcpfixture.sh, which is where all the xCAT knowledge lives: it defines a +# network and a node and runs makedhcp, so the configuration comes from xCAT. # # Each wire case is one script, so its teardown runs from a trap and the machine # is left serving its own config even when an assertion fails. @@ -13,24 +12,7 @@ # A case that cannot run says so and passes. Read a pass as coverage only when # the log shows the ok lines. -start:dhcptest_unit_tests -description:dhcptest's own unit tests: no network, no root, no scapy needed -label:mn_only,ci_test,dhcp -os:Linux -cmd:cd /opt/xcat/share/xcat/tools/autotest/dhcptest && python3 -m unittest discover -s tests -check:rc==0 -end - -start:dhcptest_validate_conf -description:Every shipped dhcptest scenario parses and is a legal DHCP exchange -label:mn_only,ci_test,dhcp -os:Linux -cmd:cd /opt/xcat/share/xcat/tools/autotest/dhcptest && python3 src/dhcptest validate conf/*.conf -check:rc==0 -check:output=~scenario -end - -start:dhcptest_provision_vs_discovery +start:blackbox_dhcp_provision_vs_discovery description:A defined node gets its reservation and an unknown one gets a pool address label:mn_only,dhcp,dhcp_wire os:Linux @@ -49,7 +31,7 @@ os:Linux # site.dhcpbackend the surrounding pass selected. cmd:#!/bin/bash set -u - FIX=/opt/xcat/share/xcat/tools/autotest/testcase/dhcptest/dhcpfixture.sh + FIX=/opt/xcat/share/xcat/tools/autotest/testcase/blackboxtest/dhcpfixture.sh # 0 run, 1 this machine cannot run the wire cases and the case passes, 2 the # machine is dirty or is a live cluster -- which must fail, or one unfinished # teardown would make every later run here green without running anything. @@ -60,7 +42,7 @@ cmd:#!/bin/bash check:rc==0 end -start:dhcptest_boot_architectures +start:blackbox_dhcp_boot_architectures description:Each client architecture is handed the loader that belongs to it label:mn_only,dhcp,dhcp_wire os:Linux @@ -70,7 +52,7 @@ os:Linux # expected loader, and the fixture puts the loaders they key on in place first. cmd:#!/bin/bash set -u - FIX=/opt/xcat/share/xcat/tools/autotest/testcase/dhcptest/dhcpfixture.sh + FIX=/opt/xcat/share/xcat/tools/autotest/testcase/blackboxtest/dhcpfixture.sh # 0 run, 1 this machine cannot run the wire cases and the case passes, 2 the # machine is dirty or is a live cluster -- which must fail, or one unfinished # teardown would make every later run here green without running anything. @@ -81,7 +63,7 @@ cmd:#!/bin/bash check:rc==0 end -start:dhcptest_netboot_methods +start:blackbox_dhcp_netboot_methods description:A node is handed the loader its netboot method names, whatever its architecture says label:mn_only,dhcp,dhcp_wire os:Linux @@ -94,7 +76,7 @@ os:Linux # to beat whatever the subnet would otherwise have said. cmd:#!/bin/bash set -u - FIX=/opt/xcat/share/xcat/tools/autotest/testcase/dhcptest/dhcpfixture.sh + FIX=/opt/xcat/share/xcat/tools/autotest/testcase/blackboxtest/dhcpfixture.sh # 0 run, 1 this machine cannot run the wire cases and the case passes, 2 the # machine is dirty or is a live cluster -- which must fail, or one unfinished # teardown would make every later run here green without running anything. @@ -105,7 +87,7 @@ cmd:#!/bin/bash check:rc==0 end -start:dhcptest_node_identity +start:blackbox_dhcp_node_identity description:Which node an address, a name and a next-server belong to label:mn_only,dhcp,dhcp_wire os:Linux @@ -117,7 +99,7 @@ os:Linux # address. cmd:#!/bin/bash set -u - FIX=/opt/xcat/share/xcat/tools/autotest/testcase/dhcptest/dhcpfixture.sh + FIX=/opt/xcat/share/xcat/tools/autotest/testcase/blackboxtest/dhcpfixture.sh # 0 run, 1 this machine cannot run the wire cases and the case passes, 2 the # machine is dirty or is a live cluster -- which must fail, or one unfinished # teardown would make every later run here green without running anything. @@ -132,7 +114,7 @@ cmd:#!/bin/bash check:rc==0 end -start:dhcptest_boot_file_conditions +start:blackbox_dhcp_boot_file_conditions description:A loader is named only when it is fetchable, when it is wanted, and its URL names the right port label:mn_only,dhcp,dhcp_wire os:Linux @@ -145,7 +127,7 @@ os:Linux # not be named a netboot script, or it reinstalls itself on every power cycle. cmd:#!/bin/bash set -u - FIX=/opt/xcat/share/xcat/tools/autotest/testcase/dhcptest/dhcpfixture.sh + FIX=/opt/xcat/share/xcat/tools/autotest/testcase/blackboxtest/dhcpfixture.sh # 0 run, 1 this machine cannot run the wire cases and the case passes, 2 the # machine is dirty or is a live cluster -- which must fail, or one unfinished # teardown would make every later run here green without running anything. @@ -160,7 +142,7 @@ cmd:#!/bin/bash check:rc==0 end -start:dhcptest_legacy_and_storage_clients +start:blackbox_dhcp_legacy_and_storage_clients description:A BOOTP-only client, a CIDR dynamic range and a diskless node are all served label:mn_only,dhcp,dhcp_wire os:Linux @@ -170,7 +152,7 @@ os:Linux # on. Each is silent when it breaks -- the machine simply never finishes booting. cmd:#!/bin/bash set -u - FIX=/opt/xcat/share/xcat/tools/autotest/testcase/dhcptest/dhcpfixture.sh + FIX=/opt/xcat/share/xcat/tools/autotest/testcase/blackboxtest/dhcpfixture.sh # 0 run, 1 this machine cannot run the wire cases and the case passes, 2 the # machine is dirty or is a live cluster -- which must fail, or one unfinished # teardown would make every later run here green without running anything. @@ -185,7 +167,7 @@ cmd:#!/bin/bash check:rc==0 end -start:dhcptest_chainload_user_class +start:blackbox_dhcp_chainload_user_class description:A chainloaded second stage is recognised in both encodings of option 77 label:mn_only,dhcp,dhcp_wire os:Linux @@ -194,7 +176,7 @@ os:Linux # loader the first stage again and the machine chainloads itself forever. cmd:#!/bin/bash set -u - FIX=/opt/xcat/share/xcat/tools/autotest/testcase/dhcptest/dhcpfixture.sh + FIX=/opt/xcat/share/xcat/tools/autotest/testcase/blackboxtest/dhcpfixture.sh # 0 run, 1 this machine cannot run the wire cases and the case passes, 2 the # machine is dirty or is a live cluster -- which must fail, or one unfinished # teardown would make every later run here green without running anything. @@ -205,13 +187,13 @@ cmd:#!/bin/bash check:rc==0 end -start:dhcptest_lease_lifecycle +start:blackbox_dhcp_lease_lifecycle description:A lease can be taken, renewed and rebound, and a foreign address is refused label:mn_only,dhcp,dhcp_wire os:Linux cmd:#!/bin/bash set -u - FIX=/opt/xcat/share/xcat/tools/autotest/testcase/dhcptest/dhcpfixture.sh + FIX=/opt/xcat/share/xcat/tools/autotest/testcase/blackboxtest/dhcpfixture.sh # 0 run, 1 this machine cannot run the wire cases and the case passes, 2 the # machine is dirty or is a live cluster -- which must fail, or one unfinished # teardown would make every later run here green without running anything. @@ -222,7 +204,7 @@ cmd:#!/bin/bash check:rc==0 end -start:dhcptest_discovery_adoption +start:blackbox_dhcp_discovery_adoption description:A discovered machine is served its own address once defined, without restarting the daemon label:mn_only,dhcp,dhcp_wire os:Linux @@ -232,7 +214,7 @@ os:Linux # config-level test in this tree. cmd:#!/bin/bash set -u - FIX=/opt/xcat/share/xcat/tools/autotest/testcase/dhcptest/dhcpfixture.sh + FIX=/opt/xcat/share/xcat/tools/autotest/testcase/blackboxtest/dhcpfixture.sh # 0 run, 1 this machine cannot run the wire cases and the case passes, 2 the # machine is dirty or is a live cluster -- which must fail, or one unfinished # teardown would make every later run here green without running anything. @@ -243,13 +225,13 @@ cmd:#!/bin/bash check:rc==0 end -start:dhcptest_hierarchy_dhcpserver +start:blackbox_dhcp_hierarchy_dhcpserver description:A network whose dynamic pool was handed to another server stops answering unknown MACs label:mn_only,dhcp,dhcp_wire os:Linux cmd:#!/bin/bash set -u - FIX=/opt/xcat/share/xcat/tools/autotest/testcase/dhcptest/dhcpfixture.sh + FIX=/opt/xcat/share/xcat/tools/autotest/testcase/blackboxtest/dhcpfixture.sh # 0 run, 1 this machine cannot run the wire cases and the case passes, 2 the # machine is dirty or is a live cluster -- which must fail, or one unfinished # teardown would make every later run here green without running anything. @@ -262,15 +244,15 @@ cmd:#!/bin/bash check:rc==0 end -start:dhcptest_discover_offer +start:blackbox_dhcp_discover_offer description:A DISCOVER on an existing provisioning NIC draws exactly one OFFER from this MN label:mn_only,dhcp os:Linux # For a real lab management node that already has a provisioning network: point -# it at that NIC with DHCPTEST_NIC and it asserts against the live config, +# it at that NIC with BLACKBOXTEST_NIC and it asserts against the live config, # building and changing nothing. A plain environment variable rather than an # xcattest $$VAR, because an undefined $$VAR fails the case outright and this one # has to be able to stand down. -cmd:NIC=${DHCPTEST_NIC:-}; if [ -z "$NIC" ]; then echo "dhcptest skipped: set DHCPTEST_NIC to a provisioning NIC to run this"; exit 0; fi; if ! python3 -c "import scapy" 2>/dev/null; then echo "dhcptest skipped: python3-scapy is not installed"; exit 0; fi; CIDR=$(ip -4 -o addr show dev $NIC 2>/dev/null | awk '{print $4; exit}'); if [ -z "$CIDR" ]; then echo "dhcptest skipped: $NIC has no address, so there is no network to assert on"; exit 0; fi; MNIP=${CIDR%%/*}; NET=$(python3 -c "import ipaddress,sys; print(ipaddress.ip_interface(sys.argv[1]).network)" $CIDR); cd /opt/xcat/share/xcat/tools/autotest/dhcptest && python3 src/dhcptest run -i $NIC --set net=$NET --set server=$MNIP conf/discover-offer.conf +cmd:NIC=${BLACKBOXTEST_NIC:-}; if [ -z "$NIC" ]; then echo "skipped: set BLACKBOXTEST_NIC to a provisioning NIC to run this"; exit 0; fi; if ! python3 -c "import scapy" 2>/dev/null; then echo "skipped: python3-scapy is not installed"; exit 0; fi; CIDR=$(ip -4 -o addr show dev $NIC 2>/dev/null | awk '{print $4; exit}'); if [ -z "$CIDR" ]; then echo "skipped: $NIC has no address, so there is no network to assert on"; exit 0; fi; MNIP=${CIDR%%/*}; NET=$(python3 -c "import ipaddress,sys; print(ipaddress.ip_interface(sys.argv[1]).network)" $CIDR); cd /opt/xcat/share/xcat/tools/autotest/blackboxtest && python3 src/blackboxtest run -i $NIC --set net=$NET --set server=$MNIP conf/dhcp/discover-offer.conf check:rc==0 end diff --git a/xCAT-test/autotest/testcase/dhcptest/dhcpfixture.sh b/xCAT-test/autotest/testcase/blackboxtest/dhcpfixture.sh similarity index 93% rename from xCAT-test/autotest/testcase/dhcptest/dhcpfixture.sh rename to xCAT-test/autotest/testcase/blackboxtest/dhcpfixture.sh index cc2ec19c1..d3d49eae9 100755 --- a/xCAT-test/autotest/testcase/dhcptest/dhcpfixture.sh +++ b/xCAT-test/autotest/testcase/blackboxtest/dhcpfixture.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# The xCAT side of the dhcptest wire cases: dhcptest knows nothing about xCAT, +# The xCAT side of the DHCP wire cases: blackboxtest knows nothing about xCAT, # so everything that does lives here. It builds a provisioning network out of a # veth pair, the server end carrying the management address and the only # interface the daemon is told to listen on, the client end carrying nothing -- @@ -8,7 +8,7 @@ # Usage: # dhcpfixture.sh check is this machine able to run the wire cases # dhcpfixture.sh setup build the network, the node and the config -# dhcpfixture.sh run run dhcptest against it +# dhcpfixture.sh run run blackboxtest against it # dhcpfixture.sh run-arch one boot file per client architecture # dhcpfixture.sh run-netboot one boot file per node netboot method # dhcpfixture.sh run-lease the lease itself: handshake, renew, rebind, NAK @@ -26,7 +26,7 @@ # dhcpfixture.sh backend-setup select a backend for a whole pass of the cases # dhcpfixture.sh backend-teardown put the backend selection back # dhcpfixture.sh delegate hand the dynamic pool to another server -# dhcpfixture.sh run-hierarchy run dhcptest against the delegated network +# dhcpfixture.sh run-hierarchy run blackboxtest against the delegated network # dhcpfixture.sh run-adoption discover a machine, define it, serve it its own address # dhcpfixture.sh teardown put everything back # @@ -127,8 +127,8 @@ STATE=/tmp/dhcptest-fixture # What backend-setup saved, so backend-teardown can put it back. Separate from # $STATE, which belongs to one case and dies with that case's teardown. BSTATE=/tmp/dhcptest-backend -DHCPTEST=/opt/xcat/share/xcat/tools/autotest/dhcptest -[ -d "$DHCPTEST" ] || DHCPTEST="$(cd "$(dirname "$0")/../../../dhcptest" 2>/dev/null && pwd)" +TOOL=/opt/xcat/share/xcat/tools/autotest/blackboxtest +[ -d "$TOOL" ] || TOOL="$(cd "$(dirname "$0")/../../../blackboxtest" 2>/dev/null && pwd)" say() { echo "dhcpfixture: $*"; } die() { echo "dhcpfixture: $*" >&2; exit 1; } @@ -138,8 +138,8 @@ die() { echo "dhcpfixture: $*" >&2; exit 1; } # from a killed run, or a cluster with nodes of its own, are refusals -- cases0 # fails on those, because skipping them would make every later run on the same # machine green without running anything. -skip() { echo "dhcptest skipped: $*"; exit 1; } -refuse() { echo "dhcptest refused: $*" >&2; exit 2; } +skip() { echo "dhcpfixture skipped: $*"; exit 1; } +refuse() { echo "dhcpfixture refused: $*" >&2; exit 2; } # The netboot method decides the boot file a reserved node is handed. grub2 is # the one every backend renders the same way without needing a loader on disk, @@ -227,7 +227,7 @@ GATING_LOADERS="xcat/xnba.kpxe xcat/xnba.efi boot/grub2/grub2.riscv64" # Put an empty file where a gating loader is missing, recording it so teardown # takes back exactly what was added. Empty is enough because nothing fetches -# one: dhcptest asserts the name in the BOOTP file field and never opens a TFTP +# one: blackboxtest asserts the name in the BOOTP file field and never opens a TFTP # session. What matters is that both backends are asked the same question. provide_loaders() { local rel path tftp @@ -296,10 +296,10 @@ do_check() { [ "$(id -u)" = 0 ] || skip "the wire cases send raw frames, which needs root" command -v ip >/dev/null 2>&1 || skip "iproute2 is not installed" command -v makedhcp >/dev/null 2>&1 || skip "makedhcp is not on PATH, so this is not a management node" - # -f rather than -x: the fixture runs it as `python3 src/dhcptest`. Testing + # -f rather than -x: the fixture runs it as `python3 src/blackboxtest`. Testing # -x made every wire case skip -- and pass -- on Debian, where dh_install # keeps the source mode. - [ -f "$DHCPTEST/src/dhcptest" ] || skip "dhcptest is not installed under $DHCPTEST" + [ -f "$TOOL/src/blackboxtest" ] || skip "blackboxtest is not installed under $TOOL" python3 -c "import scapy" 2>/dev/null || skip "python3-scapy is not installed" ip link add "${IF_SRV}probe" type veth peer name "${IF_CLI}probe" 2>/dev/null \ || skip "this kernel has no veth support" @@ -471,34 +471,34 @@ do_backend_teardown() { say "===== the $want pass is over; $was is serving again =====" } -dhcptest_run() { - ( cd "$DHCPTEST" && python3 src/dhcptest run -i "$IF_CLI" "$@" ) +dhcp_run() { + ( cd "$TOOL" && python3 src/blackboxtest run -i "$IF_CLI" "$@" ) } do_run() { local loader rc=0 - dhcptest_run \ + dhcp_run \ --set node_mac="$NODE_MAC" --set node_ip="$NODE_IP" \ --set node_loader="$(node_loader)" --set pool="$POOL" \ --set next_server="$SRV_IP" --set unknown_mac="$UNKNOWN_MAC" \ --set gateway="$SRV_IP" --set nameservers="$SRV_IP" \ --set server="$SRV_IP" --set cumulus_url="$(cumulus_url)" \ --set domain="$DOMAIN" --set mtu="$MTU" --set lease="$(lease_time)" \ - conf/provision-vs-discovery.conf || rc=1 + conf/dhcp/provision-vs-discovery.conf || rc=1 # A reservation is a reservation whichever way the node was reached, so the # same node answers static-vs-dynamic.conf: the same question from the other # end, starting from the pool. - dhcptest_run \ + dhcp_run \ --set reserved_mac="$NODE_MAC" --set reserved_ip="$NODE_IP" \ --set unreserved_mac="$UNKNOWN_MAC" --set pool="$POOL" \ - conf/static-vs-dynamic.conf || rc=1 + conf/dhcp/static-vs-dynamic.conf || rc=1 loader=$(discovery_loader) - dhcptest_run \ + dhcp_run \ --set unknown_mac="$UNKNOWN_MAC" --set pool="$POOL" \ --set discovery_loader="$loader" \ - conf/discovery-bootfile.conf || rc=1 + conf/dhcp/discovery-bootfile.conf || rc=1 return $rc } @@ -512,7 +512,7 @@ do_run() { # The riscv64 HTTP scenario asserts a prefix and a substring rather than the # whole URL, so it reads the same riscv64_loader path the TFTP scenario does. do_run_arch() { - dhcptest_run --set tftp="$SRV_IP" \ + dhcp_run --set tftp="$SRV_IP" \ --set bios_loader="$(arch_loader bios)" \ --set uefi_loader="$(arch_loader uefi)" \ --set aarch64_loader="$(arch_loader aarch64)" \ @@ -523,7 +523,7 @@ do_run_arch() { --set opal_conf="$(opal_conf)" \ --set s390x_conf="$(s390x_conf)" \ --set onie_url="$(onie_url)" \ - conf/pxe-arch-matrix.conf + conf/dhcp/pxe-arch-matrix.conf } # One node per netboot method. @@ -627,7 +627,7 @@ do_run_netboot() { eval "${method}_node=\$name; ${method}_ip=\$ip; ${method}_mac=\$mac" done - dhcptest_run \ + dhcp_run \ --set xnba_mac="$xnba_mac" --set xnba_ip="$xnba_ip" \ --set xnba_node="$xnba_node" --set xnba_loader="$(netboot_loader xnba "$xnba_node")" \ --set xnba_stage2="$(xnba_node_url "$xnba_node")" \ @@ -643,7 +643,7 @@ do_run_netboot() { --set petitboot_mac="$petitboot_mac" \ --set petitboot_conf="$(netboot_loader petitboot "$petitboot_node")" \ --set noip_mac="$NOIP_MAC" \ - conf/netboot-methods.conf + conf/dhcp/netboot-methods.conf } # The two halves of a chained network boot. @@ -657,10 +657,10 @@ do_run_netboot() { # form RFC 3004 specifies. The same firmware sends either depending on how it # was built, so a server recognising only one boots half the fleet. do_run_chainload() { - dhcptest_run --set user_class=xNBA --set stage1_loader="$(arch_loader bios)" \ + dhcp_run --set user_class=xNBA --set stage1_loader="$(arch_loader bios)" \ --set stage1_uefi_loader="$(arch_loader uefi)" \ --set stage2_loader="$(xnba_net_url)" \ - conf/ipxe-userclass.conf + conf/dhcp/ipxe-userclass.conf } # The lease itself rather than what is booted with it: the handshake, renewal, @@ -669,9 +669,9 @@ do_run_chainload() { # the address it still holds, and silence leaves it retrying forever. do_run_lease() { local rc=0 - dhcptest_run --set net="$NET/$PREFIX" conf/full-lease.conf || rc=1 - dhcptest_run --set net="$NET/$PREFIX" conf/renew-rebind.conf || rc=1 - dhcptest_run --set foreign_ip="$FOREIGN_IP" conf/nak-foreign-address.conf || rc=1 + dhcp_run --set net="$NET/$PREFIX" conf/dhcp/full-lease.conf || rc=1 + dhcp_run --set net="$NET/$PREFIX" conf/dhcp/renew-rebind.conf || rc=1 + dhcp_run --set foreign_ip="$FOREIGN_IP" conf/dhcp/nak-foreign-address.conf || rc=1 return $rc } @@ -689,9 +689,9 @@ do_run_adoption() { daemon=$(daemon_of "$(current_backend)") [ -n "$daemon" ] || die "no DHCP daemon is installed" - dhcptest_run -s machine-is-unknown \ + dhcp_run -s machine-is-unknown \ --set adopt_mac="$ADOPT_MAC" --set adopt_ip="$ADOPT_IP" --set pool="$POOL" \ - conf/discovery-adoption.conf || rc=1 + conf/dhcp/discovery-adoption.conf || rc=1 before=$(pgrep -x "$daemon" | head -1) [ -n "$before" ] || die "$daemon is not running before the node is adopted" @@ -711,9 +711,9 @@ do_run_adoption() { say "$daemon kept running while $ADOPT_NODE was adopted (pid $before)" fi - dhcptest_run -s machine-has-been-adopted \ + dhcp_run -s machine-has-been-adopted \ --set adopt_mac="$ADOPT_MAC" --set adopt_ip="$ADOPT_IP" --set pool="$POOL" \ - conf/discovery-adoption.conf || rc=1 + conf/dhcp/discovery-adoption.conf || rc=1 return $rc } @@ -773,14 +773,14 @@ do_run_nextserver() { makedhcp "$NS_TFTP_NODE,$NS_XCM_NODE,$NS_SUB_NODE" \ || die "makedhcp failed for the next-server nodes" - dhcptest_run \ + dhcp_run \ --set tftp_mac="$NS_TFTP_MAC" --set tftp_ip="$NS_TFTP_IP" \ --set tftp_server="$SN_TFTP_IP" \ --set xcm_mac="$NS_XCM_MAC" --set xcm_ip="$NS_XCM_IP" \ --set xcm_server="$SN_XCAT_IP" \ --set sub_mac="$NS_SUB_MAC" --set sub_ip="$NS_SUB_IP" \ --set sub_server="$SRV_IP" \ - conf/next-server-source.conf || rc=1 + conf/dhcp/next-server-source.conf || rc=1 return $rc } @@ -795,10 +795,10 @@ do_run_multimac() { mac="$MM_MAC!$MM_NODE|$MM_MAC2!$MM_NODE2" makedhcp "$MM_NODE" || die "makedhcp $MM_NODE failed" - dhcptest_run \ + dhcp_run \ --set first_mac="$MM_MAC" --set first_ip="$MM_IP" \ --set second_mac="$MM_MAC2" --set second_ip="$MM_IP2" \ - conf/multi-mac-node.conf || rc=1 + conf/dhcp/multi-mac-node.conf || rc=1 return $rc } @@ -814,10 +814,10 @@ do_run_iscsi() { echo done > "$STATE/iscsi" makedhcp "$ISCSI_NODE" || die "makedhcp $ISCSI_NODE failed" - dhcptest_run \ + dhcp_run \ --set iscsi_mac="$ISCSI_MAC" --set iscsi_ip="$ISCSI_IP" \ --set root_path="iscsi:$SRV_IP:6:3260:0:$ISCSI_TARGET" \ - conf/iscsi-root-path.conf || rc=1 + conf/dhcp/iscsi-root-path.conf || rc=1 return $rc } @@ -839,11 +839,11 @@ do_run_localboot() { || die "cannot set chain.currstate for $LB_PXE_NODE" makedhcp "$LB_NODE,$LB_PXE_NODE" || die "makedhcp for the installed nodes failed" - dhcptest_run \ + dhcp_run \ --set booted_mac="$LB_MAC" --set booted_ip="$LB_IP" \ --set booted_script="http://$SRV_IP/tftpboot/xcat/xnba/nodes/$LB_NODE" \ --set booted_pxe_mac="$LB_PXE_MAC" --set booted_pxe_ip="$LB_PXE_IP" \ - conf/localboot.conf || rc=1 + conf/dhcp/localboot.conf || rc=1 return $rc } @@ -862,8 +862,8 @@ do_run_loader_absent() { echo "$path" > "$STATE/loader-away" if do_generate; then - dhcptest_run --set pool="$POOL" --set present_loader="$(arch_loader uefi)" \ - conf/loader-absent.conf || rc=1 + dhcp_run --set pool="$POOL" --set present_loader="$(arch_loader uefi)" \ + conf/dhcp/loader-absent.conf || rc=1 else say "FAILED: makedhcp could not regenerate with $path missing" rc=1 @@ -891,9 +891,9 @@ do_run_httpport() { echo done > "$STATE/httpport" if do_generate; then - dhcptest_run --set tftp="$SRV_IP" --set httpport="$ALT_HTTPPORT" \ + dhcp_run --set tftp="$SRV_IP" --set httpport="$ALT_HTTPPORT" \ --set riscv64_loader="$(arch_loader riscv64)" \ - conf/http-port.conf || rc=1 + conf/dhcp/http-port.conf || rc=1 else say "FAILED: makedhcp could not regenerate with site.httpport=$ALT_HTTPPORT" rc=1 @@ -913,8 +913,8 @@ do_run_rangecidr() { echo done > "$STATE/rangecidr" if do_generate; then - dhcptest_run --set unknown_mac="$UNKNOWN_MAC" --set pool="$POOL_CIDR" \ - conf/dynamic-range-cidr.conf || rc=1 + dhcp_run --set unknown_mac="$UNKNOWN_MAC" --set pool="$POOL_CIDR" \ + conf/dhcp/dynamic-range-cidr.conf || rc=1 else say "FAILED: makedhcp could not regenerate with a CIDR dynamic range" rc=1 @@ -935,31 +935,30 @@ do_run_removal() { arch=x86_64 netboot="$NETBOOT" tftpserver="$SRV_IP" xcatmaster="$SRV_IP" makedhcp "$RM_NODE" || die "makedhcp $RM_NODE failed" - dhcptest_run -s removed-node-keeps-its-reservation-until-it-is-withdrawn \ + dhcp_run -s removed-node-keeps-its-reservation-until-it-is-withdrawn \ --set removed_mac="$RM_MAC" --set removed_ip="$RM_IP" --set pool="$POOL" \ - conf/node-removal.conf || rc=1 + conf/dhcp/node-removal.conf || rc=1 makedhcp -d "$RM_NODE" || die "makedhcp -d $RM_NODE failed" - dhcptest_run -s withdrawn-node-is-no-longer-served-its-address \ + dhcp_run -s withdrawn-node-is-no-longer-served-its-address \ --set removed_mac="$RM_MAC" --set removed_ip="$RM_IP" --set pool="$POOL" \ - conf/node-removal.conf || rc=1 + conf/dhcp/node-removal.conf || rc=1 return $rc } # S-54. Nothing to configure: a BOOTP client is served out of the dynamic range # a discovery cluster needs anyway. do_run_bootp() { - dhcptest_run --set bootp_mac="$BOOTP_MAC" --set pool="$POOL" \ - conf/bootp-client.conf + dhcp_run --set bootp_mac="$BOOTP_MAC" --set pool="$POOL" \ + conf/dhcp/bootp-client.conf } do_run_hierarchy() { - ( cd "$DHCPTEST" && python3 src/dhcptest run -i "$IF_CLI" \ - --set node_mac="$NODE_MAC" --set node_ip="$NODE_IP" \ + dhcp_run --set node_mac="$NODE_MAC" --set node_ip="$NODE_IP" \ --set node_loader="$(node_loader)" \ --set delegate="$DELEGATE_IP" --set unknown_mac="$UNKNOWN_MAC" \ - conf/hierarchy-dhcpserver.conf ) + conf/dhcp/hierarchy-dhcpserver.conf } do_teardown() { diff --git a/xCAT-test/autotest/testcase/provtest/cases0 b/xCAT-test/autotest/testcase/blackboxtest/prov_cases similarity index 78% rename from xCAT-test/autotest/testcase/provtest/cases0 rename to xCAT-test/autotest/testcase/blackboxtest/prov_cases index dd17c0dec..eb3ac3d67 100644 --- a/xCAT-test/autotest/testcase/provtest/cases0 +++ b/xCAT-test/autotest/testcase/blackboxtest/prov_cases @@ -1,12 +1,11 @@ -# provtest issues the requests a provisioning node issues -- dig for DNS, tftp, -# curl, and a small client of its own for the xcatd protocols -- and asserts on -# the answers. It never reads the xCAT database and never runs an xCAT command: -# everything it knows arrives on the command line from provfixture.sh. +# The provision-chain wire cases: blackboxtest issues the requests a +# provisioning node issues -- dig for DNS, tftp, curl, and a small client of its +# own for the xcatd protocols -- and asserts on the answers. Everything it +# expects arrives on the command line from provfixture.sh. # -# The first two cases are offline and safe anywhere. The wire cases build a -# provisioning network out of a veth pair, the client end in a namespace of its -# own because xcatd calls a discovering machine back on TCP 3001 -- the port it -# is itself listening on, which a client sharing its stack could never hold. +# Each case builds a provisioning network out of a veth pair, the client end in +# a namespace of its own: xcatd calls a discovering machine back on TCP 3001, +# the port it listens on itself, which a client sharing its stack cannot hold. # # Each wire case is one script, so its teardown runs from a trap and the machine # is left serving its own configuration even when an assertion fails. @@ -14,24 +13,7 @@ # A case that cannot run says so and passes. Read a pass as coverage only when # the log shows the ok lines. -start:provtest_unit_tests -description:provtest's own unit tests: no network and no root needed -label:mn_only,ci_test,prov -os:Linux -cmd:cd /opt/xcat/share/xcat/tools/autotest/provtest/tests && python3 -m unittest discover -s . -p 'test_*.py' -check:rc==0 -end - -start:provtest_validate_conf -description:Every shipped provtest scenario parses and every assertion names a field its step can carry -label:mn_only,ci_test,prov -os:Linux -cmd:cd /opt/xcat/share/xcat/tools/autotest/provtest && python3 src/provtest validate conf/*.conf -check:rc==0 -check:output=~scenario -end - -start:provtest_name_resolution +start:blackbox_prov_name_resolution description:Every name a node resolves during provisioning answers, in both directions label:mn_only,prov,prov_wire os:Linux @@ -46,7 +28,7 @@ os:Linux # convenience but the authentication model. cmd:#!/bin/bash set -u - FIX=/opt/xcat/share/xcat/tools/autotest/testcase/provtest/provfixture.sh + FIX=/opt/xcat/share/xcat/tools/autotest/testcase/blackboxtest/provfixture.sh # 0 run, 1 this machine cannot run the wire cases and the case passes, 2 the # machine is dirty or is a live cluster -- which must fail, or one unfinished # teardown would make every later run here green without running anything. @@ -57,7 +39,7 @@ cmd:#!/bin/bash check:rc==0 end -start:provtest_loader_and_config_fetch +start:blackbox_prov_loader_and_config_fetch description:Each netboot method's loader, config and kernel are fetchable under the names the firmware asks for label:mn_only,prov,prov_wire os:Linux @@ -70,7 +52,7 @@ os:Linux # kernel is called, which is what stops them asserting xCAT against xCAT. cmd:#!/bin/bash set -u - FIX=/opt/xcat/share/xcat/tools/autotest/testcase/provtest/provfixture.sh + FIX=/opt/xcat/share/xcat/tools/autotest/testcase/blackboxtest/provfixture.sh # 0 run, 1 this machine cannot run the wire cases and the case passes, 2 the # machine is dirty or is a live cluster -- which must fail, or one unfinished # teardown would make every later run here green without running anything. @@ -81,7 +63,7 @@ cmd:#!/bin/bash check:rc==0 end -start:provtest_install_tree_over_http +start:blackbox_prov_install_tree_over_http description:The repository, the kickstart and the port the node was told about all answer label:mn_only,prov,prov_wire os:Linux @@ -92,7 +74,7 @@ os:Linux # node was told about. cmd:#!/bin/bash set -u - FIX=/opt/xcat/share/xcat/tools/autotest/testcase/provtest/provfixture.sh + FIX=/opt/xcat/share/xcat/tools/autotest/testcase/blackboxtest/provfixture.sh # 0 run, 1 this machine cannot run the wire cases and the case passes, 2 the # machine is dirty or is a live cluster -- which must fail, or one unfinished # teardown would make every later run here green without running anything. @@ -103,7 +85,7 @@ cmd:#!/bin/bash check:rc==0 end -start:provtest_genesis_and_discovery_artefacts +start:blackbox_prov_genesis_and_discovery_artefacts description:The genesis images and the per-network discovery configs a node with no definition boots label:mn_only,prov,prov_wire os:Linux @@ -111,7 +93,7 @@ os:Linux # finds there decides whether it can announce itself at all. cmd:#!/bin/bash set -u - FIX=/opt/xcat/share/xcat/tools/autotest/testcase/provtest/provfixture.sh + FIX=/opt/xcat/share/xcat/tools/autotest/testcase/blackboxtest/provfixture.sh # 0 run, 1 this machine cannot run the wire cases and the case passes, 2 the # machine is dirty or is a live cluster -- which must fail, or one unfinished # teardown would make every later run here green without running anything. @@ -122,7 +104,7 @@ cmd:#!/bin/bash check:rc==0 end -start:provtest_discovery_requests +start:blackbox_prov_discovery_requests description:Flow control and findme, including the requests xcatd must acknowledge and then ignore label:mn_only,prov,prov_wire os:Linux @@ -133,7 +115,7 @@ os:Linux # gateway instead of putting it on the wire. cmd:#!/bin/bash set -u - FIX=/opt/xcat/share/xcat/tools/autotest/testcase/provtest/provfixture.sh + FIX=/opt/xcat/share/xcat/tools/autotest/testcase/blackboxtest/provfixture.sh # 0 run, 1 this machine cannot run the wire cases and the case passes, 2 the # machine is dirty or is a live cluster -- which must fail, or one unfinished # teardown would make every later run here green without running anything. @@ -144,7 +126,7 @@ cmd:#!/bin/bash check:rc==0 end -start:provtest_xcatd_protocol +start:blackbox_prov_xcatd_protocol description:getdestiny, getpostscript and getcredentials, from the address that is the identity label:mn_only,prov,prov_wire os:Linux @@ -155,7 +137,7 @@ os:Linux # the client's port 300 is what stands between a PTR and a signed certificate. cmd:#!/bin/bash set -u - FIX=/opt/xcat/share/xcat/tools/autotest/testcase/provtest/provfixture.sh + FIX=/opt/xcat/share/xcat/tools/autotest/testcase/blackboxtest/provfixture.sh # 0 run, 1 this machine cannot run the wire cases and the case passes, 2 the # machine is dirty or is a live cluster -- which must fail, or one unfinished # teardown would make every later run here green without running anything. @@ -166,7 +148,7 @@ cmd:#!/bin/bash check:rc==0 end -start:provtest_install_monitor +start:blackbox_prov_install_monitor description:The plain-text service on 3002 that an installing node reports progress to label:mn_only,prov,prov_wire os:Linux @@ -175,7 +157,7 @@ os:Linux # accepted installs perfectly and shows as stuck. cmd:#!/bin/bash set -u - FIX=/opt/xcat/share/xcat/tools/autotest/testcase/provtest/provfixture.sh + FIX=/opt/xcat/share/xcat/tools/autotest/testcase/blackboxtest/provfixture.sh # 0 run, 1 this machine cannot run the wire cases and the case passes, 2 the # machine is dirty or is a live cluster -- which must fail, or one unfinished # teardown would make every later run here green without running anything. @@ -186,7 +168,7 @@ cmd:#!/bin/bash check:rc==0 end -start:provtest_failing_at_one_place +start:blackbox_prov_failing_at_one_place description:A master that cannot be reached, a name that was withdrawn, and a state that was replaced label:mn_only,prov,prov_wire os:Linux @@ -198,7 +180,7 @@ os:Linux # the node's address, so a name withdrawn cannot be set again afterwards. cmd:#!/bin/bash set -u - FIX=/opt/xcat/share/xcat/tools/autotest/testcase/provtest/provfixture.sh + FIX=/opt/xcat/share/xcat/tools/autotest/testcase/blackboxtest/provfixture.sh # 0 run, 1 this machine cannot run the wire cases and the case passes, 2 the # machine is dirty or is a live cluster -- which must fail, or one unfinished # teardown would make every later run here green without running anything. diff --git a/xCAT-test/autotest/testcase/provtest/provfixture.sh b/xCAT-test/autotest/testcase/blackboxtest/provfixture.sh similarity index 96% rename from xCAT-test/autotest/testcase/provtest/provfixture.sh rename to xCAT-test/autotest/testcase/blackboxtest/provfixture.sh index 839b9be67..f9504a3c3 100755 --- a/xCAT-test/autotest/testcase/provtest/provfixture.sh +++ b/xCAT-test/autotest/testcase/blackboxtest/provfixture.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# The xCAT side of the provtest wire cases: provtest knows nothing about xCAT, +# The xCAT side of the provision-chain wire cases: blackboxtest knows nothing about xCAT, # so everything that does lives here. It builds a provisioning network out of a # veth pair, the client end holding the node's address, because xcatd names a # client by the reverse lookup of the address a request arrived from. @@ -124,8 +124,8 @@ SECOND_DESTINY=shell NEXT_DESTINY=boot STATE=/tmp/provtest-fixture -PROVTEST=/opt/xcat/share/xcat/tools/autotest/provtest -[ -d "$PROVTEST" ] || PROVTEST="$(cd "$(dirname "$0")/../../../provtest" 2>/dev/null && pwd)" +TOOL=/opt/xcat/share/xcat/tools/autotest/blackboxtest +[ -d "$TOOL" ] || TOOL="$(cd "$(dirname "$0")/../../../blackboxtest" 2>/dev/null && pwd)" say() { echo "provfixture: $*"; } die() { echo "provfixture: $*" >&2; exit 1; } @@ -134,8 +134,8 @@ die() { echo "provfixture: $*" >&2; exit 1; } # case passes; a machine holding this fixture's leftovers would have every later # run skip green forever, and a cluster with nodes of its own would be damaged # by the setup. Those two are refusals, and cases0 fails the case on them. -skip() { echo "provtest skipped: $*"; exit 1; } -refuse() { echo "provtest refused: $*" >&2; exit 2; } +skip() { echo "provfixture skipped: $*"; exit 1; } +refuse() { echo "provfixture refused: $*" >&2; exit 2; } # A stage this machine cannot run: says what is missing and passes. Counted and # named on one greppable line, because a build that is green because nothing ran @@ -143,7 +143,7 @@ refuse() { echo "provtest refused: $*" >&2; exit 2; } SKIPPED=0 stage_skip() { SKIPPED=$((SKIPPED + 1)) - echo "provtest SKIP: ${STAGE:-fixture}: $*" + echo "provfixture SKIP: ${STAGE:-fixture}: $*" return 0 } @@ -200,16 +200,16 @@ pkgdir_for() { echo "$(installdir)/$OSVERS/$1"; } # The names the loaders and the resolver ask for, arrived at the way the # firmware arrives at them: an encoding wrong by one digit produces no error -# anywhere. Calls provtest's own encoders rather than a second implementation in +# anywhere. Calls blackboxtest's own encoders rather than a second implementation in # shell, which would be a second chance to be wrong with no unit tests. netutil() { local fn=$1 shift python3 -c 'import sys sys.path.insert(0, sys.argv[1]) -from provtest_lib import netutil +from blackboxtest_lib import netutil sys.stdout.write(str(getattr(netutil, sys.argv[2])(*sys.argv[3:])))' \ - "$PROVTEST/src" "$fn" "$@" + "$TOOL/src" "$fn" "$@" } hex_ip() { netutil hex_ip "$1"; } @@ -327,9 +327,9 @@ do_check() { command -v ss >/dev/null 2>&1 || skip "ss is not installed (iproute2), and every stage asks who holds its port" command -v nodeset >/dev/null 2>&1 || skip "nodeset is not on PATH, so this is not a management node" command -v makedns >/dev/null 2>&1 || skip "makedns is not on PATH, so this is not a management node" - # -f rather than -x: the fixture runs it as `python3 src/provtest`, and + # -f rather than -x: the fixture runs it as `python3 src/blackboxtest`, and # dh_install keeps the source mode on Debian. - [ -f "$PROVTEST/src/provtest" ] || skip "provtest is not installed under $PROVTEST" + [ -f "$TOOL/src/blackboxtest" ] || skip "blackboxtest is not installed under $TOOL" # The clients the scenarios are driven with. They are real binaries on # purpose: a node fetches with the same ones. @@ -828,8 +828,8 @@ do_generate() { # the free ports and the routes are the node's own. in_ns() { ip netns exec "$NETNS" "$@"; } -provtest_run() { - ( cd "$PROVTEST" && in_ns python3 src/provtest run "$@" ) +prov_run() { + ( cd "$TOOL" && in_ns python3 src/blackboxtest run "$@" ) } # What every scenario file is told: who is asked, and from whose address. Named @@ -852,25 +852,25 @@ do_run_dns() { assert_serving udp 53 "the name server" scenarios="-s node-forward -s node-reverse -s node-alias -s local-nxdomain -s master-resolves" - provtest_run $scenarios \ + prov_run $scenarios \ "${COMMON[@]}" \ --set node="$NODE" --set domain="$DOMAIN" \ --set nodeip="$NODE_IP" --set revname="$(reverse_name "$NODE_IP")" \ --set alias="$ALIAS" --set master="$MASTER_NODE.$DOMAIN" \ --set missing="$MISSING" --set forwarded="$FORWARDED" \ --set net="$NET/$PREFIX" \ - conf/dns.conf || rc=1 + conf/prov/dns.conf || rc=1 if dig +short +time=5 +tries=1 "$FORWARDED" A >/dev/null 2>&1 && \ [ -n "$(dig +short +time=5 +tries=1 "$FORWARDED" A 2>/dev/null)" ]; then - provtest_run -s forwarded-name \ + prov_run -s forwarded-name \ "${COMMON[@]}" \ --set node="$NODE" --set domain="$DOMAIN" \ --set nodeip="$NODE_IP" --set revname="$(reverse_name "$NODE_IP")" \ --set alias="$ALIAS" --set master="$MASTER_NODE.$DOMAIN" \ --set missing="$MISSING" --set forwarded="$FORWARDED" \ --set net="$NET/$PREFIX" \ - conf/dns.conf || rc=1 + conf/prov/dns.conf || rc=1 else stage_skip "$FORWARDED does not resolve from this machine, so there is no forwarder to test" fi @@ -896,20 +896,20 @@ do_run_tftp() { stage_skip "$tftp/$loader is not present, so $NODE is not a grub2 node and only the escape scenario runs" fi # shellcheck disable=SC2086 - provtest_run $scenarios \ + prov_run $scenarios \ "${COMMON[@]}" \ --set node="$NODE" --set hexip="$(hex_ip "$NODE_IP")" \ --set macdashes="$(dashed_mac "$NODE_MAC")" --set mac="$NODE_MAC" \ --set loader="$loader" --set bootfile="boot/grub2/grub2-$NODE" \ --set master="$SRV_IP" --set xcatport="$XCATPORT" --set destiny="$DESTINY" \ - conf/tftp-grub2.conf || rc=1 + conf/prov/tftp-grub2.conf || rc=1 - provtest_run \ + prov_run \ "${COMMON[@]}" \ --set node="$PXE_NODE" --set hexip="$(hex_ip "$PXE_IP")" \ --set master="$SRV_IP" --set xcatport="$XCATPORT" --set destiny="$DESTINY" \ --set bootnode="$BOOT_NODE" \ - conf/tftp-pxelinux.conf || rc=1 + conf/prov/tftp-pxelinux.conf || rc=1 # The xnba case is the one that crosses transports, so a web server serving # the wrong tree fails it for an unrelated reason. The script is asserted @@ -922,17 +922,17 @@ do_run_tftp() { stage_skip "the web server on port $port does not serve $tftp, so the kernel half of the xnba case is left out" fi # shellcheck disable=SC2086 - provtest_run $xnba \ + prov_run $xnba \ "${COMMON[@]}" \ --set node="$XNBA_NODE" --set httpport="$port" \ - conf/tftp-xnba.conf || rc=1 + conf/prov/tftp-xnba.conf || rc=1 if [ -f "$tftp/petitboot/$PTB_NODE" ]; then - provtest_run \ + prov_run \ "${COMMON[@]}" \ --set node="$PTB_NODE" --set hexip="$(hex_ip "$PTB_IP")" \ --set master="$SRV_IP" --set xcatport="$XCATPORT" --set destiny="$DESTINY" \ - conf/tftp-petitboot.conf || rc=1 + conf/prov/tftp-petitboot.conf || rc=1 else stage_skip "no petitboot config was generated for $PTB_NODE on this management node" fi @@ -973,7 +973,7 @@ do_run_http() { esac # shellcheck disable=SC2086 - provtest_run $select \ + prov_run $select \ "${COMMON[@]}" \ --set node="$NODE" --set httpport="$port" --set otherport="$other" \ --set hexip="$(hex_ip "$NODE_IP")" \ @@ -981,7 +981,7 @@ do_run_http() { --set installpath="autoinst/$NODE" \ --set repofile="repodata/repomd.xml" \ --set master="$SRV_IP" --set arch="$NODE_ARCH" \ - conf/http.conf + conf/prov/http.conf } # Stage 5. The per-network artefacts, which exist only once genesis has been @@ -1006,11 +1006,11 @@ do_run_genesis() { # Unquoted on purpose: the selection is a list this script built itself. # shellcheck disable=SC2086 - provtest_run $select \ + prov_run $select \ "${COMMON[@]}" \ --set hexnet="$hexnet" --set netfile="$NET_FILE" \ --set master="$SRV_IP" --set xcatport="$XCATPORT" \ - conf/discovery-artefacts.conf + conf/prov/discovery-artefacts.conf } # Stages 6 and 7, in the order a discovering machine does them: it asks for a @@ -1019,14 +1019,14 @@ do_run_discovery() { local rc=0 assert_serving udp "$XCATPORT" "the xcatd flow-control listener" - provtest_run \ + prov_run \ "${COMMON[@]}" \ - conf/flowcontrol.conf || rc=1 + conf/prov/flowcontrol.conf || rc=1 - provtest_run \ + prov_run \ "${COMMON[@]}" \ --set foreign="$FOREIGN_IP" \ - conf/findme.conf || rc=1 + conf/prov/findme.conf || rc=1 return $rc } @@ -1036,35 +1036,35 @@ do_run_xcatd() { local rc=0 assert_serving tcp "$XCATPORT" "xcatd" - provtest_run \ + prov_run \ "${COMMON[@]}" \ --set xcatport="$XCATPORT" --set refused=rpower --set node="$NODE" \ - conf/xcatd-policy.conf || rc=1 + conf/prov/xcatd-policy.conf || rc=1 - provtest_run \ + prov_run \ "${COMMON[@]}" \ --set unknown="$UNKNOWN_IP" --set node="$NODE" \ --set xcatport="$XCATPORT" --set monitorport="$MONITORPORT" \ --set master="$SRV_IP" --set domain="$DOMAIN" \ - conf/xcatd-postscript.conf || rc=1 + conf/prov/xcatd-postscript.conf || rc=1 - provtest_run \ + prov_run \ "${COMMON[@]}" \ --set xcatport="$XCATPORT" --set credtype=xcat_server_cred \ --set highport="$HIGHPORT" \ - conf/xcatd-credentials.conf || rc=1 + conf/prov/xcatd-credentials.conf || rc=1 # Last, and after the node has been reset, because these scenarios end by # advancing the chain: nextdestiny is a write, not a question. Resetting is # what makes the stage repeatable. reset_node - provtest_run \ + prov_run \ "${COMMON[@]}" \ --set unknown="$UNKNOWN_IP" --set node="$NODE" \ --set destiny="$DESTINY" --set master="$SRV_IP" --set xcatport="$XCATPORT" \ --set imgserver="$ALT_IP" --set next="$NEXT_DESTINY" \ - conf/xcatd-destiny.conf || rc=1 + conf/prov/xcatd-destiny.conf || rc=1 return $rc } @@ -1072,11 +1072,11 @@ do_run_xcatd() { do_run_monitor() { assert_serving tcp "$MONITORPORT" "the xcatd install monitor" - provtest_run \ + prov_run \ "${COMMON[@]}" \ --set unknown="$UNKNOWN_IP" --set monitorport="$MONITORPORT" \ --set xcatport="$XCATPORT" \ - conf/monitor.conf + conf/prov/monitor.conf } # Failing at exactly one place. Each scenario needs the cluster in a different @@ -1105,7 +1105,7 @@ do_run_ordering() { # being told to discover itself anyway. reset_node - provtest_run -s unreachable-master "${flags[@]}" conf/ordering.conf || rc=1 + prov_run -s unreachable-master "${flags[@]}" conf/prov/ordering.conf || rc=1 # P-75 before P-74, forced rather than chosen: the second nodeset has to # resolve the node to name the config it writes, so it cannot run once the @@ -1122,7 +1122,7 @@ do_run_ordering() { nodeset "$NODE" "$SECOND_DESTINY" >/dev/null 2>&1 generated "$(node_netboot)" "$NODE" "$NODE_IP" \ || die "nodeset $NODE $SECOND_DESTINY wrote no $(node_netboot) configuration" - provtest_run -s state-replaced "${flags[@]}" conf/ordering.conf || rc=1 + prov_run -s state-replaced "${flags[@]}" conf/prov/ordering.conf || rc=1 fi # P-74: the node's name taken out of resolution, nothing else. Both the zone @@ -1132,7 +1132,7 @@ do_run_ordering() { makedns -d "$NODE" >/dev/null 2>&1 || say "makedns -d $NODE reported an error" makehosts -d "$NODE" >/dev/null 2>&1 || say "makehosts -d $NODE reported an error" echo done > "$STATE/ptrgone" - provtest_run -s missing-ptr "${flags[@]}" conf/ordering.conf || rc=1 + prov_run -s missing-ptr "${flags[@]}" conf/prov/ordering.conf || rc=1 return $rc } @@ -1159,11 +1159,11 @@ do_run_dns_removal() { makedns -d "$NODE" >/dev/null 2>&1 || say "makedns -d $NODE reported an error" echo done > "$STATE/ptrgone" - provtest_run \ + prov_run \ "${COMMON[@]}" \ --set node="$NODE" --set domain="$DOMAIN" \ --set revname="$(reverse_name "$NODE_IP")" \ - conf/dns-removal.conf + conf/prov/dns-removal.conf } # --- teardown ------------------------------------------------------------- @@ -1313,5 +1313,5 @@ dispatch "$@" rc=$? # Said once at the end, so a reader of a passing run sees that something was # left out without having to notice a line in the middle of the output. -[ "$SKIPPED" != 0 ] && echo "provtest SKIPPED-TOTAL: $STAGE: $SKIPPED" +[ "$SKIPPED" != 0 ] && echo "provfixture SKIPPED-TOTAL: $STAGE: $SKIPPED" exit $rc diff --git a/xCAT-test/blackboxtest/.gitignore b/xCAT-test/blackboxtest/.gitignore new file mode 100644 index 000000000..e4a978990 --- /dev/null +++ b/xCAT-test/blackboxtest/.gitignore @@ -0,0 +1,4 @@ +# The tool runs in place, from a checkout and from +# /opt/xcat/share/xcat/tools/autotest, so Python writes its caches here. +__pycache__/ +*.pyc diff --git a/xCAT-test/blackboxtest/README.md b/xCAT-test/blackboxtest/README.md new file mode 100644 index 000000000..efe71f38c --- /dev/null +++ b/xCAT-test/blackboxtest/README.md @@ -0,0 +1,341 @@ +# blackboxtest + +A client-side test of the services a booting node uses. It asks a management +node the questions the node asks -- a DHCP DISCOVER, a DNS lookup, a TFTP +fetch, an HTTP GET, a request to `xcatd` -- and asserts on the answers. + +It never reads the xCAT database and never runs an xCAT command. Every value it +expects comes from the command line, from the fixture that set the cluster up. +A suite that asked xCAT what to expect would pass on a cluster that cannot +boot a node. + +Requires Python 3.6 or later. DHCP steps also need Scapy (`python3-scapy`) +and root; DNS, TFTP and HTTP steps need `dig`, `tftp` and `curl`. There is +no install step: it runs from a checkout. + +## Commands + +``` +blackboxtest run [options] CONF [CONF...] # execute scenarios +blackboxtest validate CONF [CONF...] # check offline +blackboxtest list CONF [CONF...] # show what a file does +``` + +`validate` and `list` need no root, no network, no Scapy and no client +program. They catch a misspelt reply field, a reference to a step that has not +run, and a DHCP step that is not legal in the client state it runs in. + +| Option to `run` | Meaning | +| --- | --- | +| `-i`, `--interface IFACE` | interface the DHCP steps send from; overrides the file | +| `-b`, `--bind ADDR` | source address of every step that sets no `bind` | +| `--set KEY=VALUE` | define `%(KEY)s` (repeatable) | +| `-s`, `--scenario NAME` | run only this scenario (repeatable) | +| `--timeout SECS`, `--retries N` | per attempt, and attempts per step | +| `-v` | print each reply to stderr | + +| Exit code | Meaning | +| --- | --- | +| 0 | every assertion held | +| 1 | at least one assertion failed | +| 2 | the configuration or the command line is wrong | +| 3 | this host cannot run the test: no Scapy, no root, no `dig`, ... | + +3 is checked before any output. A host that tested nothing must not report a +green run. + +## Configuration + +INI, read by `configparser`. A `[step]` belongs to the `[scenario]` above it, +and steps run in file order. Unknown keys are an error. + +```ini +[vars] +domain = cluster + +[defaults] +server = %(server)s +bind = %(client)s + +[scenario boot-file-fetchable] +description = The loader DHCP names can be fetched under that name +interface = eth1 + +[step discover] +type = discover +client_arch = 0x000b +request_options = 1, 3, 6, 54, 66, 67 +assert = + yiaddr in %(net)s + bootfile present + +[step loader] +type = tftp +path = $discover.bootfile +assert = + size > 0 +``` + +`[defaults]` supplies a key to each step whose type takes it. So one +`server =` serves the DNS, TFTP and HTTP steps, and the DHCP step in the same +file is not made illegal by it. + +### Variables and references + +`%(name)s` is configparser's `BasicInterpolation`, resolved when the file is +loaded, from `[vars]` and `--set`; `--set` wins. It is resolved before `-s` +selects anything, so a file needs all its variables. `validate` prints them. + +`$step.field` is a field of an earlier reply, resolved at run time. The step +is any earlier step in the scenario, or a name the DHCP client binds: `$offer`, +`$ack`, `$lease`, `$reply`. A list field is indexed as `data.0`, a mapping read +as `header.content-type`, and the bare name of a list gives its first element. + +```ini +[step kernelpath] +type = extract +from = $config.text +pattern = ^\s*linux\s+/?(\S+) + +[step kernel] +type = tftp +path = $kernelpath.value +``` + +Nothing there says what the kernel is called. The config is fetched, the name +read out of it, and that name fetched. + +### Assertions + +One per line under `assert`, as `target op value`. + +| Op | Holds when | +| --- | --- | +| `==` `!=` | equal as addresses, then as numbers, then as text. A list equals a value when any element does | +| `in` `not-in` | inside a CIDR (`10.0.0.0/24`), a range (`10.0.0.200-10.0.0.250`) or a comma list | +| `present` `absent` | the field carries a value. An empty list and a zeroed BOOTP field are absent | +| `matches` `contains` `starts-with` `ends-with` | text; `matches` is a multi-line regex | +| `<` `<=` `>` `>=` | numeric | + +`!=` and `not-in` hold against an absent target: a reply naming no boot file +has not named the wrong one. Every other operator fails on an absent target. + +`attempt` and, on a DHCP step, `offers` (the number of servers that answered) +come from the runner. `$step.field` may be a target. + +### Expectations + +`expect` states what the transport must do: + +| Family | `expect` | Default | +| --- | --- | --- | +| DHCP | `offer` `ack` `nak` `bootreply` `any` `none` | the reply the client state waits for | +| service | `ok` `fail` `any` | `ok` | +| `extract`, `noop` | `ok` `fail` `any` | `any` | + +A service step that waited for an answer and got none fails with no +assertion. + +## Step types + +| `type` | Driven by | Keys | Reply fields | +| --- | --- | --- | --- | +| `discover` `request` `renew` `rebind` `release` `bootrequest` | raw socket | see below | see below | +| `dns` | `dig` | `server` `port` `name` `rrtype` `recursion` `bind` | `status` `rcode` `flags` `count` `data` `type` `ttl` `name` `question` `answers` `authority` `server` | +| `tftp` | `tftp` | `server` `port` `path` `mode` | `ok` `size` `sha256` `text` `error` `path` `server` | +| `http` | `curl` | `server` `port` `path` `url` `method` `header` `insecure` `bind` | `status` `size` `sha256` `text` `url` `header` `content_type` `error` `ok` | +| `xcatreq` | TLS socket | `server` `port` `command` `element` `raw` `source_port` `callback_port` `callback_listen` `callback_reply` `callback_wait` `cert` `key` `bind` | `destiny` `kernel` `initrd` `kcmdline` `imgserver` `name` `error` `serverdone` `elements` `data` `content` `desc` `text` `handshake` `ok` `callback_seen` `callback_data` | +| `monitor` | TCP socket | `server` `port` `send` `source_port` `bind` | `greeting` `lines` `text` `ok` `closed` `error` | +| `flowrequest` | UDP socket | `server` `port` `message` `replies` `source_port` `bind` | `replies` `count` `ok` `error` | +| `findme` | UDP socket | `server` `port` `payload` `encoding` `source_port` `callback_listen` `callback_wait` `bind` | `callbacks` `count` `ok` `error` | +| `extract` | -- | `from` `pattern` `group` | `value` `groups` `matched` `count` | +| `noop` | -- | -- | -- | + +Every step also takes `type`, `expect`, `timeout`, `retries` and `assert`. + +`bind` is the client's own address. `xcatd` names a client by the reverse +lookup of the address its connection came from, so the source address is +under test and is never left to the kernel. + +### DHCP steps + +The DHCP client is an RFC 2131 state machine on a raw Layer 2 socket, so it +works on an interface with no address -- the state of a provisioning NIC when a +machine boots. One client runs per scenario: + +| State | Legal steps | +| --- | --- | +| INIT | `discover`, `bootrequest`, `request` with `requested_address` (INIT-REBOOT) | +| SELECTING | `request` | +| BOUND | `renew` (unicast), `rebind`, `release` | + +A step that waited for an answer and got none leaves the state as it was. A NAK +returns the client to INIT. + +Keys: `mac`, `ciaddr`, `giaddr`, `client_id` (`auto` by default, `none`, or +hex), `user_class` with `user_class_form` = `raw` or `rfc3004`, and one key +per option: `requested_address` (50), `server_id` (54), `lease_time` (51), +`hostname` (12), `fqdn` (81), `max_message_size` (57), `vendor_class` (60), +`client_arch` (93), `client_ndi` (94), `client_uuid` (97), `vendor_specific` +(43, hex), `ipxe_options` (175, hex), `request_options` (55). + +`fqdn = S:node01` sets the flags before the colon when every character there is +a flag letter (`S` `O` `E` `N`); `E` sends the name in wire format. + +Reply fields: the BOOTP header (`msgtype` `xid` `yiaddr` `siaddr` `ciaddr` +`giaddr` `chaddr` `file` `sname` `secs` `flags` `src_ip` `src_mac`), the +aliases `address` `mac` `next_server`, any option as `54`, `option:54`, +`opt54` or `server_id`, and three fields a client reads from two places: + +- `bootfile`: option 67 when sent, else the `file` header. ISC fills the + header, dnsmasq answers in option 67, and firmware reads either. +- `dns_name`: option 81 when sent, else option 12. Kea answers option 81 in + option 81; ISC with `ignore client-updates` sends option 12 only. +- `fqdn_flags`: option 81's flags as letters, absent without option 81. + +Assert on those rather than on `file` or one option, unless the one place is +the point. `msgtype` is `BOOTREPLY` for a reply with no option 53. + +The client MAC is a random locally administered address, so the host's own +network stack never takes a reply as its own. The tool answers ARP only for an +address it was granted, so a unicast renewal arrives without the address ever +being configured on the host. + +## Shipped scenarios + +`conf/dhcp/` asserts the DHCP clauses (`S-nn`), `conf/prov/` the provision +chain clauses (`P-nn`). Each scenario's description names its clause. + +| File | What it asserts | +| --- | --- | +| `dhcp/discover-offer.conf` | one DISCOVER draws exactly one OFFER | +| `dhcp/full-lease.conf` | DISCOVER/OFFER/REQUEST/ACK yields the offered address | +| `dhcp/static-vs-dynamic.conf` | a reserved MAC gets its address, another MAC a pool address | +| `dhcp/pxe-arch-matrix.conf` | each client architecture (option 93) is offered its own loader | +| `dhcp/ipxe-userclass.conf` | stage 1 and stage 2 differ, in both user-class encodings | +| `dhcp/renew-rebind.conf` | a lease survives RENEW and REBIND | +| `dhcp/provision-vs-discovery.conf` | a known machine gets its reservation and loader; an unknown one a pool address | +| `dhcp/discovery-bootfile.conf` | an unknown machine is handed a loader too | +| `dhcp/netboot-methods.conf` | a node gets the loader its netboot method names, its own name, and a `*NOIP*` port no answer | +| `dhcp/hierarchy-dhcpserver.conf` | a delegated pool ignores unknown MACs and still serves known ones | +| `dhcp/discovery-adoption.conf` | a machine discovered from the pool gets its own address once defined | +| `dhcp/nak-foreign-address.conf` | a REQUEST for an address off this network is refused | +| `dhcp/next-server-source.conf` | next-server follows each node's attributes | +| `dhcp/multi-mac-node.conf` | a node with two ports gets a different address on each | +| `dhcp/iscsi-root-path.conf` | a diskless node is told its iSCSI target | +| `dhcp/loader-absent.conf` | an architecture whose loader is missing gets an address and no boot file | +| `dhcp/http-port.conf` | an HTTP boot URL names the port the web server listens on | +| `dhcp/dynamic-range-cidr.conf` | a dynamic range written as a CIDR serves from it | +| `dhcp/node-removal.conf` | a withdrawn node stops being offered its old address | +| `dhcp/bootp-client.conf` | a BOOTP client is still given an address | +| `dhcp/localboot.conf` | an installed node is offered its address and no boot script | +| `prov/dns.conf`, `prov/dns-removal.conf` | forward, reverse, alias, forwarded and removed names | +| `prov/tftp-{grub2,pxelinux,xnba,petitboot}.conf` | each netboot method's loader, config, kernel and initrd, by the name the firmware asks for | +| `prov/discovery-artefacts.conf` | the configs and genesis images an unknown machine fetches | +| `prov/http.conf` | the install tree, the TFTP tree over HTTP, and the port the node was told | +| `prov/flowcontrol.conf`, `prov/findme.conf` | UDP 3001: flow control and discovery callbacks | +| `prov/xcatd-{destiny,postscript,credentials,policy}.conf` | TLS 3001: identity by reverse lookup, the node's script, credentials, policy | +| `prov/monitor.conf` | TCP 3002, the install monitor | +| `prov/ordering.conf` | an unreachable master, a missing PTR, a replaced state | + +A node has one netboot method, so the four `tftp-*.conf` files are +alternatives: the fixture runs the one the node is defined with. + +Not covered: the `getcredentials` form that asks xcatd to sign a CSR, and an +operating system install. The install tree is fabricated; the scenarios fetch +what a node fetches and assert what it contains. + +## What it does to the host + +Nothing. It opens sockets, starts `dig`, `curl` and `tftp`, and writes only +temporary files it removes. It never runs `ip`, `dhclient` or `nmcli`, and +never configures a leased address. + +The addresses it binds to must exist before a run. A `findme` from a port +below 1024 and the `getcredentials` callback listener on port 300 need root. +Setting that up and putting it back is the fixtures' job: +`xCAT-test/autotest/testcase/blackboxtest/{dhcp,prov}fixture.sh`. + +## Failure output + +TAP version 13, which `prove` and xcattest read: + +``` +not ok 3 - pxe-bios-x86/bios-discover: bootfile == pxelinux.0 + --- + expected: 'pxelinux.0' + received: 'xcat/xnba.kpxe' + reply: OFFER from 10.99.0.1 (9a:3d:4f:3d:59:1b) xid=0xc45c6376 yiaddr=10.99.0.175 siaddr=10.99.0.1 file='' + options: {1 (subnet_mask)=255.255.255.0, 54 (server_id)=10.99.0.1, 67 (bootfile_name)=xcat/xnba.kpxe} + sent: DISCOVER mac=02:da:16:d0:80:dd xid=0xc45c6376 client_arch=0x0000 + attempt: 1 of 3 + ... +``` + +`sent` is what went out, so a service step can be re-run by hand with a copy +and a paste. + +## Running under xcattest + +`xCAT-test/autotest/testcase/blackboxtest/`: + +| File | Label | What runs | +| --- | --- | --- | +| `cases0` | `ci_test` | the unit tests and `validate`; a checkout is enough | +| `prov_cases` | `prov_wire` | the `conf/prov/` scenarios, through `provfixture.sh` | +| `dhcp_cases` | `dhcp_wire` | the `conf/dhcp/` scenarios, through `dhcpfixture.sh` | + +Each wire case sets its fixture up, runs, and tears it down from a `trap`. +The wire cases carry no `ci_test`: while one runs, the management node is +reconfigured. `github_action_xcat_test.pl` runs `prov_wire` after the +`ci_test` set, then `dhcp_wire` once per DHCP backend installed: + +```bash +FIX=/opt/xcat/share/xcat/tools/autotest/testcase/blackboxtest/dhcpfixture.sh +for backend in $($FIX backends); do + $FIX backend-setup $backend + xcattest -t $(xcattest -s "dhcp_wire" -l | paste -sd,) + $FIX backend-teardown $backend +done +``` + +A case that cannot run says so and passes. Read a pass as coverage only when +the log shows the `ok` lines. + +## Tests + +```bash +cd xCAT-test/blackboxtest +python3 -m unittest discover -s tests +python3 src/blackboxtest validate conf/*/*.conf +``` + +No root, no network. Tests that need Scapy, `dig` or `tftp` skip without them. +`tests/test_boundaries.py` enforces the rules the results depend on: only +`proc` starts a process, only `dhcp` imports Scapy, and no module names an +xCAT command or a DHCP server implementation. + +## Layout + +``` +src/blackboxtest entry point +src/blackboxtest_lib/ + cli.py run, validate, list + config.py .conf -> Scenario, Step + steps.py what each step type takes and returns + validate.py the offline checks + runner.py runs scenarios; the service and local steps + dhcp.py the DHCP client (the only Scapy user) + dhcpopts.py DHCP option coding, the decoded reply + dnsc.py httpc.py tftpc.py dig, curl, tftp + xcatc.py xcatd: TLS 3001, UDP 3001, TCP 3002, callbacks + proc.py the only module that starts a process + assertions.py the assertion language + subst.py $step.field + model.py Scenario, Step, Assertion, Reply + report.py TAP + netutil.py addresses, MACs, the names loaders ask for + errors.py exceptions and exit codes +conf/dhcp/, conf/prov/ shipped scenarios +tests/ unit tests, standard library only +``` diff --git a/xCAT-test/dhcptest/conf/bootp-client.conf b/xCAT-test/blackboxtest/conf/dhcp/bootp-client.conf similarity index 84% rename from xCAT-test/dhcptest/conf/bootp-client.conf rename to xCAT-test/blackboxtest/conf/dhcp/bootp-client.conf index 14fe3bde7..f03587126 100644 --- a/xCAT-test/dhcptest/conf/bootp-client.conf +++ b/xCAT-test/blackboxtest/conf/dhcp/bootp-client.conf @@ -9,8 +9,8 @@ # # Scenario names carry the review number of the spec scenario they assert. # -# dhcptest run -i eth1 --set bootp_mac=02:00:de:ad:b0:07 \ -# --set pool=10.0.0.200-10.0.0.250 conf/bootp-client.conf +# blackboxtest run -i eth1 --set bootp_mac=02:00:de:ad:b0:07 \ +# --set pool=10.0.0.200-10.0.0.250 conf/dhcp/bootp-client.conf [scenario bootp-client-is-served] description = S-54: A BOOTP-only client is given an address out of the dynamic range diff --git a/xCAT-test/dhcptest/conf/discover-offer.conf b/xCAT-test/blackboxtest/conf/dhcp/discover-offer.conf similarity index 79% rename from xCAT-test/dhcptest/conf/discover-offer.conf rename to xCAT-test/blackboxtest/conf/dhcp/discover-offer.conf index 1b25ed95c..941e5f1bb 100644 --- a/xCAT-test/dhcptest/conf/discover-offer.conf +++ b/xCAT-test/blackboxtest/conf/dhcp/discover-offer.conf @@ -1,7 +1,7 @@ # A DISCOVER draws exactly one OFFER, from the server we expect. # -# dhcptest run -i eth1 --set net=10.0.0.0/24 --set server=10.0.0.1 \ -# conf/discover-offer.conf +# blackboxtest run -i eth1 --set net=10.0.0.0/24 --set server=10.0.0.1 \ +# conf/dhcp/discover-offer.conf [defaults] timeout = 2 diff --git a/xCAT-test/dhcptest/conf/discovery-adoption.conf b/xCAT-test/blackboxtest/conf/dhcp/discovery-adoption.conf similarity index 87% rename from xCAT-test/dhcptest/conf/discovery-adoption.conf rename to xCAT-test/blackboxtest/conf/dhcp/discovery-adoption.conf index 9db5ac8ee..3038342ab 100644 --- a/xCAT-test/dhcptest/conf/discovery-adoption.conf +++ b/xCAT-test/blackboxtest/conf/dhcp/discovery-adoption.conf @@ -12,13 +12,13 @@ # The two scenarios run separately, against the same MAC, with the node defined # in between: # -# dhcptest run -i eth1 -s machine-is-unknown \ +# blackboxtest run -i eth1 -s machine-is-unknown \ # --set adopt_mac=02:00:dc:11:00:aa --set pool=10.0.0.200-10.0.0.250 \ -# --set adopt_ip=10.0.0.12 conf/discovery-adoption.conf +# --set adopt_ip=10.0.0.12 conf/dhcp/discovery-adoption.conf # ... define the node, run makedhcp for it, restart nothing ... -# dhcptest run -i eth1 -s machine-has-been-adopted \ +# blackboxtest run -i eth1 -s machine-has-been-adopted \ # --set adopt_mac=02:00:dc:11:00:aa --set pool=10.0.0.200-10.0.0.250 \ -# --set adopt_ip=10.0.0.12 conf/discovery-adoption.conf +# --set adopt_ip=10.0.0.12 conf/dhcp/discovery-adoption.conf [defaults] timeout = 3 diff --git a/xCAT-test/dhcptest/conf/discovery-bootfile.conf b/xCAT-test/blackboxtest/conf/dhcp/discovery-bootfile.conf similarity index 93% rename from xCAT-test/dhcptest/conf/discovery-bootfile.conf rename to xCAT-test/blackboxtest/conf/dhcp/discovery-bootfile.conf index a7eb022af..53033a603 100644 --- a/xCAT-test/dhcptest/conf/discovery-bootfile.conf +++ b/xCAT-test/blackboxtest/conf/dhcp/discovery-bootfile.conf @@ -7,11 +7,11 @@ # A file of its own rather than a third scenario in provision-vs-discovery.conf # only because it needs no node defined. # -# dhcptest run -i eth1 \ +# blackboxtest run -i eth1 \ # --set unknown_mac=02:00:de:ad:be:ef \ # --set pool=10.0.0.200-10.0.0.250 \ # --set discovery_loader=pxelinux.0 \ -# conf/discovery-bootfile.conf +# conf/dhcp/discovery-bootfile.conf [scenario unknown-node-is-told-what-to-boot] description = S-55, S-56: A machine the server does not know is still handed a loader diff --git a/xCAT-test/dhcptest/conf/dynamic-range-cidr.conf b/xCAT-test/blackboxtest/conf/dhcp/dynamic-range-cidr.conf similarity index 87% rename from xCAT-test/dhcptest/conf/dynamic-range-cidr.conf rename to xCAT-test/blackboxtest/conf/dhcp/dynamic-range-cidr.conf index 449f0a49f..c2ba24536 100644 --- a/xCAT-test/dhcptest/conf/dynamic-range-cidr.conf +++ b/xCAT-test/blackboxtest/conf/dhcp/dynamic-range-cidr.conf @@ -7,8 +7,8 @@ # Asserts what static-vs-dynamic.conf does, against a range in the other # notation. # -# dhcptest run -i eth1 --set unknown_mac=02:00:de:ad:be:ef \ -# --set pool=10.0.0.192/26 conf/dynamic-range-cidr.conf +# blackboxtest run -i eth1 --set unknown_mac=02:00:de:ad:be:ef \ +# --set pool=10.0.0.192/26 conf/dhcp/dynamic-range-cidr.conf [scenario cidr-range-is-served] description = S-05: A range written in CIDR notation is offered from just the same diff --git a/xCAT-test/dhcptest/conf/full-lease.conf b/xCAT-test/blackboxtest/conf/dhcp/full-lease.conf similarity index 90% rename from xCAT-test/dhcptest/conf/full-lease.conf rename to xCAT-test/blackboxtest/conf/dhcp/full-lease.conf index 51e7ec29c..d9bacf584 100644 --- a/xCAT-test/dhcptest/conf/full-lease.conf +++ b/xCAT-test/blackboxtest/conf/dhcp/full-lease.conf @@ -1,7 +1,7 @@ # The full four-way handshake: the address that was offered is the address # that gets acknowledged. # -# dhcptest run -i eth1 --set net=10.0.0.0/24 conf/full-lease.conf +# blackboxtest run -i eth1 --set net=10.0.0.0/24 conf/dhcp/full-lease.conf [scenario full-lease] description = DISCOVER/OFFER/REQUEST/ACK yields the offered address diff --git a/xCAT-test/dhcptest/conf/hierarchy-dhcpserver.conf b/xCAT-test/blackboxtest/conf/dhcp/hierarchy-dhcpserver.conf similarity index 95% rename from xCAT-test/dhcptest/conf/hierarchy-dhcpserver.conf rename to xCAT-test/blackboxtest/conf/dhcp/hierarchy-dhcpserver.conf index 08ab30435..64c6ab7c6 100644 --- a/xCAT-test/dhcptest/conf/hierarchy-dhcpserver.conf +++ b/xCAT-test/blackboxtest/conf/dhcp/hierarchy-dhcpserver.conf @@ -13,13 +13,13 @@ # Run against the machine that gave the pool away, not the one that took it; # against the delegate, run provision-vs-discovery.conf instead. # -# dhcptest run -i eth1 \ +# blackboxtest run -i eth1 \ # --set node_mac=52:54:00:aa:bb:01 \ # --set node_ip=10.0.0.11 \ # --set node_loader=/boot/grub2/grub2-cn1 \ # --set delegate=10.0.0.5 \ # --set unknown_mac=02:00:de:ad:be:ef \ -# conf/hierarchy-dhcpserver.conf +# conf/dhcp/hierarchy-dhcpserver.conf [scenario delegated-pool-is-not-served-here] description = S-40: An unknown MAC gets nothing: the dynamic pool belongs to another server diff --git a/xCAT-test/dhcptest/conf/http-port.conf b/xCAT-test/blackboxtest/conf/dhcp/http-port.conf similarity index 91% rename from xCAT-test/dhcptest/conf/http-port.conf rename to xCAT-test/blackboxtest/conf/dhcp/http-port.conf index bf6e6742d..d40c6473a 100644 --- a/xCAT-test/dhcptest/conf/http-port.conf +++ b/xCAT-test/blackboxtest/conf/dhcp/http-port.conf @@ -13,9 +13,9 @@ # # Scenario names carry the review number of the spec scenario they assert. # -# dhcptest run -i eth1 --set tftp=10.0.0.1 --set httpport=8080 \ +# blackboxtest run -i eth1 --set tftp=10.0.0.1 --set httpport=8080 \ # --set riscv64_loader=boot/grub2/grub2.riscv64 \ -# conf/http-port.conf +# conf/dhcp/http-port.conf [defaults] request_options = 1, 3, 6, 54, 60, 66, 67, 209, 239 diff --git a/xCAT-test/dhcptest/conf/ipxe-userclass.conf b/xCAT-test/blackboxtest/conf/dhcp/ipxe-userclass.conf similarity index 96% rename from xCAT-test/dhcptest/conf/ipxe-userclass.conf rename to xCAT-test/blackboxtest/conf/dhcp/ipxe-userclass.conf index 8dbc5f77c..f4a744bfd 100644 --- a/xCAT-test/dhcptest/conf/ipxe-userclass.conf +++ b/xCAT-test/blackboxtest/conf/dhcp/ipxe-userclass.conf @@ -12,11 +12,11 @@ # # Scenario names carry the review number of the spec scenario they assert. # -# dhcptest run -i eth1 --set user_class=xNBA \ +# blackboxtest run -i eth1 --set user_class=xNBA \ # --set stage1_loader=xcat/xnba.kpxe \ # --set stage2_loader=http://10.0.0.1/tftpboot/xcat/xnba/nets/10.0.0.0_24 \ # --set stage1_uefi_loader=xcat/xnba.efi \ -# conf/ipxe-userclass.conf +# conf/dhcp/ipxe-userclass.conf [defaults] client_arch = 0x0000 diff --git a/xCAT-test/dhcptest/conf/iscsi-root-path.conf b/xCAT-test/blackboxtest/conf/dhcp/iscsi-root-path.conf similarity index 95% rename from xCAT-test/dhcptest/conf/iscsi-root-path.conf rename to xCAT-test/blackboxtest/conf/dhcp/iscsi-root-path.conf index 247957f41..73073ec55 100644 --- a/xCAT-test/dhcptest/conf/iscsi-root-path.conf +++ b/xCAT-test/blackboxtest/conf/dhcp/iscsi-root-path.conf @@ -7,10 +7,10 @@ # # Scenario names carry the review number of the spec scenario they assert. # -# dhcptest run -i eth1 \ +# blackboxtest run -i eth1 \ # --set iscsi_mac=02:00:dc:11:00:51 --set iscsi_ip=10.0.0.51 \ # --set root_path=iscsi:10.0.0.1:6:3260:0:iqn.2024-01.cluster:target0 \ -# conf/iscsi-root-path.conf +# conf/dhcp/iscsi-root-path.conf [defaults] client_arch = 0x0000 diff --git a/xCAT-test/dhcptest/conf/loader-absent.conf b/xCAT-test/blackboxtest/conf/dhcp/loader-absent.conf similarity index 94% rename from xCAT-test/dhcptest/conf/loader-absent.conf rename to xCAT-test/blackboxtest/conf/dhcp/loader-absent.conf index 615f2ae6e..030535cd2 100644 --- a/xCAT-test/dhcptest/conf/loader-absent.conf +++ b/xCAT-test/blackboxtest/conf/dhcp/loader-absent.conf @@ -11,9 +11,9 @@ # # Scenario names carry the review number of the spec scenario they assert. # -# dhcptest run -i eth1 --set pool=10.0.0.200-10.0.0.250 \ +# blackboxtest run -i eth1 --set pool=10.0.0.200-10.0.0.250 \ # --set present_loader=xcat/xnba.efi \ -# conf/loader-absent.conf +# conf/dhcp/loader-absent.conf [defaults] request_options = 1, 3, 6, 54, 66, 67 diff --git a/xCAT-test/dhcptest/conf/localboot.conf b/xCAT-test/blackboxtest/conf/dhcp/localboot.conf similarity index 97% rename from xCAT-test/dhcptest/conf/localboot.conf rename to xCAT-test/blackboxtest/conf/dhcp/localboot.conf index 1f6097ecc..fadf55403 100644 --- a/xCAT-test/dhcptest/conf/localboot.conf +++ b/xCAT-test/blackboxtest/conf/dhcp/localboot.conf @@ -16,13 +16,13 @@ # as an xNBA second stage and is answered with the *network's* script. The script # URL is still supplied so the failure output can name what was wrongly sent: # -# dhcptest run -i eth1 \ +# blackboxtest run -i eth1 \ # --set booted_mac=02:00:dc:11:00:61 \ # --set booted_ip=10.0.0.61 \ # --set booted_script=http://10.0.0.1/tftpboot/xcat/xnba/nodes/dhcptestboot \ # --set booted_pxe_mac=02:00:dc:11:00:62 \ # --set booted_pxe_ip=10.0.0.62 \ -# conf/localboot.conf +# conf/dhcp/localboot.conf [defaults] request_options = 1, 3, 6, 43, 54, 60, 66, 67 diff --git a/xCAT-test/dhcptest/conf/multi-mac-node.conf b/xCAT-test/blackboxtest/conf/dhcp/multi-mac-node.conf similarity index 95% rename from xCAT-test/dhcptest/conf/multi-mac-node.conf rename to xCAT-test/blackboxtest/conf/dhcp/multi-mac-node.conf index f972010b3..753adf75b 100644 --- a/xCAT-test/dhcptest/conf/multi-mac-node.conf +++ b/xCAT-test/blackboxtest/conf/dhcp/multi-mac-node.conf @@ -10,10 +10,10 @@ # # Scenario names carry the review number of the spec scenario they assert. # -# dhcptest run -i eth1 \ +# blackboxtest run -i eth1 \ # --set first_mac=02:00:dc:11:00:41 --set first_ip=10.0.0.41 \ # --set second_mac=02:00:dc:11:00:42 --set second_ip=10.0.0.42 \ -# conf/multi-mac-node.conf +# conf/dhcp/multi-mac-node.conf [defaults] client_arch = 0x0000 diff --git a/xCAT-test/dhcptest/conf/nak-foreign-address.conf b/xCAT-test/blackboxtest/conf/dhcp/nak-foreign-address.conf similarity index 89% rename from xCAT-test/dhcptest/conf/nak-foreign-address.conf rename to xCAT-test/blackboxtest/conf/dhcp/nak-foreign-address.conf index 64b563aad..5b89ae90b 100644 --- a/xCAT-test/dhcptest/conf/nak-foreign-address.conf +++ b/xCAT-test/blackboxtest/conf/dhcp/nak-foreign-address.conf @@ -7,7 +7,7 @@ # DISCOVER. Silence leaves it retrying an address it will never get, which on a # provisioning network looks exactly like a node that will not boot. # -# dhcptest run -i eth1 --set foreign_ip=192.0.2.77 conf/nak-foreign-address.conf +# blackboxtest run -i eth1 --set foreign_ip=192.0.2.77 conf/dhcp/nak-foreign-address.conf [scenario foreign-address-is-refused] description = S-53: A request for an address off this network is refused, not ignored diff --git a/xCAT-test/dhcptest/conf/netboot-methods.conf b/xCAT-test/blackboxtest/conf/dhcp/netboot-methods.conf similarity index 98% rename from xCAT-test/dhcptest/conf/netboot-methods.conf rename to xCAT-test/blackboxtest/conf/dhcp/netboot-methods.conf index 8e0f77a1b..c889b0534 100644 --- a/xCAT-test/dhcptest/conf/netboot-methods.conf +++ b/xCAT-test/blackboxtest/conf/dhcp/netboot-methods.conf @@ -7,10 +7,10 @@ # # Scenario names carry the review number of the spec scenario they assert. # -# dhcptest run -i eth1 \ +# blackboxtest run -i eth1 \ # --set xnba_mac=02:00:dc:11:00:21 --set xnba_node=cn21 \ # --set xnba_loader=xcat/xnba.kpxe \ -# ... conf/netboot-methods.conf +# ... conf/dhcp/netboot-methods.conf [defaults] client_arch = 0x0000 diff --git a/xCAT-test/dhcptest/conf/next-server-source.conf b/xCAT-test/blackboxtest/conf/dhcp/next-server-source.conf similarity index 96% rename from xCAT-test/dhcptest/conf/next-server-source.conf rename to xCAT-test/blackboxtest/conf/dhcp/next-server-source.conf index 37fc1989d..bf050e5d7 100644 --- a/xCAT-test/dhcptest/conf/next-server-source.conf +++ b/xCAT-test/blackboxtest/conf/dhcp/next-server-source.conf @@ -12,14 +12,14 @@ # # Scenario names carry the review number of the spec scenario they assert. # -# dhcptest run -i eth1 \ +# blackboxtest run -i eth1 \ # --set tftp_mac=02:00:dc:11:00:31 --set tftp_ip=10.0.0.31 \ # --set tftp_server=10.0.0.61 \ # --set xcm_mac=02:00:dc:11:00:32 --set xcm_ip=10.0.0.32 \ # --set xcm_server=10.0.0.62 \ # --set sub_mac=02:00:dc:11:00:33 --set sub_ip=10.0.0.33 \ # --set sub_server=10.0.0.1 \ -# conf/next-server-source.conf +# conf/dhcp/next-server-source.conf [defaults] client_arch = 0x0000 diff --git a/xCAT-test/dhcptest/conf/node-removal.conf b/xCAT-test/blackboxtest/conf/dhcp/node-removal.conf similarity index 90% rename from xCAT-test/dhcptest/conf/node-removal.conf rename to xCAT-test/blackboxtest/conf/dhcp/node-removal.conf index 5de063de4..7c51154c4 100644 --- a/xCAT-test/dhcptest/conf/node-removal.conf +++ b/xCAT-test/blackboxtest/conf/dhcp/node-removal.conf @@ -10,9 +10,9 @@ # # Scenario names carry the review number of the spec scenario they assert. # -# dhcptest run -i eth1 -s removed-node-keeps-its-reservation-until-it-is-withdrawn \ +# blackboxtest run -i eth1 -s removed-node-keeps-its-reservation-until-it-is-withdrawn \ # --set removed_mac=02:00:dc:11:00:61 --set removed_ip=10.0.0.61 \ -# --set pool=10.0.0.200-10.0.0.250 conf/node-removal.conf +# --set pool=10.0.0.200-10.0.0.250 conf/dhcp/node-removal.conf [defaults] request_options = 1, 3, 6, 54, 67 diff --git a/xCAT-test/dhcptest/conf/provision-vs-discovery.conf b/xCAT-test/blackboxtest/conf/dhcp/provision-vs-discovery.conf similarity index 98% rename from xCAT-test/dhcptest/conf/provision-vs-discovery.conf rename to xCAT-test/blackboxtest/conf/dhcp/provision-vs-discovery.conf index 3bbf639bb..9b663f95a 100644 --- a/xCAT-test/dhcptest/conf/provision-vs-discovery.conf +++ b/xCAT-test/blackboxtest/conf/dhcp/provision-vs-discovery.conf @@ -13,7 +13,7 @@ # # Every address and filename is supplied on the command line: # -# dhcptest run -i eth1 \ +# blackboxtest run -i eth1 \ # --set node_mac=52:54:00:aa:bb:01 \ # --set node_ip=10.0.0.11 \ # --set node_loader=/boot/grub2/grub2-cn1 \ @@ -23,7 +23,7 @@ # --set gateway=10.0.0.1 --set nameservers=10.0.0.1 \ # --set domain=cluster --set mtu=1500 --set lease=43200 \ # --set cumulus_url=http://10.0.0.1/install/postscripts/cumulusztp \ -# conf/provision-vs-discovery.conf +# conf/dhcp/provision-vs-discovery.conf [defaults] client_arch = 0x0000 diff --git a/xCAT-test/dhcptest/conf/pxe-arch-matrix.conf b/xCAT-test/blackboxtest/conf/dhcp/pxe-arch-matrix.conf similarity index 98% rename from xCAT-test/dhcptest/conf/pxe-arch-matrix.conf rename to xCAT-test/blackboxtest/conf/dhcp/pxe-arch-matrix.conf index 02784b458..f4a9fdd95 100644 --- a/xCAT-test/dhcptest/conf/pxe-arch-matrix.conf +++ b/xCAT-test/blackboxtest/conf/dhcp/pxe-arch-matrix.conf @@ -9,12 +9,12 @@ # Expected filenames come from the command line so this file stays a statement # about the wire: # -# dhcptest run -i eth1 --set tftp=10.0.0.1 \ +# blackboxtest run -i eth1 --set tftp=10.0.0.1 \ # --set bios_loader=pxelinux.0 \ # --set uefi_loader=xcat/xnba.efi \ # --set aarch64_loader=boot/grub2/grub2.aarch64 \ # --set riscv64_loader=boot/grub2/grub2.riscv64 \ -# conf/pxe-arch-matrix.conf +# conf/dhcp/pxe-arch-matrix.conf # # Every scenario here is a machine nobody has defined, so the answer has to come # from the subnet. That is the discovery case, which has to work for a machine to diff --git a/xCAT-test/dhcptest/conf/renew-rebind.conf b/xCAT-test/blackboxtest/conf/dhcp/renew-rebind.conf similarity index 93% rename from xCAT-test/dhcptest/conf/renew-rebind.conf rename to xCAT-test/blackboxtest/conf/dhcp/renew-rebind.conf index 35dd0f0e2..f2e7f1e07 100644 --- a/xCAT-test/dhcptest/conf/renew-rebind.conf +++ b/xCAT-test/blackboxtest/conf/dhcp/renew-rebind.conf @@ -5,7 +5,7 @@ # it at Layer 2, and the tool answers ARP for that address itself, only while # the scenario is running and only for addresses this session was granted. # -# dhcptest run -i eth1 --set net=10.0.0.0/24 conf/renew-rebind.conf +# blackboxtest run -i eth1 --set net=10.0.0.0/24 conf/dhcp/renew-rebind.conf [scenario renew-and-rebind] description = A granted lease can be renewed by unicast and rebound by broadcast diff --git a/xCAT-test/dhcptest/conf/static-vs-dynamic.conf b/xCAT-test/blackboxtest/conf/dhcp/static-vs-dynamic.conf similarity index 96% rename from xCAT-test/dhcptest/conf/static-vs-dynamic.conf rename to xCAT-test/blackboxtest/conf/dhcp/static-vs-dynamic.conf index b37416019..ad9d7b8f1 100644 --- a/xCAT-test/dhcptest/conf/static-vs-dynamic.conf +++ b/xCAT-test/blackboxtest/conf/dhcp/static-vs-dynamic.conf @@ -9,12 +9,12 @@ # # Every address comes from the command line: # -# dhcptest run -i eth1 \ +# blackboxtest run -i eth1 \ # --set reserved_mac=52:54:00:aa:bb:cc \ # --set reserved_ip=10.0.0.51 \ # --set unreserved_mac=02:00:de:ad:be:ef \ # --set pool=10.0.0.100/26 \ -# conf/static-vs-dynamic.conf +# conf/dhcp/static-vs-dynamic.conf [scenario reserved-mac] description = S-01: A reserved MAC is offered and acknowledged its reserved address diff --git a/xCAT-test/provtest/conf/discovery-artefacts.conf b/xCAT-test/blackboxtest/conf/prov/discovery-artefacts.conf similarity index 98% rename from xCAT-test/provtest/conf/discovery-artefacts.conf rename to xCAT-test/blackboxtest/conf/prov/discovery-artefacts.conf index 96ae9abff..6b58d1f37 100644 --- a/xCAT-test/provtest/conf/discovery-artefacts.conf +++ b/xCAT-test/blackboxtest/conf/prov/discovery-artefacts.conf @@ -12,12 +12,12 @@ # selected with `-s` rather than skipped from inside, so a scenario that does not # run says so by its absence and never by a pass. # -# provtest run \ +# blackboxtest run \ # --set server=10.99.1.1 --set client=10.99.1.11 \ # --set hexnet=0A6301 --set netfile=10.99.1.0_24 \ # --set master=10.99.1.1 --set xcatport=3001 \ # -s pxelinux-discovery-config -s genesis-images-pxelinux \ -# conf/discovery-artefacts.conf +# conf/prov/discovery-artefacts.conf [defaults] server = %(server)s diff --git a/xCAT-test/provtest/conf/dns-removal.conf b/xCAT-test/blackboxtest/conf/prov/dns-removal.conf similarity index 95% rename from xCAT-test/provtest/conf/dns-removal.conf rename to xCAT-test/blackboxtest/conf/prov/dns-removal.conf index a566b6741..9ef5c202e 100644 --- a/xCAT-test/provtest/conf/dns-removal.conf +++ b/xCAT-test/blackboxtest/conf/prov/dns-removal.conf @@ -5,11 +5,11 @@ # opposite assertions about the same names, and a suite that ran both against # one state would always fail one of them. # -# provtest run \ +# blackboxtest run \ # --set server=10.99.1.1 --set client=10.99.1.11 \ # --set node=provtestcn --set domain=provtest.cluster \ # --set revname=11.1.99.10.in-addr.arpa \ -# conf/dns-removal.conf +# conf/prov/dns-removal.conf [defaults] server = %(server)s diff --git a/xCAT-test/provtest/conf/dns.conf b/xCAT-test/blackboxtest/conf/prov/dns.conf similarity index 98% rename from xCAT-test/provtest/conf/dns.conf rename to xCAT-test/blackboxtest/conf/prov/dns.conf index f64e98bf9..1bee0fae8 100644 --- a/xCAT-test/provtest/conf/dns.conf +++ b/xCAT-test/blackboxtest/conf/prov/dns.conf @@ -8,14 +8,14 @@ # Every expected value is supplied with --set: this file states that a record # exists and holds a given value, and nothing about how named was configured. # -# provtest run \ +# blackboxtest run \ # --set server=10.99.1.1 --set client=10.99.1.11 \ # --set node=provtestcn --set domain=provtest.cluster \ # --set nodeip=10.99.1.11 --set revname=11.1.99.10.in-addr.arpa \ # --set alias=provtestcn-eth0 --set master=10.99.1.1 \ # --set missing=nosuchnode --set forwarded=example.com \ # --set net=10.99.1.0/24 \ -# conf/dns.conf +# conf/prov/dns.conf [defaults] server = %(server)s diff --git a/xCAT-test/provtest/conf/findme.conf b/xCAT-test/blackboxtest/conf/prov/findme.conf similarity index 98% rename from xCAT-test/provtest/conf/findme.conf rename to xCAT-test/blackboxtest/conf/prov/findme.conf index 0482eed99..5e681ec9a 100644 --- a/xCAT-test/provtest/conf/findme.conf +++ b/xCAT-test/blackboxtest/conf/prov/findme.conf @@ -10,10 +10,10 @@ # # Nothing here signs the packet. Nothing in xcatd checks a signature. # -# provtest run \ +# blackboxtest run \ # --set server=10.99.1.1 --set client=10.99.1.11 \ # --set foreign=10.98.1.11 \ -# conf/findme.conf +# conf/prov/findme.conf [defaults] server = %(server)s diff --git a/xCAT-test/provtest/conf/flowcontrol.conf b/xCAT-test/blackboxtest/conf/prov/flowcontrol.conf similarity index 93% rename from xCAT-test/provtest/conf/flowcontrol.conf rename to xCAT-test/blackboxtest/conf/prov/flowcontrol.conf index 752bbed19..bbd13ad98 100644 --- a/xCAT-test/provtest/conf/flowcontrol.conf +++ b/xCAT-test/blackboxtest/conf/prov/flowcontrol.conf @@ -7,8 +7,8 @@ # the second says there is room. A server that sends the first and never the # second leaves every node in the cluster waiting silently. # -# provtest run --set server=10.99.1.1 --set client=10.99.1.11 \ -# conf/flowcontrol.conf +# blackboxtest run --set server=10.99.1.1 --set client=10.99.1.11 \ +# conf/prov/flowcontrol.conf [defaults] server = %(server)s diff --git a/xCAT-test/provtest/conf/http.conf b/xCAT-test/blackboxtest/conf/prov/http.conf similarity index 99% rename from xCAT-test/provtest/conf/http.conf rename to xCAT-test/blackboxtest/conf/prov/http.conf index 859ed2e2f..c8f6e155c 100644 --- a/xCAT-test/provtest/conf/http.conf +++ b/xCAT-test/blackboxtest/conf/prov/http.conf @@ -6,13 +6,13 @@ # daemon over another protocol, which is where the interesting failure lives -- # the node is told a name over TFTP and fetches it over HTTP. # -# provtest run \ +# blackboxtest run \ # --set server=10.99.1.1 --set client=10.99.1.11 \ # --set node=provtestcn --set httpport=80 --set otherport=8080 \ # --set hexip=0A63010B --set knownfile=boot/grub2/grub.cfg-0A63010B \ # --set installpath=autoinst/provtestcn --set repofile=repodata/repomd.xml \ # --set master=10.99.1.1 --set arch=x86_64 \ -# conf/http.conf +# conf/prov/http.conf [defaults] server = %(server)s diff --git a/xCAT-test/provtest/conf/monitor.conf b/xCAT-test/blackboxtest/conf/prov/monitor.conf similarity index 98% rename from xCAT-test/provtest/conf/monitor.conf rename to xCAT-test/blackboxtest/conf/prov/monitor.conf index 9db15703c..fb5921114 100644 --- a/xCAT-test/provtest/conf/monitor.conf +++ b/xCAT-test/blackboxtest/conf/prov/monitor.conf @@ -6,10 +6,10 @@ # peer it cannot name has its connection closed with no greeting at all, which is # what makes the greeting assertable. # -# provtest run \ +# blackboxtest run \ # --set server=10.99.1.1 --set client=10.99.1.11 \ # --set unknown=10.99.1.201 --set monitorport=3002 --set xcatport=3001 \ -# conf/monitor.conf +# conf/prov/monitor.conf [defaults] server = %(server)s diff --git a/xCAT-test/provtest/conf/ordering.conf b/xCAT-test/blackboxtest/conf/prov/ordering.conf similarity index 98% rename from xCAT-test/provtest/conf/ordering.conf rename to xCAT-test/blackboxtest/conf/prov/ordering.conf index ca7cc1241..7a8757cad 100644 --- a/xCAT-test/provtest/conf/ordering.conf +++ b/xCAT-test/blackboxtest/conf/prov/ordering.conf @@ -10,12 +10,12 @@ # nodeset to be had. The fixture drives P-75 first and P-74 last, selecting # scenarios with -s rather than running them all at once. # -# provtest run \ +# blackboxtest run \ # --set server=10.99.1.1 --set client=10.99.1.11 \ # --set node=provtestcn --set hexip=0A63010B \ # --set xcatport=3001 --set unreachable=10.99.1.250 \ # --set destiny=install --set second=boot \ -# conf/ordering.conf +# conf/prov/ordering.conf [defaults] server = %(server)s diff --git a/xCAT-test/provtest/conf/tftp-grub2.conf b/xCAT-test/blackboxtest/conf/prov/tftp-grub2.conf similarity index 93% rename from xCAT-test/provtest/conf/tftp-grub2.conf rename to xCAT-test/blackboxtest/conf/prov/tftp-grub2.conf index e186ea16a..178250dcb 100644 --- a/xCAT-test/provtest/conf/tftp-grub2.conf +++ b/xCAT-test/blackboxtest/conf/prov/tftp-grub2.conf @@ -9,14 +9,14 @@ # read out of it, and *that* path is fetched. Nothing here says what the kernel # will be called, which is what stops it asserting xCAT against xCAT. # -# provtest run \ +# blackboxtest run \ # --set server=10.99.1.1 --set client=10.99.1.11 \ # --set node=provtestcn --set hexip=0A63010B \ # --set macdashes=52-54-00-dc-11-01 --set mac=52:54:00:dc:11:01 \ # --set loader=boot/grub2/grub2.x86_64 \ # --set master=10.99.1.1 --set xcatport=3001 --set destiny=install \ # --set bootfile=boot/grub2/grub2-provtestcn \ -# conf/tftp-grub2.conf +# conf/prov/tftp-grub2.conf [defaults] server = %(server)s @@ -150,10 +150,9 @@ description = The boot file DHCP names is fetchable under exactly that name [step named] type = tftp -# Spelled by the caller, because nothing here speaks DHCP: this is the name a -# grub2 node's offer carries, and the pair of suites divides the claim -- dhcptest -# asserts that xCAT sends this name, and this asserts that the name sent can be -# fetched. Neither one alone is enough to boot a node. +# Spelled by the caller: this is the name a grub2 node's offer carries. The +# claim is split -- conf/dhcp/ asserts that xCAT sends this name, and this +# asserts that the name sent can be fetched. Neither alone boots a node. path = %(bootfile)s assert = size > 0 diff --git a/xCAT-test/provtest/conf/tftp-petitboot.conf b/xCAT-test/blackboxtest/conf/prov/tftp-petitboot.conf similarity index 96% rename from xCAT-test/provtest/conf/tftp-petitboot.conf rename to xCAT-test/blackboxtest/conf/prov/tftp-petitboot.conf index 0c7bed110..51c94708f 100644 --- a/xCAT-test/provtest/conf/tftp-petitboot.conf +++ b/xCAT-test/blackboxtest/conf/prov/tftp-petitboot.conf @@ -6,11 +6,11 @@ # file and must stay one file -- a copy that drifts is a node that boots what it # was set to three nodesets ago. # -# provtest run \ +# blackboxtest run \ # --set server=10.99.1.1 --set client=10.99.1.11 \ # --set node=provtestcn --set hexip=0A63010B \ # --set master=10.99.1.1 --set xcatport=3001 --set destiny=install \ -# conf/tftp-petitboot.conf +# conf/prov/tftp-petitboot.conf [defaults] server = %(server)s diff --git a/xCAT-test/provtest/conf/tftp-pxelinux.conf b/xCAT-test/blackboxtest/conf/prov/tftp-pxelinux.conf similarity index 93% rename from xCAT-test/provtest/conf/tftp-pxelinux.conf rename to xCAT-test/blackboxtest/conf/prov/tftp-pxelinux.conf index c23c31fe6..b37734ee6 100644 --- a/xCAT-test/provtest/conf/tftp-pxelinux.conf +++ b/xCAT-test/blackboxtest/conf/prov/tftp-pxelinux.conf @@ -8,12 +8,12 @@ # A separate file from tftp-grub2.conf, not a branch inside it: a node has one # netboot method, and running both sets against one node would always fail one. # -# provtest run \ +# blackboxtest run \ # --set server=10.99.1.1 --set client=10.99.1.11 \ # --set node=provtestcn --set hexip=0A63010B \ # --set master=10.99.1.1 --set xcatport=3001 --set destiny=install \ # --set bootnode=provtestbn \ -# conf/tftp-pxelinux.conf +# conf/prov/tftp-pxelinux.conf [defaults] server = %(server)s @@ -66,7 +66,7 @@ assert = type = extract from = $config.text pattern = ^\s*KERNEL\s+\S -# The DHCP half is dhcptest S-31: an installed node must stop being handed a boot +# The DHCP half is S-31, in conf/dhcp/localboot.conf: an installed node must stop being handed a boot # file at all. This half says that if a config is served anyway it does not # netboot. Written as an extract that must find nothing, because "contains" has # no negative form and an absent line is not an absent field. diff --git a/xCAT-test/provtest/conf/tftp-xnba.conf b/xCAT-test/blackboxtest/conf/prov/tftp-xnba.conf similarity index 94% rename from xCAT-test/provtest/conf/tftp-xnba.conf rename to xCAT-test/blackboxtest/conf/prov/tftp-xnba.conf index b13ac6cb6..6f89ae568 100644 --- a/xCAT-test/provtest/conf/tftp-xnba.conf +++ b/xCAT-test/blackboxtest/conf/prov/tftp-xnba.conf @@ -11,10 +11,10 @@ # nothing xCAT named. The second scenario fetches the script again, because a # scenario depending on another having been selected cannot be run on its own. # -# provtest run \ +# blackboxtest run \ # --set server=10.99.1.1 --set client=10.99.1.11 \ # --set node=provtestcn --set httpport=80 \ -# conf/tftp-xnba.conf +# conf/prov/tftp-xnba.conf [defaults] server = %(server)s @@ -55,7 +55,7 @@ pattern = imgfetch\s+(?:-n\s+\S+\s+)?https?://[^/\s]+(/\S+) # The host is deliberately not asserted, because xnba does not write one: the # script says ${next-server}, which gpxe fills in from the DHCP acknowledgement, # so the node fetches from whoever booted it -- its service node in a -# hierarchical cluster. Which address arrives there is dhcptest's subject. +# hierarchical cluster. Which address arrives there is conf/dhcp/'s subject. assert = matched == yes diff --git a/xCAT-test/provtest/conf/xcatd-credentials.conf b/xCAT-test/blackboxtest/conf/prov/xcatd-credentials.conf similarity index 98% rename from xCAT-test/provtest/conf/xcatd-credentials.conf rename to xCAT-test/blackboxtest/conf/prov/xcatd-credentials.conf index 07c2d2fad..ba7b6cd42 100644 --- a/xCAT-test/provtest/conf/xcatd-credentials.conf +++ b/xCAT-test/blackboxtest/conf/prov/xcatd-credentials.conf @@ -13,11 +13,11 @@ # because a CSR is not something a wire test can fabricate meaningfully, and the # callback is the same either way. # -# provtest run \ +# blackboxtest run \ # --set server=10.99.1.1 --set client=10.99.1.11 \ # --set xcatport=3001 --set credtype=xcat_server_cred \ # --set highport=30300 \ -# conf/xcatd-credentials.conf +# conf/prov/xcatd-credentials.conf [defaults] server = %(server)s diff --git a/xCAT-test/provtest/conf/xcatd-destiny.conf b/xCAT-test/blackboxtest/conf/prov/xcatd-destiny.conf similarity index 96% rename from xCAT-test/provtest/conf/xcatd-destiny.conf rename to xCAT-test/blackboxtest/conf/prov/xcatd-destiny.conf index cd440583e..49cc4648e 100644 --- a/xCAT-test/provtest/conf/xcatd-destiny.conf +++ b/xCAT-test/blackboxtest/conf/prov/xcatd-destiny.conf @@ -2,7 +2,7 @@ # # The request a genesis image makes to find out what it is for. The genesis # script pipes three lines of XML into `openssl s_client` with no client -# certificate and reads the answer back with sed; provtest does the same over a +# certificate and reads the answer back with sed; blackboxtest does the same over a # socket, from a chosen source address, because the source address is the # identity: xcatd names the client by the reverse lookup of the address the # connection arrived from. @@ -10,12 +10,12 @@ # So `known` and `unknown` differ in one thing -- which address the request left # by -- and that is the whole authentication model on the wire. # -# provtest run \ +# blackboxtest run \ # --set server=10.99.1.1 --set client=10.99.1.11 \ # --set unknown=10.99.1.201 --set node=provtestcn \ # --set destiny=install --set master=10.99.1.1 --set xcatport=3001 \ # --set imgserver=10.99.1.2 --set next=boot \ -# conf/xcatd-destiny.conf +# conf/prov/xcatd-destiny.conf [defaults] server = %(server)s diff --git a/xCAT-test/provtest/conf/xcatd-policy.conf b/xCAT-test/blackboxtest/conf/prov/xcatd-policy.conf similarity index 97% rename from xCAT-test/provtest/conf/xcatd-policy.conf rename to xCAT-test/blackboxtest/conf/prov/xcatd-policy.conf index 037a1dbab..edc4b8eda 100644 --- a/xCAT-test/provtest/conf/xcatd-policy.conf +++ b/xCAT-test/blackboxtest/conf/prov/xcatd-policy.conf @@ -7,10 +7,10 @@ # anything outside it must be refused to a client that has shown no # certificate, or a machine on the provisioning network owns the cluster. # -# provtest run \ +# blackboxtest run \ # --set server=10.99.1.1 --set client=10.99.1.11 \ # --set xcatport=3001 --set refused=rpower --set node=provtestcn \ -# conf/xcatd-policy.conf +# conf/prov/xcatd-policy.conf [defaults] server = %(server)s diff --git a/xCAT-test/provtest/conf/xcatd-postscript.conf b/xCAT-test/blackboxtest/conf/prov/xcatd-postscript.conf similarity index 98% rename from xCAT-test/provtest/conf/xcatd-postscript.conf rename to xCAT-test/blackboxtest/conf/prov/xcatd-postscript.conf index 61e7f38a4..3d03432a9 100644 --- a/xCAT-test/provtest/conf/xcatd-postscript.conf +++ b/xCAT-test/blackboxtest/conf/prov/xcatd-postscript.conf @@ -9,12 +9,12 @@ # marker reads as a hang and not an error. So the marker is asserted on the # plain transport only. # -# provtest run \ +# blackboxtest run \ # --set server=10.99.1.1 --set client=10.99.1.11 \ # --set unknown=10.99.1.201 --set node=provtestcn \ # --set xcatport=3001 --set monitorport=3002 \ # --set master=10.99.1.1 --set domain=provtest.cluster \ -# conf/xcatd-postscript.conf +# conf/prov/xcatd-postscript.conf [defaults] server = %(server)s diff --git a/xCAT-test/dhcptest/src/dhcptest b/xCAT-test/blackboxtest/src/blackboxtest similarity index 53% rename from xCAT-test/dhcptest/src/dhcptest rename to xCAT-test/blackboxtest/src/blackboxtest index 2873696e3..d57dd875e 100755 --- a/xCAT-test/dhcptest/src/dhcptest +++ b/xCAT-test/blackboxtest/src/blackboxtest @@ -1,10 +1,9 @@ #!/usr/bin/env python3 -"""dhcptest -- test a DHCP server from the wire. +"""blackboxtest -- ask a management node what a booting node asks, and check +the answers. Runs straight out of a checkout: this script puts its own directory on -sys.path, so no installation, no virtualenv and no pip are involved. The only -runtime dependency beyond the standard library is scapy, and only the -subcommands that actually send packets need it. +sys.path, so no installation, no virtualenv and no pip are involved. """ import os @@ -12,7 +11,7 @@ import sys sys.path.insert(0, os.path.dirname(os.path.realpath(__file__))) -from dhcptest_lib.cli import main # noqa: E402 (path set up above) +from blackboxtest_lib.cli import main # noqa: E402 (path set up above) if __name__ == "__main__": sys.exit(main()) diff --git a/xCAT-test/blackboxtest/src/blackboxtest_lib/__init__.py b/xCAT-test/blackboxtest/src/blackboxtest_lib/__init__.py new file mode 100644 index 000000000..fedaf1ac4 --- /dev/null +++ b/xCAT-test/blackboxtest/src/blackboxtest_lib/__init__.py @@ -0,0 +1,7 @@ +"""blackboxtest -- a client-side test of the xCAT provisioning services. + +A booting node speaks DHCP, DNS, TFTP, HTTP and the xcatd protocols, in that +order. This package speaks all of them from the client's side and asserts on +what comes back. It never reads the xCAT database and never runs an xCAT +command: every value it expects is given to it on the command line. +""" diff --git a/xCAT-test/blackboxtest/src/blackboxtest_lib/assertions.py b/xCAT-test/blackboxtest/src/blackboxtest_lib/assertions.py new file mode 100644 index 000000000..f689344d4 --- /dev/null +++ b/xCAT-test/blackboxtest/src/blackboxtest_lib/assertions.py @@ -0,0 +1,180 @@ +"""The assertion language: `target op value`, one per line under `assert`. + +Whether a target names a field the step can produce is checked offline, in +`validate`, because that depends on the step type. This module only parses a +line and evaluates it against a reply. +""" + +import re + +from . import netutil, subst +from .errors import ConfigError +from .model import Assertion + +#: Targets the runner supplies about the step, not read from the reply. +#: `offers` counts the DHCP servers that answered a DISCOVER. +META_TARGETS = frozenset(["attempt", "offers"]) + +NULLARY_OPS = frozenset(["present", "absent"]) + +#: Operators that a missing target satisfies: a reply naming no boot file has +#: certainly not named the wrong one. +NEGATIVE_OPS = frozenset(["!=", "not-in"]) + +OPS = frozenset([ + "==", "!=", "in", "not-in", "present", "absent", + "matches", "contains", "starts-with", "ends-with", + "<", "<=", ">", ">=", +]) + + +class Result(object): + """The outcome of one assertion, with enough to explain a failure.""" + + __slots__ = ("assertion", "ok", "expected", "actual", "detail") + + def __init__(self, assertion, ok, expected=None, actual=None, detail=""): + self.assertion = assertion + self.ok = ok + self.expected = expected + self.actual = actual + self.detail = detail + + +def parse(line, source=""): + """Parse one assertion line.""" + parts = line.strip().split(None, 2) + if len(parts) < 2: + raise ConfigError("an assertion needs a target and an operator: %r" + % (line.strip(),), source) + target, op = parts[0], parts[1] + value = parts[2] if len(parts) > 2 else None + + if op not in OPS: + raise ConfigError("unknown operator %r in %r (expected one of: %s)" + % (op, line.strip(), ", ".join(sorted(OPS))), source) + if op in NULLARY_OPS: + if value: + raise ConfigError("operator %r takes no value: %r" + % (op, line.strip()), source) + elif value is None: + value = "" + if op == "matches": + try: + re.compile(value) + except re.error as exc: + raise ConfigError("bad regex %r: %s" % (value, exc), source) + return Assertion(target, op, value, source) + + +def parse_block(block, source=""): + """Parse a multi-line `assert =` value. Blank and `#` lines are skipped.""" + result = [] + for number, raw in enumerate(str(block or "").splitlines(), start=1): + line = raw.strip() + if line and not line.startswith("#"): + where = "%s line %d of assert" % (source, number) if source else "" + result.append(parse(line, where)) + return tuple(result) + + +def evaluate(assertion, reply, context, extras=None): + """Check one assertion against one reply, which may be None.""" + expected = None + if assertion.value is not None: + expected = subst.resolve(assertion.value, context) + + present, actual = _actual(assertion.target, reply, context, extras or {}) + + if assertion.op in NULLARY_OPS: + ok = present if assertion.op == "present" else not present + return Result(assertion, ok, expected=assertion.op, + actual="present" if present else "absent") + + if not present: + return Result(assertion, assertion.op in NEGATIVE_OPS, + expected=expected, + detail="%s is not present in the reply" % (assertion.target,)) + + return Result(assertion, _compare(assertion.op, actual, expected), + expected=expected, actual=subst.format_value(actual)) + + +def _actual(target, reply, context, extras): + """`(present, value)` for one assertion target.""" + if target in META_TARGETS: + return (target in extras), extras.get(target) + if target.startswith("$"): + try: + return True, subst.resolve(target, context) + except ConfigError: + return False, None + if reply is None or not reply.has(target): + return False, None + return True, reply.whole(target) + + +def _compare(op, actual, expected): + if op in ("==", "!="): + return _equal(actual, expected) == (op == "==") + if op in ("in", "not-in"): + return _member(actual, expected) == (op == "in") + + text = subst.format_value(actual) + if op == "matches": + return bool(re.search(expected, text, re.MULTILINE)) + if op == "contains": + return expected in text + if op == "starts-with": + return text.startswith(expected) + if op == "ends-with": + return text.endswith(expected) + + left, right = _as_numbers(actual, expected) + if left is None: + return False + return {"<": left < right, "<=": left <= right, + ">": left > right, ">=": left >= right}[op] + + +def _equal(actual, expected): + """Compare as addresses, then as numbers, then as text. + + A list equals a value when the whole list, written out, is that value, or + when any one element is: a name with two A records satisfies + `data == `. + """ + if isinstance(actual, (list, tuple)): + return (subst.format_value(actual) == expected + or any(_equal(item, expected) for item in actual)) + + left_ip = netutil.parse_ip(subst.format_value(actual)) + right_ip = netutil.parse_ip(expected) + if left_ip is not None and right_ip is not None: + return left_ip == right_ip + + left, right = _as_numbers(actual, expected) + if left is not None: + return left == right + return subst.format_value(actual) == str(expected) + + +def _member(actual, expected): + """Membership of a CIDR, of a `first-last` range, or of a comma list.""" + text = subst.format_value(actual) + if "/" in expected and netutil.parse_ip(text) is not None: + return netutil.ip_in(text, expected) + if netutil.parse_range(expected) is not None: + return netutil.ip_in_range(text, expected) + candidates = [item.strip() for item in expected.split(",") if item.strip()] + return any(_equal(actual, candidate) for candidate in candidates) + + +def _as_numbers(actual, expected): + """Both sides as ints, or `(None, None)` when either is not a number.""" + try: + left = actual if isinstance(actual, int) else int(str(actual).strip(), 0) + right = int(str(expected).strip(), 0) + except (TypeError, ValueError): + return None, None + return left, right diff --git a/xCAT-test/blackboxtest/src/blackboxtest_lib/cli.py b/xCAT-test/blackboxtest/src/blackboxtest_lib/cli.py new file mode 100644 index 000000000..c41e3b52a --- /dev/null +++ b/xCAT-test/blackboxtest/src/blackboxtest_lib/cli.py @@ -0,0 +1,126 @@ +"""Command line interface. + + blackboxtest run [options] CONF [CONF...] # execute scenarios + blackboxtest validate CONF [CONF...] # check offline + blackboxtest list CONF [CONF...] # show what a file does + +`validate` and `list` need no root, no network, no scapy and no client +program, so they run in a checkout. +""" + +import argparse +import sys + +from . import config, report, validate +from .errors import EXIT_CONFIG, EXIT_OK, BlackboxError + + +def build_parser(): + parser = argparse.ArgumentParser( + prog="blackboxtest", + description="Ask a management node what a booting node asks, and " + "check the answers.") + sub = parser.add_subparsers(dest="command") + + run = sub.add_parser("run", help="execute scenarios against a real server") + _conf_arguments(run) + run.add_argument("-i", "--interface", metavar="IFACE", + help="interface the DHCP steps send from") + run.add_argument("-b", "--bind", metavar="ADDR", + help="source address of every step that sets no bind=") + run.add_argument("--timeout", type=float, metavar="SECS", + help="per-attempt timeout, over the files") + run.add_argument("--retries", type=int, metavar="N", + help="attempts per step, over the files") + run.add_argument("-s", "--scenario", action="append", default=[], + metavar="NAME", help="run only this scenario (repeatable)") + run.add_argument("-v", "--verbose", action="count", default=0, + help="print each reply to stderr") + + _conf_arguments(sub.add_parser("validate", help="check .conf files offline")) + _conf_arguments(sub.add_parser("list", help="show scenarios and steps")) + return parser + + +def _conf_arguments(parser): + parser.add_argument("conf", nargs="+", metavar="CONF") + parser.add_argument("--set", action="append", default=[], dest="settings", + metavar="KEY=VALUE", + help="define %%(KEY)s (repeatable)") + + +def main(argv=None): + parser = build_parser() + args = parser.parse_args(argv) + if not args.command: + parser.print_help() + return EXIT_CONFIG + try: + settings = {} + for pair in args.settings: + key, sep, value = pair.partition("=") + if not sep: + raise BlackboxError("--set needs KEY=VALUE, got %r" % (pair,)) + settings[key.strip()] = value + return COMMANDS[args.command](args, settings) + except BlackboxError as exc: + sys.stderr.write("blackboxtest: %s\n" % (exc,)) + return exc.exit_code + + +def _problems(scenarios): + problems = [p for s in scenarios for p in validate.scenario_problems(s)] + for problem in problems: + sys.stderr.write("blackboxtest: %s\n" % (problem,)) + return problems + + +def command_validate(args, settings): + # Raw: in a checkout no value has been supplied for %(name)s yet. + scenarios = config.load(args.conf, settings, raw=True) + if _problems(scenarios): + return EXIT_CONFIG + sys.stdout.write("%d scenario(s) OK in %d file(s)\n" + % (len(scenarios), len(args.conf))) + needed = sorted(validate.required_variables(scenarios) - set(settings)) + if needed: + sys.stdout.write("required variables: %s\n" % (", ".join(needed),)) + return EXIT_OK + + +def command_list(args, settings): + for scenario in config.load(args.conf, settings, raw=True): + sys.stdout.write("%s\n" % (scenario.name,)) + if scenario.description: + sys.stdout.write(" %s\n" % (scenario.description,)) + sys.stdout.write(" source: %s\n" % (scenario.source,)) + for step in scenario.steps: + sys.stdout.write(" step %-16s type=%-12s expect=%-6s asserts=%d\n" + % (step.name, step.type, step.expect or "-", + len(step.assertions))) + needed = sorted(validate.required_variables([scenario])) + if needed: + sys.stdout.write(" requires: %s\n" % (", ".join(needed),)) + return EXIT_OK + + +def command_run(args, settings): + from . import runner # opens sockets and spawns clients + + scenarios = config.load(args.conf, settings) + if args.scenario: + missing = set(args.scenario) - set(s.name for s in scenarios) + if missing: + raise BlackboxError("no such scenario: %s" + % (", ".join(sorted(missing)),)) + scenarios = [s for s in scenarios if s.name in args.scenario] + if _problems(scenarios): + return EXIT_CONFIG + options = runner.RunOptions(interface=args.interface, bind=args.bind, + timeout=args.timeout, retries=args.retries, + verbose=args.verbose) + return runner.run(scenarios, report.TapReporter(), options) + + +COMMANDS = {"run": command_run, "validate": command_validate, + "list": command_list} diff --git a/xCAT-test/blackboxtest/src/blackboxtest_lib/config.py b/xCAT-test/blackboxtest/src/blackboxtest_lib/config.py new file mode 100644 index 000000000..3f18ee777 --- /dev/null +++ b/xCAT-test/blackboxtest/src/blackboxtest_lib/config.py @@ -0,0 +1,167 @@ +"""Reading .conf files with configparser. + + [vars] optional; %(name)s values, overridden by --set + [defaults] optional; step keys for every step that takes them + [scenario ] starts a scenario + [step ] belongs to the scenario above it + +`%(name)s` is configparser's `BasicInterpolation`, applied per value here +rather than through the parser: both built-in ways of supplying the values +also make each variable an option, so `--set path=x` would overwrite the step +key `path`. `BasicInterpolation` gives `$` no meaning, so `$offer.address` +reaches the runner untouched. + +Unknown keys are an error, as an unknown `check:` is in xcattest. +""" + +import configparser +import os + +from . import assertions, steps +from .errors import ConfigError +from .model import Scenario, Step + +SCENARIO_KEYS = frozenset(["description", "interface"]) + +#: `type` and `assert` are per-step: a default type would turn a misspelt +#: `type=` into another protocol. +DEFAULTS_KEYS = frozenset( + key for kind in steps.TYPES.values() for key in kind.keys +) - frozenset(["type", "assert"]) | frozenset(["interface"]) + +INTERPOLATION = configparser.BasicInterpolation() + + +def load(paths, overrides=None, raw=False): + """Read every path into a list of scenarios. + + With `raw`, `%(name)s` stays in place, so `validate` can check a file for + which no value has been supplied. + """ + scenarios = [] + seen = {} + for path in paths: + for scenario in load_file(path, overrides or {}, raw): + if scenario.name in seen: + raise ConfigError("scenario %r is defined in both %s and %s" + % (scenario.name, seen[scenario.name], path)) + seen[scenario.name] = path + scenarios.append(scenario) + return scenarios + + +def load_file(path, overrides, raw=False): + if not os.path.isfile(path): + raise ConfigError("no such file", path) + parser = configparser.ConfigParser(interpolation=None, delimiters=("=",), + inline_comment_prefixes=(";",)) + parser.optionxform = str.lower + try: + with open(path) as handle: + parser.read_file(handle, source=path) + except configparser.Error as exc: + raise ConfigError(str(exc), path) + + variables = dict(parser["vars"]) if parser.has_section("vars") else {} + variables.update(overrides) + + scenarios = [] + defaults = {} + current = None + for section in parser.sections(): + where = "%s [%s]" % (path, section) + items = _items(parser, section, where, raw, variables) + kind, _, name = section.partition(" ") + kind, name = kind.strip().lower(), name.strip() + + if section.lower() == "vars": + continue + if section.lower() == "defaults": + _reject_unknown(items, DEFAULTS_KEYS, where) + defaults = items + elif kind == "scenario" and name: + _reject_unknown(items, SCENARIO_KEYS, where) + current = Scenario(name, items.get("description", ""), + items.get("interface") or defaults.get("interface"), + source=path) + scenarios.append((current, [])) + elif kind == "step" and name: + if current is None: + raise ConfigError("a [step] must follow a [scenario]", where) + scenarios[-1][1].append(_build_step(name, items, defaults, where)) + else: + raise ConfigError("unknown section; expected [vars], [defaults], " + "[scenario ] or [step ]", where) + + if not scenarios: + raise ConfigError("no [scenario ...] section found", path) + for scenario, found in scenarios: + if not found: + raise ConfigError("scenario %r has no steps" % (scenario.name,), path) + scenario.steps = tuple(found) + return [scenario for scenario, _ in scenarios] + + +def _build_step(name, items, defaults, where): + step_type = (items.get("type") or "").strip().lower() + if not step_type: + raise ConfigError("step %r has no type=" % (name,), where) + if step_type not in steps.TYPES: + raise ConfigError("unknown step type %r (known: %s)" + % (step_type, ", ".join(sorted(steps.TYPES))), where) + kind = steps.TYPES[step_type] + _reject_unknown(items, kind.keys, where) + + # A default reaches only the steps that can take it, so one + # `[defaults] server =` serves dns, tftp and http steps without making a + # DHCP step in the same file illegal. + merged = dict((k, v) for k, v in defaults.items() if k in kind.keys) + merged.update(items) + return Step( + name=name, + type=step_type, + expect=(merged.pop("expect", "") or "").strip().lower() or None, + assertions=assertions.parse_block(merged.pop("assert", ""), where), + timeout=_number(float, merged.pop("timeout", None), kind.timeout, + "timeout", where), + retries=_number(lambda v: int(v, 0), merged.pop("retries", None), + kind.retries, "retries", where), + params=dict((k, v) for k, v in merged.items() if k != "type"), + source=where, + ) + + +def _items(parser, section, where, raw, variables): + """One section's keys, with `%(name)s` resolved out of `variables`.""" + items = {} + for key in parser.options(section): + value = parser.get(section, key, raw=True) + if raw: + items[key] = value + continue + try: + items[key] = INTERPOLATION.before_get(parser, section, key, value, + variables) + except configparser.InterpolationMissingOptionError as exc: + raise ConfigError("%%(%s)s is not defined; pass --set %s=" + % (exc.reference, exc.reference), where) + except configparser.Error as exc: + raise ConfigError(str(exc), where) + return items + + +def _reject_unknown(items, allowed, where): + unknown = sorted(set(items) - set(allowed)) + if unknown: + raise ConfigError("unknown key(s): %s (allowed: %s)" + % (", ".join(unknown), ", ".join(sorted(allowed))), + where) + + +def _number(convert, value, default, key, where): + if value in (None, ""): + return default + try: + return convert(value.strip()) + except ValueError: + raise ConfigError("%s= must be a number, got %r" % (key, value), where) diff --git a/xCAT-test/blackboxtest/src/blackboxtest_lib/dhcp.py b/xCAT-test/blackboxtest/src/blackboxtest_lib/dhcp.py new file mode 100644 index 000000000..c2944ae4a --- /dev/null +++ b/xCAT-test/blackboxtest/src/blackboxtest_lib/dhcp.py @@ -0,0 +1,336 @@ +"""The DHCP client: an RFC 2131 state machine on a raw Layer 2 socket. + +A raw socket works on an interface that has no address, which is the normal +state of a provisioning NIC when a machine boots on it. + +This is the only module that imports scapy, and only inside `Wire`, so the +transition table and `validate` need neither scapy nor root. +""" + +import os +import select +import time + +from . import dhcpopts, netutil, subst +from .errors import ConfigError, UnsupportedError + +INIT = "INIT" +SELECTING = "SELECTING" +BOUND = "BOUND" + +BPF = "(udp and (port 67 or port 68)) or arp" + +#: Names a reply is bound under besides the step's own, with the message type +#: that earns each one. `reply` is whatever came back. +ALIASES = {"offer": "OFFER", "ack": "ACK", "lease": "ACK", "reply": None} + + +class Transition(object): + """One legal `(state, step type)` pair.""" + + __slots__ = ("message", "expects", "default_expect", "next_state", + "requires", "binds", "unicast") + + def __init__(self, message, expects, next_state, requires=(), binds=(), + unicast=False): + self.message = message + self.expects = frozenset(expects) + self.default_expect = expects[0] if expects else "none" + self.next_state = next_state + self.requires = tuple(requires) + self.binds = tuple(binds) + self.unicast = unicast + + +TRANSITIONS = { + (INIT, "discover"): Transition( + "discover", ["offer"], SELECTING, binds=("offer",)), + # BOOTP: a header with no option 53, and one reply is the whole exchange. + (INIT, "bootrequest"): Transition( + "bootrequest", ["bootreply"], INIT), + (SELECTING, "request"): Transition( + "request", ["ack", "nak"], BOUND, binds=("ack", "lease")), + # INIT-REBOOT: a REQUEST with option 50 and no option 54. + (INIT, "request"): Transition( + "request", ["ack", "nak"], BOUND, requires=("requested_address",), + binds=("ack", "lease")), + (BOUND, "renew"): Transition( + "request", ["ack", "nak"], BOUND, binds=("ack", "lease"), unicast=True), + (BOUND, "rebind"): Transition( + "request", ["ack", "nak"], BOUND, binds=("ack", "lease")), + # Nothing answers a RELEASE. Giving the lease up is the whole step. + (BOUND, "release"): Transition("release", [], INIT, unicast=True), +} + +STEP_TYPES = frozenset(kind for _, kind in TRANSITIONS) + +#: Step keys that become an option, in the order they are sent. +OPTION_KEYS = ( + ("requested_address", 50), ("server_id", 54), ("lease_time", 51), + ("hostname", 12), ("fqdn", 81), ("max_message_size", 57), + ("vendor_class", 60), ("client_arch", 93), ("client_ndi", 94), + ("client_uuid", 97), ("vendor_specific", 43), ("ipxe_options", 175), + ("request_options", 55), +) + +#: Every key a DHCP step accepts, beyond the common ones. +STEP_KEYS = frozenset(key for key, _ in OPTION_KEYS) | frozenset([ + "mac", "ciaddr", "giaddr", "client_id", "user_class", "user_class_form"]) + + +def transition(state, step_type): + return TRANSITIONS.get((state, step_type)) + + +def expected(step, trans): + """The reply types a step waits for; empty when it waits for none.""" + expect = step.expect or trans.default_expect + if expect == "none": + return frozenset() + if expect == "any": + return trans.expects + return frozenset([expect]) + + +def advance(state, trans, answered, msgtype=None): + """The client's state after a step. + + A step that waited for an answer and got none leaves the client where it + was: an unanswered DISCOVER does not put it in SELECTING. Nothing answers + a RELEASE, so it moves on regardless. `validate` walks the same rule. + """ + if trans.expects and not answered: + return state + if msgtype == "NAK" and "nak" in trans.expects: + return INIT + return trans.next_state + + +# --------------------------------------------------------------------------- +# packets + + +class Session(object): + """One client, across the DHCP steps of a scenario.""" + + def __init__(self, wire): + self.wire = wire + self.mac = netutil.random_mac() + self.state = INIT + self.xid = 0 + self.leases = set() # addresses to answer ARP for + + +def build_options(step, mac, msgtype, context): + """`[(code, bytes)]` for one outgoing message.""" + def value(key, default=None): + raw = step.param(key, default) + return subst.resolve(raw, context) if raw is not None else None + + # A BOOTP client predates options 53 and 61, so it sends neither. + bootp = msgtype == "bootrequest" + items = [] if bootp else [(53, bytes([dhcpopts.msgtype_code(msgtype)]))] + + client_id = value("client_id", "none" if bootp else "auto") + if client_id == "auto": + items.append((61, b"\x01" + netutil.mac_bytes(mac))) + elif client_id and client_id != "none": + items.append((61, dhcpopts.parse_hex(client_id))) + + for key, code in OPTION_KEYS: + text = value(key) + if text not in (None, ""): + items.append((code, dhcpopts.encode_value(code, text))) + + user_class = value("user_class") + if user_class: + raw = user_class.encode("utf-8") + form = (value("user_class_form") or "raw").lower() + if form == "rfc3004": + raw = bytes([len(raw)]) + raw + elif form != "raw": + raise ConfigError("user_class_form must be 'raw' or 'rfc3004'") + items.append((77, raw)) + return items + + +def build_frame(step, session, trans, context, scapy): + """One Ethernet frame carrying the step's message.""" + def value(key, default): + return subst.resolve(step.param(key, default), context) + + ciaddr = value("ciaddr", "0.0.0.0") + server = value("server_id", "") + unicast = trans.unicast and server + # A client that cannot yet receive unicast asks for a broadcast reply. + flags = 0 if unicast or ciaddr != "0.0.0.0" else 0x8000 + + blob = dhcpopts.MAGIC + dhcpopts.pack_options( + build_options(step, session.mac, trans.message, context)) + # RFC 1542: a BOOTP message is at least 300 bytes, 240 of them header. + blob += b"\x00" * max(0, 60 - len(blob)) + + return (scapy.Ether(src=session.mac, dst="ff:ff:ff:ff:ff:ff") / + scapy.IP(src=ciaddr, dst=server if unicast else "255.255.255.255") / + scapy.UDP(sport=68, dport=67) / + scapy.BOOTP(op=1, htype=1, hlen=6, xid=session.xid, flags=flags, + ciaddr=ciaddr, giaddr=value("giaddr", "0.0.0.0"), + chaddr=netutil.mac_bytes(session.mac)) / + scapy.Raw(load=blob)) + + +def describe(step, session, trans, context): + """A one-line summary of what went out, for a failure report.""" + parts = ["%s mac=%s xid=0x%08x" % (trans.message.upper(), session.mac, + session.xid)] + for key in sorted(step.params): + value = subst.resolve(step.params[key], context) + if value not in (None, ""): + parts.append("%s=%s" % (key, value)) + return " ".join(parts) + + +# --------------------------------------------------------------------------- +# the wire + + +def scapy_module(): + """Import scapy on demand, with a message a human can act on.""" + try: + import scapy.all + except ImportError: + raise UnsupportedError( + "scapy is not installed; install the python3-scapy package") + return scapy.all + + +def require(): + """Refuse, before any output, a host that cannot send a DHCP frame.""" + scapy_module() + if os.geteuid() != 0: + raise UnsupportedError("raw sockets need root; re-run with sudo") + + +class Wire(object): + """A Layer 2 socket, opened before the first send so no reply is missed.""" + + def __init__(self, interface): + self.scapy = scapy_module() + try: + self.socket = self.scapy.conf.L2socket(iface=interface, filter=BPF) + except Exception as exc: # scapy raises many types + raise UnsupportedError( + "cannot open a raw socket on %s: %s" % (interface, exc)) + + def close(self): + self.socket.close() + + def send(self, frame): + self.socket.send(frame) + + def collect(self, session, deadline): + """Every reply to this session's xid until `deadline`. + + A reply of an unexpected type is kept, so a NAK where an ACK was + wanted is reported as a NAK rather than as a timeout. + """ + found = [] + while True: + remaining = deadline - time.monotonic() + if remaining <= 0 or not select.select([self.socket], [], [], remaining)[0]: + return found + frame = self.socket.recv(2048) + if frame is None: + continue + if frame.haslayer("ARP"): + self._answer_arp(frame, session) + continue + if not frame.haslayer("BOOTP"): + continue + bootp = frame["BOOTP"] + if int(bootp.op) == 2 and int(bootp.xid) == session.xid \ + and bytes(bootp.chaddr)[:6] == netutil.mac_bytes(session.mac) \ + and frame["Ether"].src != session.mac: + found.append(dhcpopts.decode(frame)) + + def _answer_arp(self, frame, session): + """Answer ARP for a leased address, so a unicast renewal reply arrives + without the address ever being put on the host.""" + arp = frame["ARP"] + if int(arp.op) != 1 or arp.pdst not in session.leases: + return + self.send(self.scapy.Ether(src=session.mac, dst=frame["Ether"].src) / + self.scapy.ARP(op=2, hwsrc=session.mac, psrc=arp.pdst, + hwdst=arp.hwsrc, pdst=arp.psrc)) + + +# --------------------------------------------------------------------------- +# running a step + + +def run_step(step, session, context, timeout, retries): + """Send one message, retransmitting until a wanted reply arrives. + + Returns `(reply, extras, sent)`. `reply` is None when nothing answered. + `$reply` and the aliases the transition earns are bound here; the runner + binds the step's own name. + """ + if step.param("mac"): + session.mac = netutil.normalise_mac( + subst.resolve(step.param("mac"), context)) + + trans = transition(session.state, step.type) + if trans is None: + raise ConfigError("a %s step is not legal in state %s" + % (step.type, session.state)) + wanted = set(kind.upper() for kind in expected(step, trans)) + session.xid = netutil.random_xid() + + replies = [] + attempt = 0 + sent = "" + for attempt in range(1, retries + 1): + sent = describe(step, session, trans, context) + session.wire.send(build_frame(step, session, trans, context, + session.wire.scapy)) + replies = session.wire.collect(session, time.monotonic() + timeout) + if any(r.fields["msgtype"] in wanted for r in replies) \ + or (replies and not wanted): + break + + chosen = next((r for r in replies if r.fields["msgtype"] in wanted), + replies[0] if replies else None) + extras = { + "attempt": attempt, + "offers": len(distinct_servers( + r for r in replies if r.fields["msgtype"] == "OFFER")), + } + + msgtype = None + if chosen is not None: + chosen.sent = sent + chosen.attempt = attempt + msgtype = chosen.fields["msgtype"] + context.bind("reply", chosen) + for name in trans.binds: + if ALIASES[name] == msgtype: + context.bind(name, chosen) + if msgtype == "ACK" and chosen.fields["yiaddr"] != "0.0.0.0": + session.leases.add(chosen.fields["yiaddr"]) + + session.state = advance(session.state, trans, + bool(wanted) and chosen is not None, msgtype) + return chosen, extras, sent + + +def distinct_servers(replies): + """One reply per answering server. + + A retransmit reuses its xid, as RFC 2131 requires, so one server can be + heard twice. `offers` counts servers, not packets. + """ + seen = {} + for reply in replies: + key = reply.options.get(54) or reply.fields.get("src_ip") + seen.setdefault(key, reply) + return list(seen.values()) diff --git a/xCAT-test/dhcptest/src/dhcptest_lib/options.py b/xCAT-test/blackboxtest/src/blackboxtest_lib/dhcpopts.py similarity index 67% rename from xCAT-test/dhcptest/src/dhcptest_lib/options.py rename to xCAT-test/blackboxtest/src/blackboxtest_lib/dhcpopts.py index 3b149f92b..9dfde3ed9 100644 --- a/xCAT-test/dhcptest/src/dhcptest_lib/options.py +++ b/xCAT-test/blackboxtest/src/blackboxtest_lib/dhcpopts.py @@ -1,9 +1,8 @@ -"""DHCP option names, numbers and value coding. +"""DHCP option names, numbers and value coding, and the decoded reply. -This module is scapy-free on purpose. It owns the mapping between the names a -.conf file may use and the option codes that go on the wire, so that a scapy -version which renames one of its own option strings changes this table and -nothing else. +Scapy-free. The option area is read from raw bytes rather than through scapy's +option table, so a scapy release that renames an option cannot change what a +.conf asserts. """ import binascii @@ -11,6 +10,7 @@ import ipaddress import re from .errors import ConfigError +from .model import Reply #: DHCP message types (option 53), by name. MSGTYPE_BY_NAME = { @@ -377,55 +377,157 @@ def unpack_options(blob): return found +def encode_value(code, text): + """The bytes of one option, from the text a .conf gives for it.""" + text = "" if text is None else str(text) + if code in IP_OPTIONS: + return ipaddress.IPv4Address(text.strip()).packed + if code in IP_LIST_OPTIONS: + return b"".join(ipaddress.IPv4Address(item.strip()).packed + for item in text.split(",")) + for width, codes in ((4, U32_OPTIONS), (2, U16_OPTIONS), (1, U8_OPTIONS)): + if code in codes: + return int(text, 0).to_bytes(width, "big") + if code == 55: + return bytes(option_code(item.strip()) + for item in text.split(",") if item.strip()) + if code in TEXT_OPTIONS: + return text.encode("utf-8") + if code == 81: + return encode_fqdn(text) + return parse_hex(text) + + +#: BOOTP header fields, readable by name. +HEADER_FIELDS = ("msgtype", "xid", "yiaddr", "siaddr", "ciaddr", "giaddr", + "chaddr", "file", "sname", "secs", "flags", "src_ip", "src_mac") + +#: Other names for header fields, so a .conf can say `$offer.address`. +ALIASES = {"address": "yiaddr", "mac": "chaddr", "next_server": "siaddr"} + +#: Fields a client reads from more than one place in the message. +DERIVED_FIELDS = ("bootfile", "dns_name", "fqdn_flags") + + +def field_option(name): + """The option code a field name stands for (`opt54`, `54`, `server_id`, + `option:54`), or None.""" + if name.startswith("opt") and name[3:].isdigit(): + return int(name[3:]) + try: + return option_code(name) + except ConfigError: + return None + + +def known_field(name): + """True when a DHCP reply can carry a field called `name`.""" + name = ALIASES.get(name, name) + return (name in DERIVED_FIELDS or name in HEADER_FIELDS + or field_option(name) is not None) + + +class DhcpReply(Reply): + """A decoded DHCP or BOOTP reply.""" + + __slots__ = ("options", "raw_options") + + def __init__(self, fields, options=None, raw_options=b""): + Reply.__init__(self, kind="dhcp", fields=fields) + self.options = dict(options or {}) + # Option 81 is decoded to its name. Its flag bits are only in here. + self.raw_options = raw_options + + def bootfile(self): + """What a client boots: option 67 when sent, else the `file` header. + + ISC fills the header, dnsmasq answers in option 67, and firmware + reads whichever arrived. + """ + return self.options.get(67) or self.fields.get("file", "") + + def dns_name(self): + """The node's name: option 81 when sent, else option 12. + + Kea answers a client that sent option 81 in option 81. ISC with + `ignore client-updates` sends option 12 only. + """ + return self.options.get(81) or self.options.get(12) or "" + + def fqdn_flags(self): + """Option 81's flags as letters, `SO`; empty when there is no 81.""" + if 81 not in self.options: + return "" + decoded = decode_fqdn(unpack_options(self.raw_options).get(81, b"")) + return fqdn_flag_letters(decoded[0]) if decoded else "" + + def whole(self, name): + name = ALIASES.get(name, name) + if name in DERIVED_FIELDS: + return getattr(self, name)() + if name in self.fields: + return self.fields[name] + code = field_option(name) + if code is None: + raise KeyError("a DHCP reply has no field .%s" % (name,)) + if code not in self.options: + raise KeyError("the reply has no option %d" % (code,)) + return self.options[code] + + field = whole + + def has(self, name): + name = ALIASES.get(name, name) + if name in DERIVED_FIELDS: + return bool(self.whole(name)) + if name in self.fields: + # A zeroed header field is the header saying nothing. + return self.fields[name] not in ("", 0, "0.0.0.0") + return field_option(name) in self.options + + def summary(self): + f = self.fields + who = f.get("src_ip") or "?" + if f.get("src_mac"): + who = "%s (%s)" % (who, f["src_mac"]) + return "%s from %s xid=0x%08x yiaddr=%s siaddr=%s file=%r" % ( + f.get("msgtype") or "?", who, f.get("xid", 0), f.get("yiaddr"), + f.get("siaddr"), f.get("file", "")) + + def detail_lines(self): + return [("options", "{" + ", ".join( + "%s=%s" % (option_label(code), self.options[code]) + for code in sorted(self.options)) + "}")] + + def decode(packet): - """Build a `model.Reply` from a scapy packet. - - The option area is parsed from raw bytes rather than through scapy's own - option table, so a scapy release that renames one of its option strings - cannot change what a .conf file asserts. - """ - from .model import Reply # local import: avoids a cycle - + """A `DhcpReply` from a scapy packet that carries a BOOTP layer.""" bootp = packet["BOOTP"] blob = bytes(bootp.payload) if bootp.payload else b"" if not blob: blob = bytes(getattr(bootp, "options", b"") or b"") - options = {} - for code, raw in unpack_options(blob).items(): - options[code] = decode_value(code, raw) + options = dict((code, decode_value(code, raw)) + for code, raw in unpack_options(blob).items()) - # A BOOTREPLY with no option 53 is a plain BOOTP answer -- the whole reply - # a client that predates DHCP will ever get. Naming it rather than leaving - # the type blank is what lets a step assert on it. - msgtype = "BOOTREPLY" - if 53 in options: - msgtype = msgtype_name(options[53]) + # A BOOTREPLY with no option 53 is the whole answer a BOOTP client gets. + msgtype = msgtype_name(options[53]) if 53 in options else "BOOTREPLY" - src_ip = "" - src_mac = "" - if hasattr(packet, "haslayer") and packet.haslayer("IP"): - src_ip = packet["IP"].src - if hasattr(packet, "haslayer") and packet.haslayer("Ether"): - src_mac = packet["Ether"].src - - return Reply( - msgtype=msgtype, - xid=int(getattr(bootp, "xid", 0)), - yiaddr=str(getattr(bootp, "yiaddr", "0.0.0.0")), - siaddr=str(getattr(bootp, "siaddr", "0.0.0.0")), - ciaddr=str(getattr(bootp, "ciaddr", "0.0.0.0")), - giaddr=str(getattr(bootp, "giaddr", "0.0.0.0")), - chaddr=format_hex(bytes(getattr(bootp, "chaddr", b""))[:6]), - file=_trim(getattr(bootp, "file", b"")), - sname=_trim(getattr(bootp, "sname", b"")), - secs=int(getattr(bootp, "secs", 0)), - flags=int(getattr(bootp, "flags", 0)), - options=options, - raw_options=blob, - src_ip=src_ip, - src_mac=src_mac, - raw=packet, - ) + fields = { + "msgtype": msgtype, + "xid": int(getattr(bootp, "xid", 0)), + "yiaddr": str(getattr(bootp, "yiaddr", "0.0.0.0")), + "siaddr": str(getattr(bootp, "siaddr", "0.0.0.0")), + "ciaddr": str(getattr(bootp, "ciaddr", "0.0.0.0")), + "giaddr": str(getattr(bootp, "giaddr", "0.0.0.0")), + "chaddr": format_hex(bytes(getattr(bootp, "chaddr", b""))[:6]), + "file": _trim(getattr(bootp, "file", b"")), + "sname": _trim(getattr(bootp, "sname", b"")), + "secs": int(getattr(bootp, "secs", 0)), + "flags": int(getattr(bootp, "flags", 0)), + "src_ip": packet["IP"].src if packet.haslayer("IP") else "", + "src_mac": packet["Ether"].src if packet.haslayer("Ether") else "", + } + return DhcpReply(fields, options, blob) def _trim(value): diff --git a/xCAT-test/provtest/src/provtest_lib/dnsc.py b/xCAT-test/blackboxtest/src/blackboxtest_lib/dnsc.py similarity index 99% rename from xCAT-test/provtest/src/provtest_lib/dnsc.py rename to xCAT-test/blackboxtest/src/blackboxtest_lib/dnsc.py index 0c029db6d..50068c539 100644 --- a/xCAT-test/provtest/src/provtest_lib/dnsc.py +++ b/xCAT-test/blackboxtest/src/blackboxtest_lib/dnsc.py @@ -45,7 +45,6 @@ def query(server, name, rrtype="A", port=53, bind=None, recursion=True, done = proc.run(argv, timeout=float(timeout) + 5.0) reply = decode(done.text, name=name, rrtype=rrtype, server=server) reply.sent = done.command() - reply.raw = done.out if done.timed_out: reply.ok = False reply.error = "dig did not return within the timeout" diff --git a/xCAT-test/blackboxtest/src/blackboxtest_lib/errors.py b/xCAT-test/blackboxtest/src/blackboxtest_lib/errors.py new file mode 100644 index 000000000..3fa6449fe --- /dev/null +++ b/xCAT-test/blackboxtest/src/blackboxtest_lib/errors.py @@ -0,0 +1,35 @@ +"""Exceptions, and the exit codes they map to. + + 0 every assertion held + 1 at least one assertion failed + 2 the configuration or the command line is wrong + 3 this host cannot run the test at all + +3 is not 0. A host that tested nothing must not report a green run. +""" + +EXIT_OK = 0 +EXIT_FAILED = 1 +EXIT_CONFIG = 2 +EXIT_UNSUPPORTED = 3 + + +class BlackboxError(Exception): + """Base class of every error this tool raises on purpose.""" + + exit_code = EXIT_CONFIG + + +class ConfigError(BlackboxError): + """A .conf file or the command line asks for something impossible.""" + + def __init__(self, message, source=""): + if source: + message = "%s: %s" % (source, message) + BlackboxError.__init__(self, message) + + +class UnsupportedError(BlackboxError): + """The host lacks a program, a module or a privilege the run needs.""" + + exit_code = EXIT_UNSUPPORTED diff --git a/xCAT-test/provtest/src/provtest_lib/httpc.py b/xCAT-test/blackboxtest/src/blackboxtest_lib/httpc.py similarity index 96% rename from xCAT-test/provtest/src/provtest_lib/httpc.py rename to xCAT-test/blackboxtest/src/blackboxtest_lib/httpc.py index 34927190b..39e892310 100644 --- a/xCAT-test/provtest/src/provtest_lib/httpc.py +++ b/xCAT-test/blackboxtest/src/blackboxtest_lib/httpc.py @@ -64,8 +64,8 @@ def build(url, method="GET", bind=None, headers=(), timeout=10.0, def fetch(url, method="GET", bind=None, headers=(), timeout=10.0, insecure=False): """Make one request and return a `Reply` describing the response.""" - body_handle, body_file = tempfile.mkstemp(prefix="provtest-http-") - head_handle, header_file = tempfile.mkstemp(prefix="provtest-hdr-") + body_handle, body_file = tempfile.mkstemp(prefix="blackboxtest-http-") + head_handle, header_file = tempfile.mkstemp(prefix="blackboxtest-hdr-") os.close(body_handle) os.close(head_handle) try: @@ -113,7 +113,7 @@ def _decode(done, url, body_file, header_file): "ok": bool(status and 200 <= status < 400 and not error), } return Reply(kind="http", fields=fields, ok=fields["ok"], error=error, - sent=done.command(), raw=body) + sent=done.command()) def _write_out(text, url): diff --git a/xCAT-test/blackboxtest/src/blackboxtest_lib/model.py b/xCAT-test/blackboxtest/src/blackboxtest_lib/model.py new file mode 100644 index 000000000..3d9528c4c --- /dev/null +++ b/xCAT-test/blackboxtest/src/blackboxtest_lib/model.py @@ -0,0 +1,158 @@ +"""Plain data structures: no network, no subprocess, no scapy. + +Written for Python 3.6, so no dataclasses. +""" + + +class Assertion(object): + """One line of the assertion language: `target op value`.""" + + __slots__ = ("target", "op", "value", "source") + + def __init__(self, target, op, value, source=""): + self.target = target + self.op = op + self.value = value + self.source = source + + def render(self): + if self.value is None: + return "%s %s" % (self.target, self.op) + return "%s %s %s" % (self.target, self.op, self.value) + + def __repr__(self): + return "Assertion(%r)" % (self.render(),) + + +class Step(object): + """One client action, and the assertions made about its reply.""" + + __slots__ = ("name", "type", "expect", "params", "assertions", "timeout", + "retries", "source") + + def __init__(self, name, type, expect, params, assertions, timeout, + retries, source=""): + self.name = name + self.type = type + self.expect = expect + self.params = dict(params) + self.assertions = tuple(assertions) + self.timeout = timeout + self.retries = retries + self.source = source + + def param(self, key, default=None): + value = self.params.get(key, default) + if isinstance(value, str): + value = value.strip() + return value + + def __repr__(self): + return "Step(%r, type=%r, expect=%r)" % (self.name, self.type, self.expect) + + +class Scenario(object): + """A named sequence of steps, run in file order. + + `interface` is where the DHCP steps send from. The other steps choose their + source by address, through `bind`. + """ + + __slots__ = ("name", "description", "interface", "steps", "source") + + def __init__(self, name, description="", interface=None, steps=(), + source=""): + self.name = name + self.description = description + self.interface = interface + self.steps = tuple(steps) + self.source = source + + def __repr__(self): + return "Scenario(%r, steps=%d)" % (self.name, len(self.steps)) + + +class Reply(object): + """What one step got back, as a flat namespace of named fields. + + A field may hold a list or a mapping. `data.0` indexes a list and + `header.content-type` reads a mapping. The bare name of a list resolves to + its first element in `$step.field`, because a substitution wants one value; + an assertion reads the whole list through `whole()`. + """ + + __slots__ = ("kind", "fields", "ok", "error", "sent", "attempt") + + def __init__(self, kind="", fields=None, ok=True, error="", sent="", + attempt=0): + self.kind = kind + self.fields = dict(fields or {}) + self.ok = ok + self.error = error + self.sent = sent + self.attempt = attempt + + def whole(self, name): + """The value of `name`, a list kept whole, or KeyError.""" + if name in self.fields: + return self.fields[name] + head, _, tail = name.partition(".") + container = self.fields.get(head) + if isinstance(container, dict): + if tail not in container: + raise KeyError("%s reply has no .%s" % (self.kind, name)) + return container[tail] + if isinstance(container, (list, tuple)): + try: + return container[int(tail)] + except ValueError: + raise KeyError("%s is not an index into .%s" % (tail, head)) + except IndexError: + raise KeyError("%s reply has %d item(s) in .%s" + % (self.kind, len(container), head)) + raise KeyError("%s reply has no field .%s" % (self.kind or "this", name)) + + def field(self, name): + """The value `$step.name` substitutes: a list gives its first item.""" + value = self.whole(name) + if isinstance(value, (list, tuple)): + return value[0] if value else "" + return value + + def has(self, name): + """True when `name` carries a value. An empty list is absence.""" + try: + value = self.whole(name) + except KeyError: + return False + if isinstance(value, (list, tuple)): + return bool(value) + return value not in (None, "") + + def summary(self): + parts = ["%s %s" % (self.kind or "reply", "ok" if self.ok else "failed")] + if self.error: + parts.append("(%s)" % (self.error,)) + for key in sorted(self.fields): + if key not in ("text", "raw"): + parts.append("%s=%s" % (key, _short(self.fields[key]))) + return " ".join(parts) + + def detail_lines(self): + """Extra `(label, text)` lines for a failure report.""" + return [] + + def __repr__(self): + return "Reply(%s)" % (self.summary(),) + + +def _short(value, limit=60): + if isinstance(value, (list, tuple)): + text = "[" + ", ".join(str(item) for item in value) + "]" + elif isinstance(value, dict): + text = "{" + ", ".join("%s=%s" % (k, v) for k, v in sorted(value.items())) + "}" + else: + text = str(value) + if len(text) > limit: + text = text[:limit - 3] + "..." + return text diff --git a/xCAT-test/blackboxtest/src/blackboxtest_lib/netutil.py b/xCAT-test/blackboxtest/src/blackboxtest_lib/netutil.py new file mode 100644 index 000000000..940a8cdd5 --- /dev/null +++ b/xCAT-test/blackboxtest/src/blackboxtest_lib/netutil.py @@ -0,0 +1,126 @@ +"""Address arithmetic and interface lookups, standard library only.""" + +import ipaddress +import os +import random + +from .errors import ConfigError + +SYS_NET = "/sys/class/net" + + +def parse_ip(text): + """An `IPv4Address`, or None when `text` is not one.""" + try: + return ipaddress.IPv4Address(str(text).strip()) + except (ipaddress.AddressValueError, ValueError): + return None + + +def ip_in(address, network): + """True when `address` is inside the CIDR block `network`.""" + ip = parse_ip(address) + try: + net = ipaddress.IPv4Network(str(network).strip(), strict=False) + except (ipaddress.AddressValueError, ipaddress.NetmaskValueError, ValueError): + return False + return ip is not None and ip in net + + +def parse_range(text): + """`first-last` as a sorted pair of addresses, or None. + + A dynamic pool rarely lines up on a prefix boundary, so it is written as + two addresses rather than as a CIDR block. + """ + first, sep, last = str(text).strip().partition("-") + low, high = parse_ip(first), parse_ip(last) + if not sep or low is None or high is None: + return None + return (low, high) if low <= high else (high, low) + + +def ip_in_range(address, text): + """True when `address` is inside the inclusive range `first-last`.""" + bounds = parse_range(text) + ip = parse_ip(address) + return bounds is not None and ip is not None and bounds[0] <= ip <= bounds[1] + + +def hex_ip(address): + """The eight hex digits a PXE or grub2 loader asks for its address by. + + The loader does this encoding, not xCAT: 10.99.1.11 becomes 0A63010B. + """ + ip = parse_ip(address) + if ip is None: + raise ValueError("not an IPv4 address: %r" % (address,)) + return "%08X" % (int(ip),) + + +def hex_net(address, prefix): + """The name of a per-network configuration. + + A loader that finds no file for its own address asks for shorter + prefixes of it. The per-network file carries as many hex digits as the + netmask covers, rounded up: 10.99.1.0/24 is 0A6301. + """ + try: + bits = int(prefix) + except (TypeError, ValueError): + bits = -1 + if not 0 <= bits <= 32: + raise ValueError("not a prefix length: %r" % (prefix,)) + return hex_ip(address)[:(bits + 3) // 4] + + +def reverse_name(address): + """The in-addr.arpa name a PTR is asked for by.""" + ip = parse_ip(address) + if ip is None: + raise ValueError("not an IPv4 address: %r" % (address,)) + return ip.reverse_pointer + + +def normalise_mac(text): + """`aa:bb:..`, `aa-bb-..` or `aabb..` in the lowercase colon form.""" + raw = str(text).strip().lower().replace("-", ":") + if ":" not in raw and len(raw) == 12: + raw = ":".join(raw[i:i + 2] for i in range(0, 12, 2)) + parts = raw.split(":") + try: + octets = [int(part, 16) for part in parts] + except ValueError: + octets = [] + if len(parts) != 6 or len(octets) != 6 \ + or any(len(p) > 2 or not p for p in parts): + raise ConfigError("not a MAC address: %r" % (text,)) + return ":".join("%02x" % octet for octet in octets) + + +def dashed_mac(text): + """The `aa-bb-..` form a grub2 or pxelinux per-MAC file is named by.""" + return normalise_mac(text).replace(":", "-") + + +def mac_bytes(mac): + return bytes(int(part, 16) for part in normalise_mac(mac).split(":")) + + +def random_mac(): + """A locally administered unicast MAC. + + The host's own address would make NetworkManager and any dhclient read + the replies as theirs. With a synthetic one, the kernel drops the unicast + replies because neither the MAC nor the IP is local. + """ + return ":".join("%02x" % o for o in [0x02] + [random.randint(0, 255) + for _ in range(5)]) + + +def random_xid(): + return random.randint(1, 0xFFFFFFFF) + + +def interface_exists(name): + return os.path.isdir(os.path.join(SYS_NET, name)) diff --git a/xCAT-test/provtest/src/provtest_lib/proc.py b/xCAT-test/blackboxtest/src/blackboxtest_lib/proc.py similarity index 100% rename from xCAT-test/provtest/src/provtest_lib/proc.py rename to xCAT-test/blackboxtest/src/blackboxtest_lib/proc.py diff --git a/xCAT-test/blackboxtest/src/blackboxtest_lib/report.py b/xCAT-test/blackboxtest/src/blackboxtest_lib/report.py new file mode 100644 index 000000000..05e0790df --- /dev/null +++ b/xCAT-test/blackboxtest/src/blackboxtest_lib/report.py @@ -0,0 +1,102 @@ +"""TAP version 13 output, which `prove` and xcattest both read. + +A failure carries what was expected, what came back, the reply and what was +sent, so it can be understood from the log after the test network is gone. +""" + +import sys + +from .errors import EXIT_FAILED, EXIT_OK + + +class Record(object): + """One assertion outcome.""" + + __slots__ = ("scenario", "step", "description", "ok", "skip_reason", + "expected", "actual", "detail", "reply", "sent", "attempt", + "attempts") + + def __init__(self, scenario, step, description, ok, skip_reason=None, + expected=None, actual=None, detail="", reply=None, sent="", + attempt=0, attempts=0): + self.scenario = scenario + self.step = step + self.description = description + self.ok = ok + self.skip_reason = skip_reason + self.expected = expected + self.actual = actual + self.detail = detail + self.reply = reply + self.sent = sent + self.attempt = attempt + self.attempts = attempts + + +class TapReporter(object): + """Writes one TAP line per record, and the plan at the end.""" + + def __init__(self, stream=None): + self.stream = stream or sys.stdout + self.count = 0 + self.failed = 0 + self.skipped = 0 + + def write(self, text): + self.stream.write(text) + self.stream.flush() + + def start(self): + self.write("TAP version 13\n") + + def add(self, record): + self.count += 1 + line = "%s %d - %s/%s: %s" % ( + "not ok" if not record.ok and record.skip_reason is None else "ok", + self.count, record.scenario, record.step, record.description) + if record.skip_reason is not None: + self.skipped += 1 + self.write("%s # SKIP %s\n" % (line, record.skip_reason)) + elif record.ok: + self.write(line + "\n") + else: + self.failed += 1 + self.write(line + "\n ---\n") + for key, value in diagnostics(record): + self.write(" %s: %s\n" % (key, value)) + self.write(" ...\n") + + def finish(self): + self.write("1..%d\n# passed %d, failed %d, skipped %d\n" + % (self.count, self.count - self.failed - self.skipped, + self.failed, self.skipped)) + + def exit_code(self): + return EXIT_FAILED if self.failed else EXIT_OK + + +def diagnostics(record): + """`(key, value)` lines explaining a failed record.""" + lines = [] + if record.expected is not None: + lines.append(("expected", repr(record.expected))) + lines.append(("received", repr(record.actual))) + if record.detail: + lines.append(("detail", _oneline(record.detail))) + if record.reply is not None: + lines.append(("reply", _oneline(record.reply.summary()))) + lines.extend((key, _oneline(text, 2000)) + for key, text in record.reply.detail_lines()) + else: + lines.append(("reply", "none received")) + if record.sent: + lines.append(("sent", _oneline(record.sent))) + if record.attempts: + lines.append(("attempt", "%d of %d" % (record.attempt, record.attempts))) + return lines + + +def _oneline(text, limit=400): + """One line, because a TAP diagnostic is read line by line.""" + flat = " ".join(str(text).split()) + return flat if len(flat) <= limit else flat[:limit - 3] + "..." diff --git a/xCAT-test/blackboxtest/src/blackboxtest_lib/runner.py b/xCAT-test/blackboxtest/src/blackboxtest_lib/runner.py new file mode 100644 index 000000000..0ba7b4ecd --- /dev/null +++ b/xCAT-test/blackboxtest/src/blackboxtest_lib/runner.py @@ -0,0 +1,303 @@ +"""Running scenarios: one step at a time, in file order. + +Each reply is bound under its step's name before the step's assertions run, +so a later step, or an assertion, can use it: TFTP fetches a grub config, an +extract step reads the kernel path out of it, and HTTP fetches that path. +""" + +import re +import sys +import time + +from . import (assertions, dhcp, dnsc, httpc, netutil, proc, report, steps, + subst, tftpc, xcatc) +from .errors import BlackboxError, ConfigError +from .model import Reply + + +class RunOptions(object): + """Command-line overrides, applied over what the .conf says.""" + + def __init__(self, interface=None, bind=None, timeout=None, retries=None, + verbose=0): + self.interface = interface + self.bind = bind + self.timeout = timeout + self.retries = retries + self.verbose = verbose + + +def preflight(scenarios): + """Refuse a host that lacks a client program or DHCP's root and scapy. + + Done before any output: a run of skips would be a green result that + proves nothing. + """ + types = set(step.type for scenario in scenarios for step in scenario.steps) + for step_type in sorted(types): + kind = steps.TYPES[step_type] + if kind.program: + proc.require(kind.program, step_type) + if any(steps.TYPES[t].family == "dhcp" for t in types): + dhcp.require() + + +def run(scenarios, reporter, options): + """Run every scenario and return the process exit code.""" + preflight(scenarios) + reporter.start() + for scenario in scenarios: + _run_scenario(scenario, reporter, options) + reporter.finish() + return reporter.exit_code() + + +def _run_scenario(scenario, reporter, options): + session = None + if any(steps.TYPES[s.type].family == "dhcp" for s in scenario.steps): + interface = options.interface or scenario.interface + # A shared .conf may name a NIC that only some hosts have. + if not interface or not netutil.interface_exists(interface): + reason = "no such interface: %s" % (interface,) if interface \ + else "no interface; pass -i" + for step in scenario.steps: + for assertion in step.assertions: + reporter.add(report.Record(scenario.name, step.name, + assertion.render(), True, + skip_reason=reason)) + return + session = dhcp.Session(dhcp.Wire(interface)) + + context = subst.Context() + try: + for step in scenario.steps: + try: + reply, extras = _run_step(step, context, session, options) + except BlackboxError as exc: + reporter.add(report.Record(scenario.name, step.name, + "step could not be run", False, + detail=str(exc))) + return + if options.verbose: + sys.stderr.write("# %s/%s: %s\n" % ( + scenario.name, step.name, + reply.summary() if reply else "no reply")) + _record(scenario, step, reply, context, extras, reporter) + finally: + if session is not None: + session.wire.close() + + +def _run_step(step, context, session, options): + """Run one step; return `(reply or None, extras)`, the step name bound.""" + kind = steps.TYPES[step.type] + timeout = options.timeout if options.timeout is not None else step.timeout + retries = max(1, options.retries if options.retries is not None + else step.retries) + + if kind.family == "dhcp": + reply, extras, sent = dhcp.run_step(step, session, context, timeout, + retries) + extras["sent"] = sent + extras["attempts"] = retries + else: + handler = HANDLERS[step.type] + param = _resolver(step, context, options) + expect = step.expect or ("any" if kind.family == "local" else "ok") + # Retry only a transport that was expected to work. + for attempt in range(1, retries + 1): + reply = handler(step, param, timeout) + reply.attempt = attempt + if expect != "ok" or reply.ok or attempt == retries: + break + time.sleep(0.2) + extras = {"attempt": attempt, "attempts": retries, "expect": expect, + "got": "ok" if reply.ok else "failed", + "met": reply.ok == (expect == "ok")} + + if reply is not None: + context.bind(step.name, reply) + return reply, extras + + +def _record(scenario, step, reply, context, extras, reporter): + common = dict(reply=reply, sent=reply.sent if reply else extras.get("sent", ""), + attempt=extras["attempt"], attempts=extras["attempts"]) + + # The expectation is a test point of its own. A service step that waited + # for an answer and got none fails with no assertion about it. + expect = extras.get("expect", "any") + if expect != "any": + reporter.add(report.Record( + scenario.name, step.name, "expect %s" % (expect,), extras["met"], + expected=expect, actual=extras["got"], + detail=reply.error if reply else "", **common)) + + for assertion in step.assertions: + try: + result = assertions.evaluate(assertion, reply, context, extras) + except ConfigError as exc: + reporter.add(report.Record(scenario.name, step.name, + assertion.render(), False, + detail=str(exc), **common)) + continue + reporter.add(report.Record( + scenario.name, step.name, assertion.render(), result.ok, + expected=result.expected, actual=result.actual, + detail=result.detail, **common)) + + +# --------------------------------------------------------------------------- +# handlers for the service and local steps + + +def _resolver(step, context, options): + """A `param(key)` that resolves `$step.field` and applies `-b`.""" + + def param(key, default=""): + value = step.param(key, default) + if key == "bind" and not value: + value = options.bind or "" + if value in (None, ""): + return value + return subst.resolve(value, context) + + return param + + +def _dns(step, param, timeout): + return dnsc.query(server=param("server"), name=param("name"), + rrtype=param("rrtype") or "A", port=_int(param("port"), 53), + bind=param("bind"), timeout=timeout, + recursion=_bool(param("recursion"), True)) + + +def _tftp(step, param, timeout): + return tftpc.fetch(server=param("server"), path=param("path"), + port=_int(param("port"), 69), + mode=param("mode") or "octet", timeout=timeout) + + +def _http(step, param, timeout): + url = param("url") or httpc.url_for(param("server"), param("path"), + _int(param("port"), 80)) + return httpc.fetch(url=url, method=param("method") or "GET", + bind=param("bind"), timeout=timeout, + headers=_lines(param("header")), + insecure=_bool(param("insecure"), False)) + + +def _xcatreq(step, param, timeout): + elements = [tuple(part.strip() for part in line.partition("=")[::2]) + for line in _lines(param("element")) if not line.startswith("#")] + return xcatc.request( + server=param("server"), command=param("command"), + port=_int(param("port"), 3001), elements=elements, bind=param("bind"), + timeout=timeout, cert=param("cert") or None, key=param("key") or None, + callback_port=param("callback_port") or None, + callback_listen=param("callback_listen") or None, + callback_reply=param("callback_reply") or None, + callback_wait=_float(param("callback_wait"), 2.0), + raw=param("raw") or None, source_port=_int(param("source_port"), 0)) + + +def _monitor(step, param, timeout): + return xcatc.monitor(server=param("server"), send=_lines(param("send")), + port=_int(param("port"), 3002), bind=param("bind"), + timeout=timeout, + source_port=_int(param("source_port"), 0)) + + +def _flowrequest(step, param, timeout): + return xcatc.flowrequest( + server=param("server"), port=_int(param("port"), 3001), + message=param("message") or "resourcerequest: xcatd", + bind=param("bind"), timeout=timeout, + source_port=_int(param("source_port"), 0), + expected=_int(param("replies"), 1)) + + +def _findme(step, param, timeout): + return xcatc.findme( + server=param("server"), + payload=param("payload") or xcatc.discovery_packet(), + port=_int(param("port"), 3001), + encoding=(param("encoding") or "gzip").lower(), bind=param("bind"), + source_port=_int(param("source_port"), 301), timeout=timeout, + callback_listen=_int(param("callback_listen"), 3001), + callback_wait=_float(param("callback_wait"), 5.0)) + + +def _extract(step, param, timeout): + """Read a value out of an earlier reply, so a later step can fetch it. + + A grub2 config names a kernel. Fetching the name read out of the config + tests that what the node was told to fetch can be fetched, without the + scenario stating what xCAT will call it. + """ + pattern = step.param("pattern") or "" + try: + match = re.compile(pattern, re.MULTILINE).search(param("from") or "") + except re.error as exc: + raise ConfigError("bad regex %r: %s" % (pattern, exc), step.source) + groups = list(match.groups()) if match else [] + index = _int(param("group"), 1 if groups else 0) + if match and index > len(groups): + raise ConfigError("group=%d, but %r has %d capturing group(s)" + % (index, pattern, len(groups)), step.source) + fields = { + "value": (match.group(index) or "") if match else "", + "groups": [text or "" for text in groups], + "matched": bool(match), + "count": len(groups), + } + return Reply("extract", fields, ok=bool(match), + error="" if match else "%r matched nothing" % (pattern,), + sent="match %r" % (pattern,)) + + +def _noop(step, param, timeout): + return Reply("noop") + + +HANDLERS = { + "dns": _dns, "tftp": _tftp, "http": _http, "xcatreq": _xcatreq, + "monitor": _monitor, "flowrequest": _flowrequest, "findme": _findme, + "extract": _extract, "noop": _noop, +} + +assert set(HANDLERS) == set(t for t, k in steps.TYPES.items() + if k.family != "dhcp") + + +def _lines(block): + return [line.strip() for line in (block or "").splitlines() if line.strip()] + + +def _int(value, default): + if value in (None, ""): + return default + try: + return int(str(value).strip(), 0) + except ValueError: + raise ConfigError("expected a number, got %r" % (value,)) + + +def _float(value, default): + if value in (None, ""): + return default + try: + return float(str(value).strip()) + except ValueError: + raise ConfigError("expected a number, got %r" % (value,)) + + +def _bool(value, default): + text = str(value or "").strip().lower() + if not text: + return default + if text in ("1", "yes", "true", "on"): + return True + if text in ("0", "no", "false", "off"): + return False + raise ConfigError("expected yes or no, got %r" % (value,)) diff --git a/xCAT-test/blackboxtest/src/blackboxtest_lib/steps.py b/xCAT-test/blackboxtest/src/blackboxtest_lib/steps.py new file mode 100644 index 000000000..a2824a426 --- /dev/null +++ b/xCAT-test/blackboxtest/src/blackboxtest_lib/steps.py @@ -0,0 +1,110 @@ +"""What each step type accepts, needs and returns. + +Adding a protocol means adding a row here and a handler in `runner`. The +table is read offline by `config` and `validate`, so a misspelt key or reply +field fails before a packet is sent. +""" + +from . import dhcp, dhcpopts + +#: Keys every step accepts. +COMMON_KEYS = frozenset(["type", "expect", "timeout", "retries", "assert"]) + + +class StepType(object): + """One row of the table. + + `family` is `dhcp` for the raw-socket client, `service` for a request to + a server, and `local` for a step that asks nothing of the network. + """ + + __slots__ = ("family", "keys", "required", "fields", "timeout", "retries", + "program") + + def __init__(self, family, keys=(), required=(), fields=(), timeout=5.0, + retries=2, program=None): + self.family = family + self.keys = COMMON_KEYS | frozenset(keys) + self.required = tuple(required) + self.fields = frozenset(fields) + self.timeout = timeout + self.retries = retries + self.program = program + + def expectations(self): + """The `expect=` values a step of this family can give.""" + if self.family == "dhcp": + return frozenset(["offer", "ack", "nak", "bootreply", "any", "none"]) + return frozenset(["ok", "fail", "any"]) + + +def _dhcp(): + # Retransmits come quicker than a service request's retries: a DHCP + # server that is up answers in milliseconds. + return StepType("dhcp", dhcp.STEP_KEYS, timeout=2.0, retries=3) + + +#: `bind` is the client's own address. xcatd names a client by the reverse +#: lookup of the address its connection came from, so the source address is +#: under test and is never left to the kernel's route selection. +TYPES = dict((name, _dhcp()) for name in dhcp.STEP_TYPES) +TYPES.update({ + "dns": StepType( + "service", ["bind", "server", "port", "name", "rrtype", "recursion"], + required=("server", "name"), program="dig", + fields=["status", "flags", "count", "data", "type", "ttl", "name", + "question", "answers", "authority", "server", "rcode"]), + "tftp": StepType( + "service", ["bind", "server", "port", "path", "mode"], + required=("server", "path"), program="tftp", + fields=["ok", "size", "sha256", "text", "error", "path", "server"]), + "http": StepType( + "service", ["bind", "server", "port", "path", "url", "method", + "header", "insecure"], program="curl", + fields=["status", "size", "sha256", "text", "url", "header", + "content_type", "error", "ok"]), + "xcatreq": StepType( + "service", ["bind", "server", "port", "command", "element", "raw", + "callback_port", "callback_listen", "callback_reply", + "callback_wait", "cert", "key", "source_port"], + required=("server", "command"), + # getcredentials nests its payload as . + fields=["destiny", "kernel", "initrd", "kcmdline", "imgserver", "name", + "error", "serverdone", "elements", "data", "text", "handshake", + "ok", "callback_seen", "callback_data", "content", "desc"]), + "monitor": StepType( + "service", ["bind", "server", "port", "send", "source_port"], + required=("server", "send"), + fields=["greeting", "lines", "text", "ok", "closed", "error"]), + "flowrequest": StepType( + "service", ["bind", "server", "port", "message", "replies", + "source_port"], + required=("server",), fields=["replies", "count", "ok", "error"]), + "findme": StepType( + "service", ["bind", "server", "port", "payload", "encoding", + "source_port", "callback_listen", "callback_wait"], + required=("server",), fields=["callbacks", "count", "ok", "error"]), + "extract": StepType( + "local", ["from", "pattern", "group"], required=("from", "pattern"), + fields=["value", "groups", "matched", "count"]), + "noop": StepType("local"), +}) + +#: Reply fields read by key, as `header.content-type`. +MAPPING_FIELDS = frozenset(["header", "elements"]) + + +def known_field(step_type, name): + """True when a reply of `step_type` can carry a field called `name`. + + A list is indexed as `data.0`, a mapping read as `header.content-type`. + """ + kind = TYPES[step_type] + if kind.family == "dhcp": + return dhcpopts.known_field(name) + if name in kind.fields: + return True + head, _, tail = name.partition(".") + if head not in kind.fields or not tail: + return False + return head in MAPPING_FIELDS or tail.lstrip("-").isdigit() diff --git a/xCAT-test/provtest/src/provtest_lib/subst.py b/xCAT-test/blackboxtest/src/blackboxtest_lib/subst.py similarity index 55% rename from xCAT-test/provtest/src/provtest_lib/subst.py rename to xCAT-test/blackboxtest/src/blackboxtest_lib/subst.py index 28728229b..b68efc61e 100644 --- a/xCAT-test/provtest/src/provtest_lib/subst.py +++ b/xCAT-test/blackboxtest/src/blackboxtest_lib/subst.py @@ -1,15 +1,9 @@ """Runtime references to earlier replies: `$step.field`. -Variables are configparser's job. `%(name)s` is interpolated by -`BasicInterpolation` out of `[vars]` and `--set` before a value ever reaches -this module, and `BasicInterpolation` gives `$` no meaning of its own, so the -two schemes share a file with no escaping. - -What configparser cannot do is `$grubcfg.text`, because that value is a reply -that has not arrived when the file is read. Cross-stage assertions are the -whole point of this suite -- TFTP fetches a config, HTTP fetches what that -config named -- so this form is resolved at step-execution time instead, when -the earlier replies exist. +`%(name)s` is configparser's job and is gone before a value reaches this +module. `$step.field` cannot be: its value is a reply that has not arrived +when the file is read. `BasicInterpolation` gives `$` no meaning, so the two +forms share a file with no escaping. """ import re @@ -23,27 +17,20 @@ VAR_RE = re.compile(r"%\(([^)]+)\)s") class Context(object): - """Every reply a `$reference` can resolve against, at one point in a run.""" + """The replies a `$reference` can resolve against, by binding name.""" def __init__(self): self.bindings = {} - self.order = [] def bind(self, name, reply): - if name not in self.bindings: - self.order.append(name) self.bindings[name] = reply - def get_binding(self, name): + def get(self, name): if name not in self.bindings: raise ConfigError( - "$%s is not available here; no step of that name has run yet" - % (name,)) + "$%s is not available here; nothing has bound it yet" % (name,)) return self.bindings[name] - def __repr__(self): - return "Context(bindings=%r)" % (self.order,) - def references(text): """Every `$step.field` in a value, as `[(step, field), ...]`.""" @@ -65,18 +52,17 @@ def resolve(text, context): return None def replace(match): - name, field = match.group(1), match.group(2).rstrip(".") - reply = context.get_binding(name) + reply = context.get(match.group(1)) try: - return format_value(reply.field(field)) + return format_value(reply.field(match.group(2).rstrip("."))) except KeyError as exc: - raise ConfigError(str(exc)) + raise ConfigError(str(exc).strip("'\"")) return REF_RE.sub(replace, str(text)) def format_value(value): - """Render a resolved value the way a .conf author would have typed it.""" + """Render a value the way a .conf author would have typed it.""" if isinstance(value, bool): return "yes" if value else "no" if isinstance(value, bytes): diff --git a/xCAT-test/provtest/src/provtest_lib/tftpc.py b/xCAT-test/blackboxtest/src/blackboxtest_lib/tftpc.py similarity index 82% rename from xCAT-test/provtest/src/provtest_lib/tftpc.py rename to xCAT-test/blackboxtest/src/blackboxtest_lib/tftpc.py index b94e4b3cc..c268841cf 100644 --- a/xCAT-test/provtest/src/provtest_lib/tftpc.py +++ b/xCAT-test/blackboxtest/src/blackboxtest_lib/tftpc.py @@ -35,17 +35,14 @@ def build(server, path, port=69, mode="octet", local=None): "-c", "get", str(path), str(local)] -def fetch(server, path, port=69, mode="octet", timeout=10.0, keep=None): +def fetch(server, path, port=69, mode="octet", timeout=10.0): """Fetch one file and return a `Reply` describing what arrived.""" - handle, local = tempfile.mkstemp(prefix="provtest-tftp-") + handle, local = tempfile.mkstemp(prefix="blackboxtest-tftp-") os.close(handle) try: argv = build(server, path, port, mode, local) done = proc.run(argv, timeout=float(timeout) + 5.0) - reply = _decode(done, local, server, path) - if keep and reply.fields.get("ok"): - _copy(local, keep) - return reply + return _decode(done, local, server, path) finally: try: os.unlink(local) @@ -85,9 +82,8 @@ def _decode(done, local, server, path): "path": path, "server": server, } - reply = Reply(kind="tftp", fields=fields, ok=ok, error=error, - sent=done.command(), raw=body) - return reply + return Reply(kind="tftp", fields=fields, ok=ok, error=error, + sent=done.command()) def _digest(local): @@ -97,9 +93,3 @@ def _digest(local): digest.update(chunk) return digest.hexdigest() - -def _copy(local, target): - with open(local, "rb") as source: - with open(target, "wb") as sink: - for chunk in iter(lambda: source.read(65536), b""): - sink.write(chunk) diff --git a/xCAT-test/blackboxtest/src/blackboxtest_lib/validate.py b/xCAT-test/blackboxtest/src/blackboxtest_lib/validate.py new file mode 100644 index 000000000..8098a6fbf --- /dev/null +++ b/xCAT-test/blackboxtest/src/blackboxtest_lib/validate.py @@ -0,0 +1,110 @@ +"""Offline checks: what `validate` reports without opening a socket. + +Walks each scenario the way the runner will, so a REQUEST with no DISCOVER +before it, a misspelt reply field, or a reference to a step that has not run +is reported in a checkout, with no root, no network and no server. +""" + +from . import dhcp, netutil, steps, subst +from .assertions import META_TARGETS + + +def scenario_problems(scenario): + """Every problem in one scenario, as strings. Empty means consistent.""" + problems = [] + bound = {} # binding name -> the step type that bound it + state = dhcp.INIT + + for step in scenario.steps: + where = "%s: %s/%s" % (scenario.source, scenario.name, step.name) + kind = steps.TYPES[step.type] + + def check(text, names): + for name, field in subst.references(text): + if name not in names: + problems.append("%s: $%s.%s is used before anything binds $%s" + % (where, name, field, name)) + elif not steps.known_field(names[name], field): + problems.append("%s: a %s reply has no field .%s" + % (where, names[name], field)) + + if step.name in bound: + problems.append("%s: step name %r is used twice" % (where, step.name)) + for key in kind.required: + if not step.param(key): + problems.append("%s: a %s step needs %s=" % (where, step.type, key)) + if step.type == "http" and not step.param("url") \ + and not (step.param("server") and step.param("path")): + problems.append("%s: an http step needs url=, or server= and path=" + % (where,)) + if step.expect and step.expect not in kind.expectations(): + problems.append("%s: unknown expect=%s (known: %s)" + % (where, step.expect, + ", ".join(sorted(kind.expectations())))) + for value in step.params.values(): + check(value, bound) + + if kind.family == "dhcp": + state = _walk_dhcp(step, state, bound, where, problems) + bound[step.name] = step.type + + # An assertion runs after its own step has bound its reply. + for assertion in step.assertions: + check(assertion.value, bound) + target = assertion.target + if target.startswith("$"): + if len(subst.references(target)) != 1: + problems.append("%s: target %r must be one $step.field" + % (where, target)) + check(target, bound) + elif target not in META_TARGETS \ + and not steps.known_field(step.type, target): + problems.append("%s: a %s reply has no field %r" + % (where, step.type, target)) + return problems + + +def _walk_dhcp(step, state, bound, where, problems): + """Check one DHCP step against the client state; return the next state.""" + mac = step.param("mac") or "" + if mac and "$" not in mac and not subst.variables(mac): + try: + netutil.normalise_mac(mac) + except Exception as exc: + problems.append("%s: %s" % (where, exc)) + + trans = dhcp.transition(state, step.type) + if trans is None: + legal = sorted(s for s, kind in dhcp.TRANSITIONS if kind == step.type) + problems.append("%s: a %s step is not legal in state %s (legal in: %s)" + % (where, step.type, state, ", ".join(legal))) + return state + for key in trans.requires: + if not step.param(key): + problems.append("%s: a %s step in state %s needs %s=" + % (where, step.type, state, key)) + expect = step.expect or trans.default_expect + if expect not in ("none", "any") and expect not in trans.expects: + problems.append("%s: expect=%s is not a reply to %s in state %s " + "(possible: %s)" + % (where, expect, step.type, state, + ", ".join(sorted(trans.expects)) or "nothing")) + answered = expect != "none" + if answered: + bound["reply"] = step.type + for name in trans.binds: + bound[name] = step.type + return dhcp.advance(state, trans, answered) + + +def required_variables(scenarios): + """Every `%(name)s` the scenarios, loaded raw, still need a value for.""" + names = set() + for scenario in scenarios: + for step in scenario.steps: + for value in step.params.values(): + names.update(subst.variables(value)) + for assertion in step.assertions: + names.update(subst.variables(assertion.value)) + names.update(subst.variables(assertion.target)) + return names diff --git a/xCAT-test/provtest/src/provtest_lib/xcatc.py b/xCAT-test/blackboxtest/src/blackboxtest_lib/xcatc.py similarity index 98% rename from xCAT-test/provtest/src/provtest_lib/xcatc.py rename to xCAT-test/blackboxtest/src/blackboxtest_lib/xcatc.py index 679b2fafc..751cf5668 100644 --- a/xCAT-test/provtest/src/provtest_lib/xcatc.py +++ b/xCAT-test/blackboxtest/src/blackboxtest_lib/xcatc.py @@ -191,8 +191,7 @@ def request(server, command, port=3001, elements=(), bind=None, timeout=10.0, fields["handshake"] = handshake fields["ok"] = handshake and bool(text) and not fields.get("error") return Reply(kind="xcatreq", fields=fields, ok=fields["ok"], error=error, - sent=body.strip().replace("\n", " "), - raw=text.encode("utf-8", "replace")) + sent=body.strip().replace("\n", " ")) def decode_elements(text): @@ -429,7 +428,7 @@ def findme(server, payload, port=3001, encoding="gzip", bind=None, } return Reply(kind="findme", fields=fields, ok=fields["ok"], error=error, sent="findme %d bytes from port %s (%s)" - % (len(body), source_port, encoding), raw=body) + % (len(body), source_port, encoding)) def discovery_packet(elements=()): diff --git a/xCAT-test/blackboxtest/tests/context.py b/xCAT-test/blackboxtest/tests/context.py new file mode 100644 index 000000000..6e37d5224 --- /dev/null +++ b/xCAT-test/blackboxtest/tests/context.py @@ -0,0 +1,45 @@ +"""Put `src/` first on sys.path, so the tests import the checkout. + +Every test module imports this one first. +""" + +import os +import sys +import tempfile + +TESTS = os.path.dirname(os.path.realpath(__file__)) +ROOT = os.path.dirname(TESTS) +SRC = os.path.join(ROOT, "src") +LIB = os.path.join(SRC, "blackboxtest_lib") +CONF = os.path.join(ROOT, "conf") + +if SRC not in sys.path: + sys.path.insert(0, SRC) + + +def conf_files(): + """Every shipped scenario file, sorted.""" + found = [] + for directory, _, names in os.walk(CONF): + found.extend(os.path.join(directory, n) for n in names + if n.endswith(".conf")) + return sorted(found) + + +def modules(): + """Every module of the tool, as `(name, path)`.""" + found = [("blackboxtest", os.path.join(SRC, "blackboxtest"))] + for name in sorted(os.listdir(LIB)): + if name.endswith(".py"): + found.append((name[:-3], os.path.join(LIB, name))) + return found + + +def write_conf(test, text, name="t.conf"): + """Write `text` to a scratch file removed after `test`.""" + directory = tempfile.mkdtemp(prefix="blackboxtest-") + test.addCleanup(__import__("shutil").rmtree, directory, True) + path = os.path.join(directory, name) + with open(path, "w") as handle: + handle.write(text) + return path diff --git a/xCAT-test/blackboxtest/tests/test_assertions.py b/xCAT-test/blackboxtest/tests/test_assertions.py new file mode 100644 index 000000000..9f0fbb726 --- /dev/null +++ b/xCAT-test/blackboxtest/tests/test_assertions.py @@ -0,0 +1,171 @@ +"""The assertion language and `$step.field` substitution.""" + +import unittest + +import context # noqa: F401 + +from blackboxtest_lib import assertions, subst +from blackboxtest_lib.dhcpopts import DhcpReply +from blackboxtest_lib.errors import ConfigError +from blackboxtest_lib.model import Reply + + +def check(line, reply, extras=None, ctx=None): + return assertions.evaluate(assertions.parse(line), reply, + ctx or subst.Context(), extras) + + +def offer(options=None): + fields = {"msgtype": "OFFER", "xid": 7, "yiaddr": "10.0.0.101", + "siaddr": "10.0.0.1", "ciaddr": "0.0.0.0", "giaddr": "0.0.0.0", + "chaddr": "02:00:00:00:00:01", "file": "", "sname": "", + "secs": 0, "flags": 0, "src_ip": "10.0.0.1", "src_mac": ""} + base = {54: "10.0.0.1", 51: 600} + base.update(options or {}) + return DhcpReply(fields, base) + + +DNS = Reply("dns", {"status": "NOERROR", "data": ["10.0.0.11", "10.0.1.11"], + "count": 2, "header": {"content-type": "text/plain"}, + "empty": [], "text": "line one\nlinux /boot/k\n"}) + + +class Parsing(unittest.TestCase): + + def test_the_three_shapes(self): + self.assertEqual(assertions.parse("x present").value, None) + self.assertEqual(assertions.parse("x ==").value, "") + self.assertEqual(assertions.parse("x contains a b c").value, "a b c") + + def test_what_is_refused(self): + for line in ("x", "x ~= 1", "x present 1", "x matches ("): + self.assertRaises(ConfigError, assertions.parse, line) + + def test_blank_and_comment_lines_are_skipped(self): + block = "\n # note\n x == 1\n\n y present\n" + self.assertEqual(len(assertions.parse_block(block)), 2) + + +class Comparison(unittest.TestCase): + + def test_equality_is_address_then_number_then_text(self): + self.assertTrue(check("yiaddr == 10.0.0.101", offer()).ok) + self.assertTrue(check("51 == 0x258", offer()).ok) + self.assertTrue(check("status == NOERROR", DNS).ok) + self.assertFalse(check("status == noerror", DNS).ok) + + def test_a_list_means_any_element_or_the_whole(self): + self.assertTrue(check("data == 10.0.1.11", DNS).ok) + self.assertTrue(check("data == 10.0.0.11, 10.0.1.11", DNS).ok) + self.assertFalse(check("data == 10.0.2.11", DNS).ok) + + def test_membership_of_a_cidr_a_range_and_a_list(self): + self.assertTrue(check("yiaddr in 10.0.0.0/24", offer()).ok) + self.assertTrue(check("yiaddr in 10.0.0.100-10.0.0.110", offer()).ok) + self.assertFalse(check("yiaddr in 10.0.0.102-10.0.0.110", offer()).ok) + self.assertTrue(check("status in NXDOMAIN, NOERROR", DNS).ok) + self.assertTrue(check("status not-in SERVFAIL, REFUSED", DNS).ok) + + def test_text_and_numeric_operators(self): + self.assertTrue(check("text matches ^linux\\s", DNS).ok) + self.assertTrue(check("text contains /boot/k", DNS).ok) + self.assertTrue(check("status starts-with NO", DNS).ok) + self.assertTrue(check("status ends-with ERROR", DNS).ok) + self.assertTrue(check("count >= 2", DNS).ok) + self.assertFalse(check("status > 2", DNS).ok) + + def test_indexing_and_mapping(self): + self.assertTrue(check("data.1 == 10.0.1.11", DNS).ok) + self.assertTrue(check("header.content-type == text/plain", DNS).ok) + self.assertFalse(check("data.5 present", DNS).ok) + + +class Presence(unittest.TestCase): + + def test_an_empty_list_and_a_zeroed_header_are_absence(self): + self.assertTrue(check("empty absent", DNS).ok) + self.assertTrue(check("giaddr absent", offer()).ok) + self.assertTrue(check("yiaddr present", offer()).ok) + + def test_no_boot_file_is_absent_not_empty(self): + self.assertTrue(check("bootfile absent", offer()).ok) + self.assertFalse(check("bootfile matches ^$", offer()).ok) + + def test_a_negative_comparison_holds_against_an_absent_target(self): + self.assertTrue(check("bootfile != http://x/nodes/n1", offer()).ok) + self.assertTrue(check("67 not-in a, b", offer()).ok) + self.assertFalse(check("bootfile == x", offer()).ok) + + def test_no_reply_fails_every_positive_assertion(self): + result = check("msgtype == OFFER", None) + self.assertFalse(result.ok) + self.assertIn("not present", result.detail) + self.assertTrue(check("msgtype absent", None).ok) + + def test_meta_targets_come_from_the_runner(self): + self.assertTrue(check("offers == 1", None, {"offers": 1}).ok) + self.assertFalse(check("attempt present", DNS).ok) + + +class DhcpFields(unittest.TestCase): + + def test_options_by_number_name_and_prefix(self): + reply = offer() + for target in ("54", "option:54", "server_id", "opt54"): + self.assertTrue(check("%s == 10.0.0.1" % target, reply).ok, target) + + def test_the_boot_file_prefers_option_67(self): + reply = offer({67: "grub2.aarch64"}) + reply.fields["file"] = "pxelinux.0" + self.assertTrue(check("bootfile == grub2.aarch64", reply).ok) + self.assertTrue(check("file == pxelinux.0", reply).ok) + + def test_the_dns_name_prefers_option_81_and_falls_back_to_12(self): + self.assertTrue(check("dns_name == n1", offer({12: "n1"})).ok) + self.assertTrue(check("dns_name == n2", + offer({12: "n1", 81: "n2"})).ok) + self.assertTrue(check("dns_name absent", offer()).ok) + + def test_fqdn_flags_are_read_from_the_raw_option(self): + from blackboxtest_lib import dhcpopts + raw = dhcpopts.pack_options([(81, dhcpopts.encode_fqdn("SO:n1"))]) + reply = offer({81: "n1"}) + reply.raw_options = raw + self.assertTrue(check("fqdn_flags == SO", reply).ok) + self.assertTrue(check("fqdn_flags absent", offer()).ok) + + +class Substitution(unittest.TestCase): + + def test_references_and_variables_are_found_apart(self): + text = "%(net)s $offer.address $config.header.content-type." + self.assertEqual(subst.references(text), + [("offer", "address"), + ("config", "header.content-type")]) + self.assertEqual(subst.variables(text), set(["net"])) + + def test_a_reference_resolves_against_an_earlier_reply(self): + ctx = subst.Context() + ctx.bind("offer", offer()) + ctx.bind("q", DNS) + self.assertEqual(subst.resolve("ip=$offer.address", ctx), "ip=10.0.0.101") + self.assertEqual(subst.resolve("$q.data", ctx), "10.0.0.11") + self.assertTrue(check("$offer.server_id == 10.0.0.1", None, ctx=ctx).ok) + + def test_an_unbound_name_or_missing_field_is_a_config_error(self): + ctx = subst.Context() + ctx.bind("offer", offer()) + self.assertRaisesRegex(ConfigError, "nothing has bound", + subst.resolve, "$ack.address", ctx) + self.assertRaisesRegex(ConfigError, "no option 67", + subst.resolve, "$offer.opt67", ctx) + + def test_format_value(self): + self.assertEqual(subst.format_value(True), "yes") + self.assertEqual(subst.format_value(b"k"), "k") + self.assertEqual(subst.format_value(["a", 1]), "a, 1") + self.assertEqual(subst.format_value({"b": 1, "a": 2}), "a=2, b=1") + + +if __name__ == "__main__": + unittest.main() diff --git a/xCAT-test/blackboxtest/tests/test_boundaries.py b/xCAT-test/blackboxtest/tests/test_boundaries.py new file mode 100644 index 000000000..eb7b0c26c --- /dev/null +++ b/xCAT-test/blackboxtest/tests/test_boundaries.py @@ -0,0 +1,129 @@ +"""Rules about this tool's own code, which are why its results mean anything. + + * It never asks xCAT what to expect: no database, no xCAT command, no xCAT + path. A suite that asked xCAT would pass on a cluster that cannot boot. + * It never knows which DHCP server answers, so ISC and Kea get one test. + * Only `proc` starts a process, and never through a shell, so nothing a + .conf supplies becomes a command. + * Only `dhcp` imports scapy, lazily, so `validate` runs without it. +""" + +import ast +import re +import unittest + +import context + +from blackboxtest_lib import steps + +SPAWNING_MODULES = frozenset(["subprocess", "pty", "multiprocessing"]) +SPAWNING_CALLS = frozenset(["system", "popen", "spawn", "spawnl", "spawnv", + "execv", "execve", "execvp", "fork", "forkpty"]) +DATABASE_MODULES = frozenset(["sqlite3", "psycopg2", "psycopg", "pymysql", + "MySQLdb", "mysql", "sqlalchemy", "dbm", "shelve"]) +STDLIB = frozenset([ + "argparse", "binascii", "configparser", "gzip", "hashlib", "ipaddress", + "os", "random", "re", "select", "shutil", "socket", "ssl", "subprocess", + "sys", "tempfile", "threading", "time"]) + +#: xCAT commands. `xcatd` is absent: it is the daemon under test. +XCAT = re.compile( + r"\b(lsdef|mkdef|chdef|rmdef|nodels|nodech|tabdump|chtab|gettab|nodeset|" + r"makedns|makedhcp|makehosts|rinstall|copycds|genimage|xcatconfig)\b") + +#: xCAT's own files. A scenario may request one to assert it is refused. +XCAT_FILES = re.compile(r"/opt/xcat/s?bin|/etc/xcat/|site\.tab") + +#: DHCP server implementations. +SERVERS = re.compile(r"\bkea\b|kea-dhcp4|dnsmasq|dhcpd\.conf", re.IGNORECASE) + + +def trees(): + for name, path in context.modules(): + with open(path) as handle: + yield name, ast.parse(handle.read(), filename=path) + + +def imports(tree): + names = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + names.update(alias.name.split(".")[0] for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.level == 0: + names.add(node.module.split(".")[0]) + return names + + +def code_strings(tree): + """String literals that are not docstrings: prose may name what the code + must not use.""" + docs = set(id(node.body[0].value) for node in ast.walk(tree) + if isinstance(node, (ast.Module, ast.ClassDef, ast.FunctionDef)) + and node.body and isinstance(node.body[0], ast.Expr) + and isinstance(node.body[0].value, ast.Constant)) + return [node.value for node in ast.walk(tree) + if isinstance(node, ast.Constant) and isinstance(node.value, str) + and id(node) not in docs] + + +class Boundaries(unittest.TestCase): + + def test_only_proc_starts_a_process_and_never_through_a_shell(self): + for name, tree in trees(): + calls = set(getattr(node.func, "attr", getattr(node.func, "id", "")) + for node in ast.walk(tree) if isinstance(node, ast.Call)) + if name != "proc": + self.assertFalse(imports(tree) & SPAWNING_MODULES, name) + self.assertFalse(calls & SPAWNING_CALLS, name) + for node in ast.walk(tree): + if isinstance(node, ast.keyword) and node.arg == "shell": + self.assertFalse(getattr(node.value, "value", True), name) + + def test_only_the_standard_library_and_scapy_in_dhcp(self): + for name, tree in trees(): + outside = imports(tree) - STDLIB - set(["blackboxtest_lib"]) + if name == "dhcp": + outside -= set(["scapy"]) + self.assertEqual(outside, set(), name) + self.assertFalse(imports(tree) & DATABASE_MODULES, name) + + def test_scapy_is_imported_inside_a_function(self): + for name, tree in trees(): + for node in tree.body: + if isinstance(node, (ast.Import, ast.ImportFrom)): + self.assertNotIn("scapy", imports(ast.Module(body=[node])), + name) + + def test_no_module_names_an_xcat_command_or_a_dhcp_server(self): + for name, tree in trees(): + for text in code_strings(tree): + self.assertIsNone(XCAT.search(text), "%s: %r" % (name, text)) + self.assertIsNone(XCAT_FILES.search(text), "%s: %r" % (name, text)) + self.assertIsNone(SERVERS.search(text), "%s: %r" % (name, text)) + + def test_the_shipped_scenarios_name_neither(self): + # Comments and descriptions are prose, and may explain how two + # servers differ. What decides a verdict may not know which answers. + for path in context.conf_files(): + with open(path) as handle: + text = "\n".join( + line for line in handle.read().splitlines() + if not re.match(r"\s*(#|description\s*=)", line)) + self.assertIsNone(XCAT.search(text), path) + self.assertIsNone(SERVERS.search(text), path) + + def test_no_step_type_runs_a_program_of_the_confs_choosing(self): + for name, kind in steps.TYPES.items(): + for key in ("exec", "shell", "program", "script", "command_line"): + self.assertNotIn(key, kind.keys, name) + + def test_the_offline_commands_never_import_the_runner(self): + cli = dict(trees())["cli"] + top = set(alias.name for node in cli.body + if isinstance(node, (ast.Import, ast.ImportFrom)) + for alias in node.names) + self.assertNotIn("runner", top) + + +if __name__ == "__main__": + unittest.main() diff --git a/xCAT-test/provtest/tests/test_parsers.py b/xCAT-test/blackboxtest/tests/test_clients.py similarity index 97% rename from xCAT-test/provtest/tests/test_parsers.py rename to xCAT-test/blackboxtest/tests/test_clients.py index 6f7e91c66..eded5e2a8 100644 --- a/xCAT-test/provtest/tests/test_parsers.py +++ b/xCAT-test/blackboxtest/tests/test_clients.py @@ -1,4 +1,4 @@ -"""Decoding what dig, curl and tftp actually printed. +"""Decoding what dig, curl and tftp printed, and the process runner. The output below is captured, not invented: these are the exact shapes the three clients produce, including the ones that matter most -- an NXDOMAIN whose @@ -14,9 +14,9 @@ import shutil import tempfile import unittest -import context # noqa: F401 (sys.path) +import context # noqa: F401 -from provtest_lib import dnsc, httpc, proc, tftpc +from blackboxtest_lib import dnsc, httpc, proc, tftpc ANSWER = """ @@ -121,7 +121,7 @@ class DnsCommandTests(unittest.TestCase): class HttpDecodingTests(unittest.TestCase): def setUp(self): - self.dir = tempfile.mkdtemp(prefix="provtest-http-test-") + self.dir = tempfile.mkdtemp(prefix="blackboxtest-http-") self.addCleanup(shutil.rmtree, self.dir, True) def files(self, body=b"", headers=""): @@ -235,7 +235,7 @@ class HttpUrlTests(unittest.TestCase): class TftpDecodingTests(unittest.TestCase): def setUp(self): - self.dir = tempfile.mkdtemp(prefix="provtest-tftp-test-") + self.dir = tempfile.mkdtemp(prefix="blackboxtest-tftp-") self.addCleanup(shutil.rmtree, self.dir, True) def decode(self, content=b"", out="", err="", rc=0, timed_out=False): @@ -302,7 +302,7 @@ class TftpCommandTests(unittest.TestCase): class ProcessTests(unittest.TestCase): def test_a_missing_program_is_an_unsupported_host_not_a_failure(self): - from provtest_lib.errors import UnsupportedError + from blackboxtest_lib.errors import UnsupportedError self.assertRaises(UnsupportedError, proc.require, "nosuchclientprogram", "nothing") diff --git a/xCAT-test/blackboxtest/tests/test_config.py b/xCAT-test/blackboxtest/tests/test_config.py new file mode 100644 index 000000000..82a92238d --- /dev/null +++ b/xCAT-test/blackboxtest/tests/test_config.py @@ -0,0 +1,135 @@ +"""Loading .conf files: sections, defaults, variables, and what is refused.""" + +import unittest + +import context + +from blackboxtest_lib import config +from blackboxtest_lib.errors import ConfigError + +MIXED = """ +[vars] +server = 10.0.0.1 + +[defaults] +server = %(server)s +bind = 10.0.0.11 +timeout = 3 +client_arch = 0x0007 + +[scenario boot] +description = a DHCP step and a TFTP step share a file +interface = eth9 + +[step discover] +type = discover +assert = + msgtype == OFFER + +[step fetch] +type = tftp +path = $discover.bootfile +retries = 5 +assert = + size > 0 +""" + + +class Loading(unittest.TestCase): + + def load(self, text, overrides=None, raw=False): + return config.load([context.write_conf(self, text)], overrides, raw) + + def test_sections_become_scenarios_and_steps(self): + scenario, = self.load(MIXED) + self.assertEqual(scenario.name, "boot") + self.assertEqual(scenario.interface, "eth9") + self.assertEqual([s.name for s in scenario.steps], ["discover", "fetch"]) + self.assertEqual(scenario.steps[0].assertions[0].render(), + "msgtype == OFFER") + + def test_a_default_reaches_only_the_steps_that_take_it(self): + discover, fetch = self.load(MIXED)[0].steps + self.assertEqual(fetch.param("server"), "10.0.0.1") + self.assertEqual(fetch.param("bind"), "10.0.0.11") + self.assertNotIn("server", discover.params) + self.assertEqual(discover.param("client_arch"), "0x0007") + self.assertNotIn("client_arch", fetch.params) + + def test_timeouts_and_retries_default_per_family(self): + discover, fetch = self.load(MIXED)[0].steps + self.assertEqual((discover.timeout, discover.retries), (3.0, 3)) + self.assertEqual((fetch.timeout, fetch.retries), (3.0, 5)) + scenario, = self.load("[scenario s]\n[step n]\ntype = noop\n") + self.assertEqual((scenario.steps[0].timeout, scenario.steps[0].retries), + (5.0, 2)) + + def test_a_command_line_value_beats_vars(self): + fetch = self.load(MIXED, {"server": "10.0.0.2"})[0].steps[1] + self.assertEqual(fetch.param("server"), "10.0.0.2") + + def test_a_variable_never_becomes_a_step_key(self): + fetch = self.load(MIXED, {"path": "/etc/passwd"})[0].steps[1] + self.assertEqual(fetch.param("path"), "$discover.bootfile") + + def test_an_undefined_variable_names_the_flag_that_fixes_it(self): + with self.assertRaisesRegex(ConfigError, r"pass --set net="): + self.load("[scenario s]\n[step n]\ntype = noop\n" + "assert =\n $n.x in %(net)s\n") + + def test_raw_loading_leaves_variables_in_place(self): + scenario, = self.load("[scenario s]\n[step t]\ntype = tftp\n" + "server = %(server)s\npath = x\n", raw=True) + self.assertEqual(scenario.steps[0].param("server"), "%(server)s") + + def test_a_semicolon_starts_an_inline_comment(self): + scenario, = self.load("[scenario s]\n[step t]\ntype = tftp\n" + "server = 10.0.0.1 ; the MN\npath = x\n") + self.assertEqual(scenario.steps[0].param("server"), "10.0.0.1") + + def test_what_is_refused(self): + cases = { + "unknown key": "[scenario s]\n[step t]\ntype = tftp\nbogus = 1\n", + "DHCP key on a tftp step": "[scenario s]\n[step t]\ntype = tftp\n" + "client_arch = 7\n", + "no type": "[scenario s]\n[step t]\npath = x\n", + "unknown type": "[scenario s]\n[step t]\ntype = gopher\n", + "step before scenario": "[step t]\ntype = noop\n", + "scenario without steps": "[scenario s]\n", + "unknown section": "[scenario s]\n[step t]\ntype = noop\n[bogus]\n", + "type as a default": "[defaults]\ntype = noop\n[scenario s]\n" + "[step t]\ntype = noop\n", + "a repeated section": "[scenario s]\n[step t]\ntype = noop\n" + "[step t]\ntype = noop\n", + "bad timeout": "[scenario s]\n[step t]\ntype = noop\ntimeout = soon\n", + "bad operator": "[scenario s]\n[step t]\ntype = noop\n" + "assert =\n x ~= y\n", + "bad regex": "[scenario s]\n[step t]\ntype = noop\n" + "assert =\n x matches (\n", + "nothing": "# empty\n", + } + for why, text in cases.items(): + with self.assertRaises(ConfigError, msg=why): + self.load(text) + + def test_a_scenario_name_may_not_be_reused_across_files(self): + text = "[scenario s]\n[step t]\ntype = noop\n" + paths = [context.write_conf(self, text, "a.conf"), + context.write_conf(self, text, "b.conf")] + self.assertRaisesRegex(ConfigError, "defined in both", + config.load, paths) + + def test_a_missing_file_names_itself(self): + self.assertRaisesRegex(ConfigError, "/no/such.conf: no such file", + config.load, ["/no/such.conf"]) + + +class ShippedFiles(unittest.TestCase): + + def test_every_shipped_scenario_has_a_description(self): + for scenario in config.load(context.conf_files(), raw=True): + self.assertTrue(scenario.description, scenario.name) + + +if __name__ == "__main__": + unittest.main() diff --git a/xCAT-test/blackboxtest/tests/test_dhcp.py b/xCAT-test/blackboxtest/tests/test_dhcp.py new file mode 100644 index 000000000..d7907509f --- /dev/null +++ b/xCAT-test/blackboxtest/tests/test_dhcp.py @@ -0,0 +1,307 @@ +"""The DHCP client: option coding, decoding, frames and the step loop.""" + +import unittest + +import context + +from blackboxtest_lib import config, dhcp, dhcpopts as opt, subst +from blackboxtest_lib.errors import ConfigError + +try: + import scapy.all as scapy +except ImportError: # pragma: no cover + scapy = None + + +class Options(unittest.TestCase): + + def test_names_numbers_and_prefixes(self): + for spec in (54, "54", "0x36", "server_id", "server-id", + "option:server_id"): + self.assertEqual(opt.option_code(spec), 54, spec) + self.assertRaises(ConfigError, opt.option_code, "not_an_option") + self.assertRaises(ConfigError, opt.option_code, "256") + + def test_message_types(self): + self.assertEqual(opt.msgtype_code("offer"), 2) + self.assertEqual(opt.msgtype_name(2), "OFFER") + self.assertEqual(opt.msgtype_name(99), "TYPE-99") + + def test_value_encoding_by_option(self): + self.assertEqual(opt.encode_value(54, "10.0.0.1"), b"\x0a\x00\x00\x01") + self.assertEqual(opt.encode_value(93, "0x000b"), b"\x00\x0b") + self.assertEqual(opt.encode_value(55, "1, 3, server_id"), b"\x01\x03\x36") + self.assertEqual(opt.encode_value(60, "PXEClient"), b"PXEClient") + self.assertEqual(opt.encode_value(43, "01:02"), b"\x01\x02") + + def test_value_decoding_by_option(self): + self.assertEqual(opt.decode_value(1, b"\xff\xff\xff\x00"), "255.255.255.0") + self.assertEqual(opt.decode_value(51, b"\x00\x00\xa8\xc0"), 43200) + self.assertEqual(opt.decode_value(55, b"\x01\x03\x06"), [1, 3, 6]) + # An address-list option is a list even with one address in it. + self.assertEqual(opt.decode_value(3, b"\x0a\x63\x00\x01"), ["10.99.0.1"]) + self.assertEqual(opt.decode_value(175, b"\xab\xcd"), "ab:cd") + + def test_the_option_area(self): + blob = opt.pack_options([(53, b"\x02"), (54, b"\x0a\x00\x00\x01")]) + self.assertTrue(blob.endswith(b"\xff")) + self.assertEqual(opt.unpack_options(blob)[54], b"\x0a\x00\x00\x01") + # Magic cookie, padding, no END; and RFC 3396 split options. + self.assertEqual(opt.unpack_options(opt.MAGIC + b"\x00" + bytes([53, 1, 5])), + {53: b"\x05"}) + split = bytes([60, 2]) + b"PX" + bytes([60, 7]) + b"EClient" + self.assertEqual(opt.unpack_options(split)[60], b"PXEClient") + + def test_name_lists_follow_pointers_and_never_loop(self): + self.assertEqual(opt.decode_value(119, b"\x03foo\x03com\x00\x03bar\xc0\x04"), + ["foo.com", "bar.com"]) + self.assertEqual(opt.decode_value(119, b"\x03foo\xc0\x00"), + "03:66:6f:6f:c0:00") + self.assertEqual(opt.decode_value(119, b"\x09foo"), "09:66:6f:6f") + + +class ClientFqdn(unittest.TestCase): + """Option 81, RFC 4702.""" + + def test_flags_come_from_a_prefix_of_flag_letters_only(self): + self.assertEqual(opt.encode_fqdn("node01"), b"\x00\x00\x00node01") + self.assertEqual(opt.encode_fqdn("SO:node01")[0], 0x03) + self.assertEqual(opt.decode_fqdn(opt.encode_fqdn("host:1")), (0, "host:1")) + self.assertEqual(opt.decode_fqdn(opt.encode_fqdn("SX:n")), (0, "SX:n")) + self.assertRaises(ConfigError, opt.fqdn_flag_bits, "SX") + + def test_the_e_flag_sends_wire_format(self): + raw = opt.encode_fqdn("SE:node01.cluster") + self.assertEqual(raw[3:], b"\x06node01\x07cluster\x00") + self.assertEqual(opt.decode_fqdn(raw), (0x05, "node01.cluster")) + + def test_decoding_gives_the_name_and_a_short_option_stays_hex(self): + self.assertEqual(opt.decode_value(81, b"\x03\x00\x00n1."), "n1.") + self.assertEqual(opt.decode_value(81, b"\x03\x00"), "03:00") + self.assertEqual(opt.fqdn_flag_letters(0), "-") + + +class _FakeFrame(object): + """Enough of a scapy frame for `decode`.""" + + def __init__(self, option_area): + self.payload = option_area + self.xid = 0x1234 + self.yiaddr, self.siaddr = "10.0.0.2", "10.0.0.1" + self.ciaddr = self.giaddr = "0.0.0.0" + self.chaddr = b"\x02\x00\x11\x22\x33\x44" + b"\x00" * 10 + self.file = b"pxelinux.0" + b"\x00" * 118 + self.sname = b"\x00" * 64 + self.secs = self.flags = 0 + + def __getitem__(self, _layer): + return self + + def haslayer(self, _layer): + return False + + +class Decoding(unittest.TestCase): + + def test_a_dhcp_reply_is_named_by_option_53(self): + reply = opt.decode(_FakeFrame(opt.pack_options([(53, b"\x02")]))) + self.assertEqual(reply.fields["msgtype"], "OFFER") + self.assertEqual(reply.fields["chaddr"], "02:00:11:22:33:44") + + def test_a_reply_with_no_option_53_is_plain_bootp(self): + reply = opt.decode(_FakeFrame(opt.MAGIC + b"\xff")) + self.assertEqual(reply.fields["msgtype"], "BOOTREPLY") + self.assertEqual(reply.bootfile(), "pxelinux.0") + + def test_known_fields(self): + for name in ("address", "bootfile", "fqdn_flags", "yiaddr", "opt67", + "server_id", "option:3", "12"): + self.assertTrue(opt.known_field(name), name) + self.assertFalse(opt.known_field("sha256")) + + +class Transitions(unittest.TestCase): + + def test_the_table(self): + self.assertEqual(dhcp.STEP_TYPES, set(["discover", "bootrequest", + "request", "renew", "rebind", + "release"])) + self.assertIsNone(dhcp.transition(dhcp.BOUND, "discover")) + self.assertTrue(dhcp.transition(dhcp.BOUND, "renew").unicast) + self.assertFalse(dhcp.transition(dhcp.BOUND, "rebind").unicast) + self.assertEqual(dhcp.transition(dhcp.INIT, "bootrequest").next_state, + dhcp.INIT) + + def test_advance(self): + request = dhcp.transition(dhcp.SELECTING, "request") + self.assertEqual(dhcp.advance(dhcp.SELECTING, request, False), + dhcp.SELECTING) + self.assertEqual(dhcp.advance(dhcp.SELECTING, request, True, "NAK"), + dhcp.INIT) + self.assertEqual(dhcp.advance(dhcp.SELECTING, request, True, "ACK"), + dhcp.BOUND) + release = dhcp.transition(dhcp.BOUND, "release") + self.assertEqual(dhcp.advance(dhcp.BOUND, release, False), dhcp.INIT) + + +def reply(msgtype, server="10.0.0.1", yiaddr="10.0.0.101", xid=0): + fields = {"msgtype": msgtype, "xid": xid, "yiaddr": yiaddr, + "siaddr": server, "ciaddr": "0.0.0.0", "giaddr": "0.0.0.0", + "chaddr": "", "file": "", "sname": "", "secs": 0, "flags": 0, + "src_ip": server, "src_mac": ""} + return opt.DhcpReply(fields, {54: server, 51: 600}) + + +class FakeWire(object): + """Answers each send with the next list of replies.""" + + scapy = None + + def __init__(self, *answers): + self.answers = list(answers) + self.sent = 0 + + def send(self, frame): + self.sent += 1 + + def collect(self, session, deadline): + return self.answers.pop(0) if self.answers else [] + + def close(self): + pass + + +class StepLoop(unittest.TestCase): + + def setUp(self): + original = dhcp.build_frame + dhcp.build_frame = lambda *args: None + self.addCleanup(setattr, dhcp, "build_frame", original) + + def step(self, text): + scenario, = config.load([context.write_conf( + self, "[scenario s]\n" + text)]) + return scenario.steps + + def test_a_discover_binds_the_offer_and_counts_servers_once(self): + step, = self.step("[step d]\ntype = discover\n") + first = reply("OFFER") + session = dhcp.Session(FakeWire([first, reply("OFFER"), + reply("OFFER", "10.0.0.2")])) + ctx = subst.Context() + got, extras, sent = dhcp.run_step(step, session, ctx, 0.1, 3) + self.assertIs(got, first) + self.assertIs(ctx.get("offer"), first) + self.assertEqual(extras["offers"], 2) + self.assertEqual(session.state, dhcp.SELECTING) + self.assertEqual(session.wire.sent, 1) + self.assertTrue(sent.startswith("DISCOVER mac=02:")) + + def test_silence_retransmits_and_leaves_the_state(self): + step, = self.step("[step d]\ntype = discover\n") + session = dhcp.Session(FakeWire()) + got, extras, _ = dhcp.run_step(step, session, subst.Context(), 0.1, 3) + self.assertIsNone(got) + self.assertEqual(session.wire.sent, 3) + self.assertEqual(session.state, dhcp.INIT) + + def test_a_nak_is_reported_as_a_nak_and_does_not_bind_the_lease(self): + _, step = self.step("[step d]\ntype = discover\n[step r]\n" + "type = request\nrequested_address = 10.0.0.9\n") + session = dhcp.Session(FakeWire([reply("NAK")])) + session.state = dhcp.SELECTING + ctx = subst.Context() + got, extras, _ = dhcp.run_step(step, session, ctx, 0.1, 1) + self.assertEqual(got.fields["msgtype"], "NAK") + self.assertRaises(ConfigError, ctx.get, "lease") + self.assertEqual(session.state, dhcp.INIT) + + def test_an_ack_binds_the_lease_and_arms_arp(self): + _, step = self.step("[step d]\ntype = discover\n[step r]\n" + "type = request\n") + session = dhcp.Session(FakeWire([reply("ACK")])) + session.state = dhcp.SELECTING + ctx = subst.Context() + dhcp.run_step(step, session, ctx, 0.1, 1) + self.assertEqual(ctx.get("lease").fields["yiaddr"], "10.0.0.101") + self.assertEqual(session.leases, set(["10.0.0.101"])) + self.assertEqual(session.state, dhcp.BOUND) + + def test_a_step_mac_replaces_the_random_one(self): + step, = self.step("[step d]\ntype = discover\nmac = 02-00-00-00-00-AA\n") + session = dhcp.Session(FakeWire()) + dhcp.run_step(step, session, subst.Context(), 0.1, 1) + self.assertEqual(session.mac, "02:00:00:00:00:aa") + + +@unittest.skipUnless(scapy, "scapy is not installed") +class Frames(unittest.TestCase): + """A real frame, serialised and read back. Needs scapy, not root.""" + + CONF = """ +[scenario s] +[step d] +type = discover +mac = 02:00:de:ad:be:ef +vendor_class = PXEClient:Arch:00011:UNDI:003000 +client_arch = 0x000b +user_class = xNBA +user_class_form = rfc3004 +request_options = 1, 3, 6, 43, 54, 60, 66, 67 +max_message_size = 1260 +hostname = probe +fqdn = N:probe.cluster.local +[step renew] +type = renew +ciaddr = 10.0.0.101 +server_id = 10.0.0.1 +""" + + def setUp(self): + scenario, = config.load([context.write_conf(self, self.CONF)]) + self.discover, self.renew = scenario.steps + self.session = dhcp.Session(None) + self.session.mac = "02:00:de:ad:be:ef" + self.session.xid = 0x1234 + + def frame(self, step, state=dhcp.INIT): + return dhcp.build_frame(step, self.session, + dhcp.transition(state, step.type), + subst.Context(), scapy) + + def test_a_discover_is_broadcast_and_carries_every_option_shape(self): + frame = self.frame(self.discover) + self.assertEqual(frame["Ether"].dst, "ff:ff:ff:ff:ff:ff") + self.assertEqual((frame["IP"].src, frame["IP"].dst), + ("0.0.0.0", "255.255.255.255")) + self.assertEqual((frame["UDP"].sport, frame["UDP"].dport), (68, 67)) + self.assertEqual(int(frame["BOOTP"].flags), 0x8000) + self.assertGreaterEqual(len(bytes(frame["BOOTP"])), 300) + + decoded = opt.decode(scapy.Ether(bytes(frame))) + self.assertEqual(decoded.fields["msgtype"], "DISCOVER") + self.assertEqual(decoded.fields["xid"], 0x1234) + self.assertEqual(decoded.options[60], "PXEClient:Arch:00011:UNDI:003000") + self.assertEqual(decoded.options[93], 11) + self.assertEqual(decoded.options[57], 1260) + self.assertEqual(decoded.options[55], [1, 3, 6, 43, 54, 60, 66, 67]) + self.assertEqual(decoded.options[61], "01:02:00:de:ad:be:ef") + self.assertEqual(decoded.options[81], "probe.cluster.local") + self.assertEqual(decoded.fqdn_flags(), "N") + raw = opt.unpack_options(bytes(frame["BOOTP"].payload)) + self.assertEqual(raw[77], b"\x04xNBA") + + def test_a_raw_user_class_is_not_prefixed(self): + self.discover.params["user_class_form"] = "raw" + raw = opt.unpack_options(bytes(self.frame(self.discover)["BOOTP"].payload)) + self.assertEqual(raw[77], b"xNBA") + + def test_a_renew_is_unicast_to_the_server(self): + frame = self.frame(self.renew, dhcp.BOUND) + self.assertEqual((frame["IP"].src, frame["IP"].dst), + ("10.0.0.101", "10.0.0.1")) + self.assertEqual(int(frame["BOOTP"].flags), 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/xCAT-test/blackboxtest/tests/test_report.py b/xCAT-test/blackboxtest/tests/test_report.py new file mode 100644 index 000000000..e35903a16 --- /dev/null +++ b/xCAT-test/blackboxtest/tests/test_report.py @@ -0,0 +1,80 @@ +"""TAP output, exit codes, and the address helpers the fixtures call.""" + +import io +import unittest + +import context # noqa: F401 + +from blackboxtest_lib import netutil, report +from blackboxtest_lib.errors import ConfigError +from blackboxtest_lib.model import Reply + + +def tap(*records): + stream = io.StringIO() + reporter = report.TapReporter(stream) + reporter.start() + for record in records: + reporter.add(record) + reporter.finish() + return stream.getvalue(), reporter.exit_code() + + +class Tap(unittest.TestCase): + + def test_a_pass_is_one_line_and_the_plan_counts_it(self): + text, code = tap(report.Record("s", "t", "size > 0", True)) + self.assertEqual(text, "TAP version 13\nok 1 - s/t: size > 0\n1..1\n" + "# passed 1, failed 0, skipped 0\n") + self.assertEqual(code, 0) + + def test_a_failure_explains_itself_on_single_lines(self): + reply = Reply("dns", {"status": "NXDOMAIN"}, ok=False) + text, code = tap(report.Record( + "s", "ptr", "data contains n1", False, expected="n1", actual=None, + detail="data is not\npresent", reply=reply, sent="dig @x y PTR", + attempt=2, attempts=2)) + self.assertEqual(code, 1) + self.assertIn("not ok 1 - s/ptr: data contains n1\n ---\n", text) + for line in (" expected: 'n1'", " received: None", + " detail: data is not present", " sent: dig @x y PTR", + " attempt: 2 of 2", " reply: dns failed status=NXDOMAIN"): + self.assertIn(line + "\n", text) + + def test_a_skip_is_ok_and_neither_passes_nor_fails(self): + text, code = tap(report.Record("s", "t", "x", True, skip_reason="no nic")) + self.assertIn("ok 1 - s/t: x # SKIP no nic\n", text) + self.assertIn("# passed 0, failed 0, skipped 1", text) + self.assertEqual(code, 0) + + +class Addresses(unittest.TestCase): + + def test_the_names_a_loader_asks_for(self): + self.assertEqual(netutil.hex_ip("10.99.1.11"), "0A63010B") + self.assertEqual(netutil.hex_net("10.99.1.0", 24), "0A6301") + self.assertEqual(netutil.hex_net("10.99.1.0", "20"), "0A630") + self.assertRaises(ValueError, netutil.hex_net, "10.99.1.0", 33) + self.assertRaises(ValueError, netutil.hex_ip, "10.99.1") + self.assertEqual(netutil.reverse_name("10.99.1.11"), + "11.1.99.10.in-addr.arpa") + + def test_mac_forms(self): + for text in ("02:AA:bb:0c:0d:0e", "02-aa-bb-0c-0d-0e", "02aabb0c0d0e", + "2:aa:bb:c:d:e"): + self.assertEqual(netutil.normalise_mac(text), "02:aa:bb:0c:0d:0e") + self.assertEqual(netutil.dashed_mac("02:aa:bb:0c:0d:0e"), + "02-aa-bb-0c-0d-0e") + for bad in ("02:aa:bb", "02:aa:bb:0c:0d:0e:0f", "zz:aa:bb:0c:0d:0e"): + self.assertRaises(ConfigError, netutil.normalise_mac, bad) + self.assertEqual(netutil.random_mac()[:3], "02:") + + def test_membership(self): + self.assertTrue(netutil.ip_in("10.0.0.5", "10.0.0.0/24")) + self.assertTrue(netutil.ip_in_range("10.0.0.5", "10.0.0.9-10.0.0.1")) + self.assertFalse(netutil.ip_in("x", "10.0.0.0/24")) + self.assertIsNone(netutil.parse_range("a-b")) + + +if __name__ == "__main__": + unittest.main() diff --git a/xCAT-test/blackboxtest/tests/test_runner.py b/xCAT-test/blackboxtest/tests/test_runner.py new file mode 100644 index 000000000..3970b0b22 --- /dev/null +++ b/xCAT-test/blackboxtest/tests/test_runner.py @@ -0,0 +1,96 @@ +"""The runner, end to end, on steps that need no network.""" + +import io +import unittest + +import context + +from blackboxtest_lib import config, report, runner + + +def run(test, text, **options): + scenarios = config.load([context.write_conf(test, text)]) + stream = io.StringIO() + code = runner.run(scenarios, report.TapReporter(stream), + runner.RunOptions(**options)) + return stream.getvalue(), code + + +class LocalSteps(unittest.TestCase): + + def test_a_chain_resolves_references_between_steps(self): + text, code = run(self, """ +[scenario chain] +description = extract reads what an earlier step returned +[step config] +type = extract +from = linux /tftpboot/xcat/k initrd=/i +pattern = ^linux\\s+/?(?:tftpboot/)?(\\S+) +expect = ok +[step again] +type = extract +from = path=$config.value +pattern = path=(.*) +assert = + value == xcat/k + $config.matched == yes +""") + self.assertEqual(code, 0, text) + self.assertIn("ok 1 - chain/config: expect ok", text) + self.assertIn("ok 2 - chain/again: value == xcat/k", text) + + def test_an_expectation_is_a_test_point_of_its_own(self): + text, code = run(self, """ +[scenario s] +description = nothing matches +[step miss] +type = extract +from = abc +pattern = z +expect = ok +""") + self.assertEqual(code, 1) + self.assertIn("not ok 1 - s/miss: expect ok", text) + self.assertIn("'z' matched nothing", text) + + def test_a_step_that_cannot_run_fails_and_stops_its_scenario(self): + text, code = run(self, """ +[scenario s] +description = a reference to a field the reply lacks +[step a] +type = noop +[step b] +type = extract +from = $a.value +pattern = x +[step c] +type = noop +assert = + $a.x absent +""") + self.assertEqual(code, 1) + self.assertIn("not ok 1 - s/b: step could not be run", text) + self.assertNotIn("s/c", text) + + def test_a_dhcp_scenario_without_an_interface_is_skipped(self): + # dhcp.require() refuses a host without scapy or root before any of + # this runs, so stand it down to reach the per-scenario check. + original = runner.dhcp.require + runner.dhcp.require = lambda: None + self.addCleanup(setattr, runner.dhcp, "require", original) + text, code = run(self, """ +[scenario d] +description = no NIC on this host +interface = nosuchnic0 +[step discover] +type = discover +assert = + msgtype == OFFER +""") + self.assertEqual(code, 0) + self.assertIn("# SKIP no such interface: nosuchnic0", text) + + + +if __name__ == "__main__": + unittest.main() diff --git a/xCAT-test/blackboxtest/tests/test_validate.py b/xCAT-test/blackboxtest/tests/test_validate.py new file mode 100644 index 000000000..2216034f0 --- /dev/null +++ b/xCAT-test/blackboxtest/tests/test_validate.py @@ -0,0 +1,144 @@ +"""What `validate` catches offline: the DHCP state walk, fields, references.""" + +import unittest + +import context + +from blackboxtest_lib import config, validate + + +def problems(test, text): + scenario, = config.load([context.write_conf(test, text)], raw=True) + return validate.scenario_problems(scenario) + + +LEASE = """ +[scenario lease] +description = discover then request +[step discover] +type = discover +[step request] +type = request +requested_address = $offer.address +server_id = $offer.server_id +assert = + msgtype == ACK +[step fetch] +type = tftp +server = $lease.server_id +path = $discover.bootfile +assert = + size > 0 + $request.address == $offer.address +""" + + +class DhcpWalk(unittest.TestCase): + + def test_a_clean_lease_then_a_fetch_has_no_problems(self): + self.assertEqual(problems(self, LEASE), []) + + def test_a_request_without_a_discover_needs_a_requested_address(self): + found = problems(self, "[scenario s]\n[step r]\ntype = request\n") + self.assertTrue(any("needs requested_address=" in p for p in found), found) + + def test_a_renew_before_bound_is_illegal(self): + found = problems(self, "[scenario s]\n[step r]\ntype = renew\n") + self.assertTrue(any("not legal in state INIT" in p for p in found), found) + + def test_an_impossible_expectation_is_rejected(self): + found = problems(self, "[scenario s]\n[step d]\ntype = discover\n" + "expect = ack\n") + self.assertTrue(any("expect=ack is not a reply" in p for p in found)) + found = problems(self, "[scenario s]\n[step d]\ntype = tftp\n" + "server = x\npath = y\nexpect = offer\n") + self.assertTrue(any("unknown expect=offer" in p for p in found)) + + def test_expect_none_binds_nothing_and_leaves_the_state(self): + found = problems(self, "[scenario s]\n[step d]\ntype = discover\n" + "expect = none\n[step r]\ntype = request\n" + "server_id = $offer.server_id\n") + self.assertTrue(any("before anything binds $offer" in p for p in found)) + self.assertTrue(any("needs requested_address=" in p for p in found)) + + def test_a_release_gives_the_lease_up(self): + text = LEASE.replace("[step fetch]", "[step rel]\ntype = release\n" + "[step d2]\ntype = discover\n[step fetch]") + self.assertEqual(problems(self, text), []) + + def test_a_literal_mac_is_checked(self): + found = problems(self, "[scenario s]\n[step d]\ntype = discover\n" + "mac = 02:00:zz:00:00:01\n") + self.assertTrue(any("not a MAC" in p for p in found), found) + + +class Fields(unittest.TestCase): + + def test_a_field_the_step_type_cannot_carry(self): + found = problems(self, "[scenario s]\n[step t]\ntype = tftp\n" + "server = x\npath = y\nassert =\n" + " status == 200\n") + self.assertTrue(any("tftp reply has no field 'status'" in p + for p in found), found) + + def test_a_reference_field_is_checked_against_the_step_it_names(self): + found = problems(self, LEASE.replace("$discover.bootfile", + "$discover.sha256")) + self.assertTrue(any("discover reply has no field .sha256" in p + for p in found), found) + + def test_indexed_and_mapped_targets(self): + text = ("[scenario s]\n[step h]\ntype = http\nurl = http://x/\n" + "assert =\n header.content-type == a\n size > 0\n" + "[step q]\ntype = dns\nserver = x\nname = y\nassert =\n" + " data.0 == 1.2.3.4\n data.x == 1\n") + found = problems(self, text) + self.assertEqual(len(found), 1, found) + self.assertIn("data.x", found[0]) + + def test_dhcp_targets_include_options_by_name_and_number(self): + text = ("[scenario s]\n[step d]\ntype = discover\nassert =\n" + " option:54 present\n router present\n 67 absent\n" + " offers == 1\n nonsense present\n") + found = problems(self, text) + self.assertEqual(len(found), 1, found) + self.assertIn("nonsense", found[0]) + + +class References(unittest.TestCase): + + def test_a_forward_reference_is_rejected(self): + found = problems(self, "[scenario s]\n[step a]\ntype = extract\n" + "from = $b.text\npattern = x\n" + "[step b]\ntype = noop\n") + self.assertTrue(any("before anything binds $b" in p for p in found)) + + def test_an_assertion_may_read_its_own_step(self): + found = problems(self, "[scenario s]\n[step a]\ntype = extract\n" + "from = abc\npattern = (b)\nassert =\n" + " $a.value == b\n") + self.assertEqual(found, []) + + def test_required_keys_and_duplicate_names(self): + found = problems(self, "[scenario s]\n[step a]\ntype = dns\n" + "server = x\n[step h]\ntype = http\n") + self.assertTrue(any("needs name=" in p for p in found)) + self.assertTrue(any("url=, or server= and path=" in p for p in found)) + + def test_required_variables(self): + scenario, = config.load([context.write_conf( + self, "[scenario s]\n[step a]\ntype = tftp\nserver = %(srv)s\n" + "path = x\nassert =\n size > %(min)s\n")], raw=True) + self.assertEqual(validate.required_variables([scenario]), + set(["srv", "min"])) + + +class ShippedFiles(unittest.TestCase): + + def test_every_shipped_scenario_validates(self): + for scenario in config.load(context.conf_files(), raw=True): + self.assertEqual(validate.scenario_problems(scenario), []) + + +if __name__ == "__main__": + unittest.main() diff --git a/xCAT-test/debian/install b/xCAT-test/debian/install index 0f1c9257e..46632a087 100644 --- a/xCAT-test/debian/install +++ b/xCAT-test/debian/install @@ -5,5 +5,4 @@ share/doc/man1/* opt/xcat/share/doc/man1 autotest opt/xcat/share/xcat/tools unit opt/xcat/share/xcat/tools/autotest integration opt/xcat/share/xcat/tools/autotest -dhcptest opt/xcat/share/xcat/tools/autotest -provtest opt/xcat/share/xcat/tools/autotest +blackboxtest opt/xcat/share/xcat/tools/autotest diff --git a/xCAT-test/debian/xcat-test.substvars b/xCAT-test/debian/xcat-test.substvars new file mode 100644 index 000000000..061cba3b0 --- /dev/null +++ b/xCAT-test/debian/xcat-test.substvars @@ -0,0 +1,3 @@ +perl:Depends=perl:any +misc:Depends= +misc:Pre-Depends= diff --git a/xCAT-test/dhcptest/.gitignore b/xCAT-test/dhcptest/.gitignore deleted file mode 100644 index 7a60b85e1..000000000 --- a/xCAT-test/dhcptest/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -__pycache__/ -*.pyc diff --git a/xCAT-test/dhcptest/README.md b/xCAT-test/dhcptest/README.md deleted file mode 100644 index 248ee27d8..000000000 --- a/xCAT-test/dhcptest/README.md +++ /dev/null @@ -1,380 +0,0 @@ -# dhcptest - -A wire-level DHCP client test tool. It builds real DHCP packets, sends them -from a raw Layer 2 socket, decodes what comes back, and asserts on it. - -It is agnostic to what is answering. It never reads a server's configuration, -never runs a server's tools, and never looks at an xCAT database — it exchanges -packets and checks fields. Any expectation that differs between servers is -written by whoever writes the `.conf`, never assumed by the tool. - -Requires Python 3.6+ and Scapy (`python3-scapy`). Nothing else: no pip, no -`setup.py`. It runs straight from a checkout. - -## Why - -xCAT's DHCP behaviour was tested from two directions, neither of which touched -the wire: unit tests asserting on *generated config text*, and integration -tests feeding that config to a real `dhcpd`/`kea-dhcp4` to check it parses. -Both stop at "the server accepted our config". Nothing verified that a client -sending option 93 = `0x000b` actually gets `boot/grub2/grub2.aarch64` back. - -The one existing wire tool, `xCAT-probe/subcmds/detect_dhcpd`, cannot fill that -gap: it sends no option 60, 77 or 93, so it never reaches any architecture -branch; it parses replies by regexing `tcpdump -vvvvvv` output; and it binds a -UDP socket to an existing local address, so it cannot run on an interface that -has no address yet — the normal state of a provisioning NIC. - -## Commands - -``` -dhcptest run [-i IFACE] [options] CONF [CONF...] # execute scenarios -dhcptest validate CONF [CONF...] # check offline -dhcptest list CONF [CONF...] # show what a file does -dhcptest discover -i IFACE [--arch N] [--vendor-class S] [--user-class S] -``` - -`validate` and `list` need no root, no network and no scapy, so they run in a -checkout-only CI job. `run` and `discover` need root, for the raw socket. - -| Flag | Meaning | -| --- | --- | -| `-i, --interface` | interface to bind to; overrides the `.conf` | -| `--set KEY=VALUE` | define `%(KEY)s` (repeatable) | -| `-s, --scenario NAME` | run only this scenario (repeatable) | -| `--mac` | `random` (default), `iface`, or an explicit address | -| `--timeout` / `--retries` | per attempt, and attempts per step | -| `--format` | `tap` (default), `pretty`, `json` | -| `--pcap FILE` | write every frame sent and received | -| `-v` / `-vv` | progress / per-packet trace | - -Exit codes: `0` everything passed, `1` at least one assertion failed, `2` -configuration or usage error, `3` the host cannot run the test at all (no -scapy, not root). A host that cannot test anything **fails** rather than -reporting a run of skipped tests, because a green result that proves nothing is -worse than a red one. - -## Configuration - -INI, read with `configparser`. Sections: an optional `[vars]` and `[defaults]`, -then `[scenario NAME]` followed by the `[step NAME]` sections belonging to it. -Unknown keys are a hard error rather than a silent skip. - -```ini -[defaults] -interface = eth1 -timeout = 2 -retries = 3 - -[scenario full-lease] -description = DISCOVER/OFFER/REQUEST/ACK yields the offered address - -[step discover] -type = discover -request_options = 1, 3, 6, 51, 54 -expect = offer -assert = - msgtype == OFFER - yiaddr in %(net)s - -[step request] -type = request -requested_address = $offer.address -server_id = $offer.server_id -expect = ack -assert = - msgtype == ACK - yiaddr == $offer.address - option:51 present -``` - -### Variables and references - -Two substitutions, resolved at two different times. - -`%(name)s` is **configparser's own `BasicInterpolation`**, not something this -tool implements. Values come from the `[vars]` section and from -`--set name=value`, with `--set` winning, and both are folded into -configparser's defaults so a variable resolves from any section. An undefined -name is an error *before* any packet is sent, naming the flag that fixes it: - - conf/discover-offer.conf [step discover]: %(net)s is not defined; - pass --set net= - -`$step.field` is a field of an earlier **reply**, so it cannot exist until the -run reaches that step — which is exactly why it is not configparser's job. The -step name is any earlier step, or one of the bindings the tool maintains: -`$offer`, `$ack`, `$nak`, `$lease`, `$reply`. - -The two never collide: `BasicInterpolation` gives `$` no meaning of its own, so -`$offer.address` reaches the runtime untouched and no escaping is needed. -(`ExtendedInterpolation` would have claimed `${...}` and then choked on a bare -`$`, which is why it is not used.) - -`validate` and `list` load a file **raw**, leaving `%(name)s` in place, and -report which variables a run would have to supply. - -Fields: `address` (`yiaddr`), `bootfile`, `next_server` (`siaddr`), `mac`, -`server_id`, `lease_time`, `subnet_mask`, `router`, any header field by name, -and `optN` for any option by number. - -### Assertions - -One per line under a single multi-line `assert` key, as `target op value`. -`assert_all` applies to every OFFER collected, not only the selected one. - -| Target | Source | -| --- | --- | -| `msgtype` | option 53, by name: `OFFER`, `ACK`, `NAK` | -| `yiaddr` `siaddr` `ciaddr` `giaddr` `file` `sname` `xid` `chaddr` | BOOTP header | -| `bootfile` | option 67 if the server sent one, else the `file` header | -| `dns_name` | option 81 if the server sent one, else option 12 | -| `fqdn_flags` | option 81's flag letters: `S` `O` `E` `N` | -| `option:` / `option:` / bare `` | a DHCP option | -| `offers` | how many *servers* answered | -| `$step.field` | a field of an earlier reply | - -| Op | Meaning | -| --- | --- | -| `==` `!=` | type-aware: addresses as addresses, numbers as numbers | -| `in` `not-in` | a subnet (`10.0.0.0/24`), an address range (`10.0.0.200-10.0.0.250`) or a comma list | -| `present` `absent` | option presence, no value | -| `matches` `contains` `starts-with` `ends-with` | text | -| `<` `<=` `>` `>=` | numeric | - -"Nothing to fetch" is `bootfile absent`, not a match against an empty string: -a server that names no boot file sends an empty `file` header and no option 67, -which is an absent value, and `matches ^$` can never hold against it. - -The negative operators `!=` and `not-in` hold against an absent target, since a -reply that says nothing on the subject has certainly not said the wrong thing. -So `bootfile != http://.../nodes/node01` passes both for a server that names -some other loader and for one that names none, which is what "must not be sent -its install script" means. Every other operator fails on an absent target, -because there is nothing to compare against. - -Assert on `dns_name`, not on option 12 or option 81, unless one option in -particular is the point. The two servers answer a client that sent option 81 in -different places -- Kea replies in option 81, ISC under `ignore client-updates` -replies in option 12 only -- and the node reads whichever arrived. This is the -same reason `bootfile` exists. - -`fqdn_flags` is the other half of option 81: `S` says the server will update the -A record, `O` that it overrode what the client asked for, `E` that the name is in -wire format, `N` that no update is wanted. It is absent when the reply carries no -option 81 at all. - -Assert on `bootfile`, not on `file`, unless the header itself is the point. -Servers genuinely differ — ISC dhcpd fills the BOOTP header, dnsmasq answers in -option 67 once the client has asked for it — and firmware reads whichever -arrived. `bootfile` is what a client would actually boot. - -A dynamic pool is written as two addresses rather than a CIDR, because it rarely -lines up on a prefix boundary, so `in` accepts `first-last` as well and reads it -inclusively: `yiaddr in 10.0.0.200-10.0.0.250` means what it says. A value -containing a hyphen that is not two addresses is still treated as a list entry. - -`offers` counts servers, not packets. A retransmit reuses its xid, as RFC 2131 -requires, so the same server can be heard twice; a second answer from a server -already heard from does not inflate the count. - -### Step keys - -Control: `type`, `expect`, `mac`, `xid`, `timeout`, `retries`, -`broadcast_flag`, `min_size`, `collect_extra`, `select`, `arp_respond`, -`assert`, `assert_all`, `duration`, `dest_mac`. - -Message: `request_options` (55), `vendor_class` (60), `user_class` (77) with -`user_class_form` = `raw` or `rfc3004`, `client_arch` (93), `client_ndi` (94), -`client_uuid` (97), `client_id` (61), `hostname` (12), `fqdn` (81), -`max_message_size` (57), -`vendor_specific` (43, hex), `ipxe_options` (175, hex), `requested_address` -(50), `server_id` (54), `lease_time` (51), `ciaddr`, `giaddr`, and -`option:` for anything else. - -`fqdn` takes a name, optionally prefixed with the flags the client is setting: -`fqdn = node01` sends none, `fqdn = S:node01` asks the server to do the update, -`fqdn = N:node01` asks for no update, and `E` among the letters sends the name in -RFC 1035 wire format rather than ASCII. The prefix is read as flags only when -every character before the colon is a flag letter, so a name holding a colon is -still a name. - -Step types: `discover`, `request`, `renew`, `rebind`, `release`, `decline`, -`inform`, `bootrequest`, `noop`, `sleep`. `expect` is `offer`, `ack`, `nak`, -`bootreply`, `any`, or `none` — with `none`, silence is the passing result. - -`bootrequest` is plain BOOTP: the same header sent with no option 53 at all, -which is what hardware predating DHCP puts on the wire. Its answer carries no -option 53 either, so `msgtype` reads `BOOTREPLY`. - -## Shipped scenarios - -| File | What it asserts | Needs | -| --- | --- | --- | -| `discover-offer.conf` | one DISCOVER draws exactly one OFFER | `net`, `server` | -| `full-lease.conf` | DISCOVER/OFFER/REQUEST/ACK yields the offered address | `net` | -| `static-vs-dynamic.conf` | a reserved MAC gets its address; an unreserved one gets a pool address | `reserved_mac`, `reserved_ip`, `unreserved_mac`, `pool` | -| `pxe-arch-matrix.conf` | each client architecture (option 93) is offered its own loader | `tftp`, `*_loader` | -| `ipxe-userclass.conf` | stage 1 and stage 2 differ, in both user-class encodings | `user_class`, `stage1_loader` | -| `renew-rebind.conf` | a lease survives RENEW and REBIND | `net` | -| `provision-vs-discovery.conf` | a known machine gets its reservation and its own loader; an unknown one gets a pool address | `node_mac`, `node_ip`, `node_loader`, `pool`, `next_server`, `unknown_mac` | -| `discovery-bootfile.conf` | an unknown machine is handed a loader too, not just an address | `unknown_mac`, `pool`, `discovery_loader` | -| `netboot-methods.conf` | a node is handed the loader its netboot method names, is told its own name whatever it calls itself, and a `*NOIP*` port is not answered | `*_mac`, `*_ip`, `*_loader`, `xnba_node`, `petitboot_conf`, `noip_mac` | -| `hierarchy-dhcpserver.conf` | a subnet whose pool belongs to another server ignores unknown MACs but still points known ones at it | `node_mac`, `node_ip`, `delegate`, `unknown_mac` | -| `discovery-adoption.conf` | a machine discovered out of the pool is served its own address once defined | `adopt_mac`, `adopt_ip`, `pool` | -| `nak-foreign-address.conf` | a REQUEST for an address off this network is refused | `foreign_ip` | -| `next-server-source.conf` | each node's next-server follows its own attributes, not the subnet's | `tftp_mac`, `tftp_ip`, `tftp_server`, and the same three for `xcm_` and `sub_` | -| `multi-mac-node.conf` | a node with two provisioning ports is served a different address on each | `first_mac`, `first_ip`, `second_mac`, `second_ip` | -| `iscsi-root-path.conf` | a diskless node is told its iSCSI target, and an ISAN client in the vendor space | `iscsi_mac`, `iscsi_ip`, `root_path` | -| `loader-absent.conf` | an architecture whose loader is missing is served an address and no boot file | `pool`, `present_loader` | -| `http-port.conf` | an HTTP boot URL names the port the web server actually listens on | `tftp`, `httpport`, `riscv64_loader` | -| `dynamic-range-cidr.conf` | a dynamic range written as a CIDR block serves addresses out of it | `unknown_mac`, `pool` | -| `node-removal.conf` | a withdrawn node stops being offered the address it used to hold | `removed_mac`, `removed_ip`, `pool` | -| `bootp-client.conf` | a client that speaks BOOTP and not DHCP is still given an address | `bootp_mac`, `pool` | - -`provision-vs-discovery.conf` and `discovery-bootfile.conf` are split apart for -a different reason from the rest: one asserts the boot file a *known* machine -is handed, the other the boot file an *unknown* one is handed, and a run that -only has a node defined can use the first without the second. Both hold on any -xCAT-served network — see the spec, S-56, which requires the subnet to answer -an unknown machine with a loader on either backend. - -Splitting on that boundary rather than using `-s` is deliberate: `--set` values -are resolved when a file is loaded, before `-s` selects anything, so an unused -scenario's missing variable would abort a run that never intended to use it. -One file's variables are all required together. - -Every expected address and filename comes from `--set`, so no shipped file -states anything about how a server was configured, or by what. - -```bash -dhcptest run -i eth1 \ - --set tftp=10.0.0.1 \ - --set bios_loader=xcat/xnba.kpxe \ - --set aarch64_loader=boot/grub2/grub2.aarch64 \ - conf/pxe-arch-matrix.conf -``` - -## What it does to the host - -Nothing. The default MAC is a freshly generated locally-administered address -(`02:…`), so NetworkManager and any running `dhclient` never see the replies as -theirs, and the kernel drops the unicasts because neither the destination MAC -nor the IP is local. No address, route or resolver entry is ever created, and -the leased address is never configured. The tool never calls `ip`, `dhclient` -or `nmcli`; reading `/sys/class/net//address` for `--mac iface` is the only -host state it touches. - -Renewals are unicast from the leased address, and a server may ARP for that -address before replying. Rather than configure the address, the tool answers -ARP itself — only for addresses this session was actually granted, only on its -own synthetic MAC, only on this interface. - -## Failure output - -TAP version 13, which `prove` consumes directly: - -``` -not ok 3 - pxe-bios-x86/bios-discover: bootfile == %(bios_loader)s - --- - expected: 'WRONG.0' - received: 'pxelinux.0' - reply: OFFER from 10.99.0.1 (9a:3d:4f:3d:59:1b) xid=0xc45c6376 yiaddr=10.99.0.175 siaddr=10.99.0.1 - options: {1 (subnet_mask)=255.255.255.0, 3 (router)=['10.99.0.1'], 51 (lease_time)=120, 54 (server_id)=10.99.0.1, 67 (bootfile_name)=pxelinux.0} - sent: DISCOVER mac=02:da:16:d0:80:dd client_arch=0x0000 vendor_class=PXEClient:Arch:00000:UNDI:002001 - attempt: 1 of 3 - ... -``` - -A reply of the wrong type is reported as that type rather than as a timeout: a -NAK where an ACK was wanted says so. - -## Running under xcattest - -`xCAT-test/autotest/testcase/dhcptest/cases0` drives all of this from a -management node. Two of its cases are the offline ones below; the rest are wire -cases, and everything they know about xCAT lives in `dhcpfixture.sh` next to -them, never in here. - -The fixture builds a provisioning network out of a veth pair — the server end -carries the management address and is the only interface named in -`site.dhcpinterfaces`, the client end has no address at all, which is the state -a real provisioning NIC is in when a machine boots on it. It then defines a -network with a dynamic range and one node with a MAC, runs `makedhcp` so the -configuration comes from xCAT rather than from hand-editing, points dhcptest at -the client end, and puts everything back from a trap afterwards. That is what -makes a wire test on a single-node management node mean something instead of -passing with nothing to talk to. - -```bash -xcattest -t dhcptest_provision_vs_discovery -xcattest -s "dhcp_wire" # every wire case, on whatever is configured -``` - -A case that cannot run — no root, no scapy, no `makedhcp`, no veth — says so and -passes. Read a pass as coverage only when the log shows the `ok` lines. - -### Both backends - -The wire cases do not choose a backend. Each serves whatever `site.dhcpbackend` -is set to, and the caller runs the whole set once per backend: - -```bash -FIX=/opt/xcat/share/xcat/tools/autotest/testcase/dhcptest/dhcpfixture.sh -for backend in $($FIX backends); do - $FIX backend-setup $backend - xcattest -t $(xcattest -s "dhcp_wire" -l | paste -sd,) - $FIX backend-teardown $backend -done -``` - -`backend-setup` points `site.dhcpbackend` at one backend and stops the other -daemon — two servers on one wire both answer the same DISCOVER — and -`backend-teardown` restores the site table and restarts what was running before. -The CI driver does exactly this in `run_dhcp_wire_test`, as the last phase of -the run: the wire cases carry `dhcp_wire` and deliberately not `ci_test`, so -they run after the ci_test set rather than inside it, and are not run twice. - -Running every case under one backend and then every case under the other, rather -than switching inside each case, reconfigures the daemon once per pass instead -of once per case, and gives each failure a backend name. Which backend a cluster -runs is an implementation default of the management node's distro, so a -behaviour that holds on one and not the other is a node that boots on one -release and hangs on the next. - -## Tests - -```bash -cd xCAT-test/dhcptest -python3 -m unittest discover -s tests # no root, no network -python3 src/dhcptest validate conf/*.conf -``` - -The suite needs no privileges and no server. `tests/test_wire.py` builds real -frames and reads them back, and skips where scapy is absent. -`tests/test_boundaries.py` enforces the two rules this tool is built on: only -`runner.py` may import scapy, and no module may shell out, open a database, or -name a server implementation or xCAT in code. - -## Layout - -``` -src/dhcptest entry point -src/dhcptest_lib/ - cli.py argparse, subcommands, exit codes - config.py configparser -> Scenario/Step, strict validation - model.py Scenario, Step, Assertion, Reply - subst.py $step.field, resolved at run time - assertions.py the assertion mini-language - options.py DHCP option table, raw-byte encode/decode - machine.py RFC 2131 client FSM, offline scenario validation - runner.py packets, socket, retransmit (the only scapy user) - report.py TAP / pretty / json - netutil.py addresses, /sys/class/net -conf/ shipped scenarios -tests/ stdlib unittest -``` - -Options are encoded and decoded from raw bytes rather than through scapy's -option-name table, so a scapy release that renames an option cannot change what -a `.conf` means. diff --git a/xCAT-test/dhcptest/src/dhcptest_lib/__init__.py b/xCAT-test/dhcptest/src/dhcptest_lib/__init__.py deleted file mode 100644 index 5413d72be..000000000 --- a/xCAT-test/dhcptest/src/dhcptest_lib/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -"""dhcptest -- a wire-level DHCP client test tool. - -The package is deliberately layered so that most of it can be unit tested -without root, without a network, and without scapy installed: - - model, netutil, options, subst, assertions, machine, config, report, cli - pure python, standard library only - - packets, transport - the only modules allowed to import scapy - -`tests/test_layering.py` enforces that split. -""" - -__version__ = "0.1.0" diff --git a/xCAT-test/dhcptest/src/dhcptest_lib/assertions.py b/xCAT-test/dhcptest/src/dhcptest_lib/assertions.py deleted file mode 100644 index c26d10d38..000000000 --- a/xCAT-test/dhcptest/src/dhcptest_lib/assertions.py +++ /dev/null @@ -1,282 +0,0 @@ -"""The assertion mini-language: `target op value`. - -One assertion per line, under a single multi-line `assert` key. A single key -holding many lines sidesteps configparser's one-value-per-key rule and keeps -the grammar uniform, so `option:54 == 10.0.0.1` and `yiaddr in 10.0.0.0/24` -parse through exactly the same path. -""" - -import re - -from . import netutil, options, subst -from .errors import ConfigError -from .model import Assertion - -#: Targets that come straight out of the BOOTP header. -HEADER_TARGETS = frozenset([ - "msgtype", "xid", "yiaddr", "siaddr", "ciaddr", "giaddr", "chaddr", - "file", "sname", "secs", "flags", "src_ip", "src_mac", -]) - -#: Targets that describe the step rather than one reply. -META_TARGETS = frozenset(["offers", "attempt"]) - -#: Targets a client works out from more than one place in the message. Assert -#: on these rather than on `file` unless the header itself is the point. -DERIVED_TARGETS = frozenset(["bootfile", "fqdn_flags", "dns_name"]) - -#: Operators that take no value. -NULLARY_OPS = frozenset(["present", "absent"]) - -#: Operators asserting that the reply is *not* something, which a reply that -#: says nothing on the subject satisfies. -NEGATIVE_OPS = frozenset(["!=", "not-in"]) - -OPS = frozenset([ - "==", "!=", "in", "not-in", "present", "absent", - "matches", "contains", "starts-with", "ends-with", - "<", "<=", ">", ">=", -]) - - -class Result(object): - """The outcome of one assertion, carrying enough to explain a failure.""" - - __slots__ = ("assertion", "ok", "expected", "actual", "detail") - - def __init__(self, assertion, ok, expected=None, actual=None, detail=""): - self.assertion = assertion - self.ok = ok - self.expected = expected - self.actual = actual - self.detail = detail - - def __repr__(self): - return "Result(%r, ok=%r, expected=%r, actual=%r)" % ( - self.assertion.render(), self.ok, self.expected, self.actual) - - -def parse(line, source=""): - """Parse one assertion line into an `Assertion`.""" - text = line.strip() - if not text: - raise ConfigError("empty assertion", source) - - parts = text.split(None, 2) - if len(parts) < 2: - raise ConfigError("assertion needs at least a target and an operator: %r" - % (text,), source) - - target, op = parts[0], parts[1] - value = parts[2] if len(parts) > 2 else None - - if op not in OPS: - raise ConfigError( - "unknown operator %r in %r (expected one of: %s)" - % (op, text, ", ".join(sorted(OPS))), source) - - if op in NULLARY_OPS: - if value: - raise ConfigError("operator %r takes no value: %r" % (op, text), source) - value = None - elif value is None: - # `file != ` with nothing after it is a legitimate "must be non-empty". - value = "" - - validate_target(target, source) - if op == "matches": - try: - re.compile(value) - except re.error as exc: - raise ConfigError("bad regex %r: %s" % (value, exc), source) - - return Assertion(target=target, op=op, value=value, source=source, text=text) - - -def parse_block(block, source=""): - """Parse a multi-line `assert = ...` value into a tuple of assertions.""" - result = [] - for number, raw in enumerate(str(block or "").splitlines(), start=1): - line = raw.strip() - if not line or line.startswith("#"): - continue - where = "%s line %d of assert block" % (source, number) if source else "" - result.append(parse(line, where)) - return tuple(result) - - -def validate_target(target, source=""): - """Reject a target that could never resolve, before any packet is sent.""" - if target in HEADER_TARGETS or target in META_TARGETS \ - or target in DERIVED_TARGETS: - return - if target.startswith("$"): - refs = subst.references(target) - if len(refs) != 1: - raise ConfigError( - "assertion target %r must be a single $step.field reference" - % (target,), source) - _, field = refs[0] - if not subst.check_field(field): - raise ConfigError("unknown reply field .%s in %r" % (field, target), - source) - return - # Anything else has to name an option. - options.option_code(target) - - -# --------------------------------------------------------------------------- -# evaluation - - -def evaluate(assertion, reply, context, extras=None): - """Check one assertion against one reply.""" - extras = extras or {} - expected = None - if assertion.value is not None: - expected = subst.resolve(assertion.value, context) - - present, actual = _actual(assertion.target, reply, context, extras) - - if assertion.op == "present": - return Result(assertion, present, expected="present", - actual="present" if present else "absent") - if assertion.op == "absent": - return Result(assertion, not present, expected="absent", - actual="present" if present else "absent") - - if not present: - # A negative comparison is satisfied by a target that is not there at - # all: a reply naming no boot file has certainly not named the node's - # install script. Failing it instead would make "is not X" quietly - # stronger than it reads, and would fail the one reply that is most - # obviously right. `absent` remains the assertion to write when the - # absence itself is the point. - if assertion.op in NEGATIVE_OPS: - return Result(assertion, True, expected=expected, actual=None, - detail="%s is not present in the reply" % (assertion.target,)) - return Result(assertion, False, expected=expected, actual=None, - detail="%s is not present in the reply" % (assertion.target,)) - - ok, detail = _compare(assertion.op, actual, expected) - return Result(assertion, ok, expected=expected, - actual=subst.format_value(actual), detail=detail) - - -def _actual(target, reply, context, extras): - """Return `(present, value)` for one assertion target.""" - if target in META_TARGETS: - if target not in extras: - return False, None - return True, extras[target] - - if target.startswith("$"): - return True, subst.resolve(target, context) - - if target in DERIVED_TARGETS: - if reply is None: - return False, None - value = { - "fqdn_flags": reply.fqdn_flags, - "dns_name": reply.dns_name, - "bootfile": reply.bootfile, - }[target]() - return bool(value), value - - if target in HEADER_TARGETS: - if reply is None: - return False, None - value = getattr(reply, target) - # A zeroed header field counts as absent for present/absent. - empty = value in ("", 0, "0.0.0.0") - return (not empty), value - - if reply is None: - return False, None - code = options.option_code(target) - if code not in reply.options: - return False, None - return True, reply.options[code] - - -def _compare(op, actual, expected): - """Type-aware comparison. Returns `(ok, detail)`.""" - if op in ("==", "!="): - same = _equal(actual, expected) - if op == "==": - return same, "" - return (not same), "" - - if op in ("in", "not-in"): - inside = _member(actual, expected) - if op == "in": - return inside, "" - return (not inside), "" - - text = subst.format_value(actual) - if op == "matches": - return bool(re.search(expected, text)), "" - if op == "contains": - return expected in text, "" - if op == "starts-with": - return text.startswith(expected), "" - if op == "ends-with": - return text.endswith(expected), "" - - left, right = _as_numbers(actual, expected) - if left is None: - return False, "%r and %r are not both numbers" % (actual, expected) - if op == "<": - return left < right, "" - if op == "<=": - return left <= right, "" - if op == ">": - return left > right, "" - if op == ">=": - return left >= right, "" - raise ConfigError("unhandled operator %r" % (op,)) - - -def _equal(actual, expected): - """Compare as addresses, then as numbers, then as message types, then text.""" - left_ip = netutil.parse_ip(subst.format_value(actual)) - right_ip = netutil.parse_ip(expected) - if left_ip is not None and right_ip is not None: - return left_ip == right_ip - - left, right = _as_numbers(actual, expected) - if left is not None: - return left == right - - left_text = subst.format_value(actual) - if _looks_like_msgtype(left_text) and _looks_like_msgtype(expected): - return left_text.upper() == expected.upper() - return left_text == expected - - -def _member(actual, expected): - """`in` means membership: of a CIDR, of a `first-last` range, or of a list.""" - text = subst.format_value(actual) - if "/" in expected: - return netutil.ip_in(text, expected) - if netutil.parse_range(expected) is not None: - return netutil.ip_in_range(text, expected) - candidates = [item.strip() for item in expected.split(",") if item.strip()] - for candidate in candidates: - if _equal(actual, candidate): - return True - return False - - -def _as_numbers(actual, expected): - """Both sides as ints, or `(None, None)` when either is not numeric.""" - try: - left = actual if isinstance(actual, int) else int(str(actual).strip(), 0) - right = expected if isinstance(expected, int) else int(str(expected).strip(), 0) - except (TypeError, ValueError): - return None, None - return left, right - - -def _looks_like_msgtype(text): - return str(text).strip().lower().replace("-", "_") in options.MSGTYPE_BY_NAME diff --git a/xCAT-test/dhcptest/src/dhcptest_lib/cli.py b/xCAT-test/dhcptest/src/dhcptest_lib/cli.py deleted file mode 100644 index 048fa18ca..000000000 --- a/xCAT-test/dhcptest/src/dhcptest_lib/cli.py +++ /dev/null @@ -1,215 +0,0 @@ -"""Command line interface. - - dhcptest run [-i IFACE] [options] CONF [CONF...] - dhcptest validate CONF [CONF...] - dhcptest list CONF [CONF...] - dhcptest discover -i IFACE [options] - -`validate` and `list` need neither root nor scapy, which makes them the safe -entry points for a checkout-only CI job. -""" - -import argparse -import sys - -from . import config, machine, report -from .errors import (DhcpTestError, EXIT_CONFIG, EXIT_FAILED, EXIT_OK) - - -def build_parser(): - parser = argparse.ArgumentParser( - prog="dhcptest", - description="Test a DHCP server from the wire, one declarative " - ".conf file at a time.") - subparsers = parser.add_subparsers(dest="command") - - run = subparsers.add_parser( - "run", help="execute scenarios against a real DHCP server") - _add_conf_arguments(run) - _add_wire_arguments(run) - run.add_argument("--format", default="tap", - choices=sorted(report.REPORTERS), - help="output format (default: tap)") - run.add_argument("--pcap", metavar="FILE", - help="write every frame sent and received to FILE") - run.add_argument("-s", "--scenario", action="append", default=[], - metavar="NAME", help="run only this scenario (repeatable)") - - validate = subparsers.add_parser( - "validate", help="parse and check .conf files without any network I/O") - _add_conf_arguments(validate) - - listing = subparsers.add_parser( - "list", help="list the scenarios and steps in .conf files") - _add_conf_arguments(listing) - - discover = subparsers.add_parser( - "discover", help="send one DISCOVER and print what comes back") - _add_wire_arguments(discover) - discover.add_argument("--arch", metavar="N", - help="client architecture, DHCP option 93 " - "(e.g. 0x0000 for BIOS, 0x000b for aarch64)") - discover.add_argument("--vendor-class", metavar="STRING", - help="DHCP option 60") - discover.add_argument("--user-class", metavar="STRING", - help="DHCP option 77") - discover.add_argument("--request-options", metavar="LIST", - help="DHCP option 55, e.g. '1,3,6,54,66,67'") - discover.add_argument("--format", default="pretty", - choices=sorted(report.REPORTERS)) - discover.add_argument("--pcap", metavar="FILE") - - return parser - - -def _add_conf_arguments(parser): - parser.add_argument("conf", nargs="+", metavar="CONF", - help="a .conf file in INI syntax") - parser.add_argument("--set", action="append", default=[], metavar="KEY=VALUE", - dest="settings", - help="define ${KEY} for substitution (repeatable)") - - -def _add_wire_arguments(parser): - parser.add_argument("-i", "--interface", metavar="IFACE", - help="network interface to bind to") - parser.add_argument("--mac", metavar="ADDR", - help="client MAC: 'random' (default), 'iface', " - "or an explicit address") - parser.add_argument("--timeout", type=float, metavar="SECS", - help="per-attempt reply timeout (default: 2)") - parser.add_argument("--retries", type=int, metavar="N", - help="attempts per step (default: 3)") - parser.add_argument("-v", "--verbose", action="count", default=0, - help="-v for progress, -vv for a per-packet trace") - - -def parse_settings(pairs): - settings = {} - for pair in pairs: - if "=" not in pair: - raise DhcpTestError("--set needs KEY=VALUE, got %r" % (pair,)) - key, _, value = pair.partition("=") - settings[key.strip()] = value - return settings - - -def main(argv=None): - parser = build_parser() - args = parser.parse_args(argv) - if not args.command: - parser.print_help() - return EXIT_CONFIG - - try: - if args.command == "validate": - return command_validate(args) - if args.command == "list": - return command_list(args) - if args.command == "run": - return command_run(args) - if args.command == "discover": - return command_discover(args) - except DhcpTestError as exc: - sys.stderr.write("dhcptest: %s\n" % (exc,)) - return exc.exit_code - parser.print_help() - return EXIT_CONFIG - - -def command_validate(args): - # Raw: a file is checked in a checkout, where no value has been supplied - # for %(name)s yet, so interpolating it here would fail every time. - settings = parse_settings(args.settings) - scenarios, _ = config.load(args.conf, settings, raw=True) - problems = [] - for scenario in scenarios: - problems.extend(machine.validate_scenario(scenario)) - - for problem in problems: - sys.stderr.write("dhcptest: %s\n" % (problem,)) - if problems: - return EXIT_CONFIG - - needed = sorted(machine.required_variables(scenarios) - set(settings)) - sys.stdout.write("%d scenario(s) OK in %d file(s)\n" - % (len(scenarios), len(args.conf))) - if needed: - sys.stdout.write("required variables: %s\n" % (", ".join(needed),)) - return EXIT_OK - - -def command_list(args): - scenarios, _ = config.load(args.conf, parse_settings(args.settings), raw=True) - for scenario in scenarios: - sys.stdout.write("%s\n" % (scenario.name,)) - if scenario.description: - sys.stdout.write(" %s\n" % (scenario.description,)) - sys.stdout.write(" source: %s\n" % (scenario.source,)) - if scenario.interface: - sys.stdout.write(" interface: %s\n" % (scenario.interface,)) - for step in scenario.steps: - sys.stdout.write(" step %-16s type=%-9s expect=%-6s asserts=%d\n" - % (step.name, step.type, step.expect or "-", - len(step.assertions))) - needed = sorted(machine.required_variables([scenario])) - if needed: - sys.stdout.write(" requires: %s\n" % (", ".join(needed),)) - return EXIT_OK - - -def command_run(args): - from . import runner # imports scapy; keep it out of validate - - scenarios, _ = config.load( - args.conf, parse_settings(args.settings), args.interface) - if args.scenario: - wanted = set(args.scenario) - missing = wanted - set(s.name for s in scenarios) - if missing: - raise DhcpTestError( - "no such scenario: %s" % (", ".join(sorted(missing)),)) - scenarios = [s for s in scenarios if s.name in wanted] - - problems = [] - for scenario in scenarios: - problems.extend(machine.validate_scenario(scenario)) - if problems: - for problem in problems: - sys.stderr.write("dhcptest: %s\n" % (problem,)) - return EXIT_CONFIG - - reporter = report.make(args.format) - options = runner.RunOptions( - interface=args.interface, - mac=args.mac, - timeout=args.timeout, - retries=args.retries, - pcap=args.pcap, - verbose=args.verbose, - ) - return runner.run(scenarios, reporter, options) - - -def command_discover(args): - from . import runner - - if not args.interface: - raise DhcpTestError("discover needs -i/--interface") - reporter = report.make(args.format) - options = runner.RunOptions( - interface=args.interface, - mac=args.mac, - timeout=args.timeout, - retries=args.retries, - pcap=args.pcap, - verbose=args.verbose, - ) - scenario = runner.ad_hoc_discover( - interface=args.interface, - client_arch=args.arch, - vendor_class=args.vendor_class, - user_class=args.user_class, - request_options=args.request_options, - ) - return runner.run([scenario], reporter, options) diff --git a/xCAT-test/dhcptest/src/dhcptest_lib/config.py b/xCAT-test/dhcptest/src/dhcptest_lib/config.py deleted file mode 100644 index e90e0da78..000000000 --- a/xCAT-test/dhcptest/src/dhcptest_lib/config.py +++ /dev/null @@ -1,300 +0,0 @@ -"""Reading declarative .conf files with configparser. - -Layout of a file:: - - [vars] optional; %(name)s values, overridable with --set - [defaults] optional; step keys applied to every step - [scenario ] starts a scenario - [step ] belongs to the scenario above it - -Variables use configparser's own `BasicInterpolation`: `%(name)s` is resolved -by the library, out of the `[vars]` section and of anything passed with -`--set`. `BasicInterpolation` gives `$` no meaning, so `$offer.address` reaches -us untouched -- and that one has to, because its value is a reply that has not -arrived yet when the file is read. `ExtendedInterpolation` would have taken -`${...}` for itself and then raised on the bare `$`. - -The interpolation is applied per value in `_items()` rather than through the -parser, to keep variable names out of the sections' key space -- see the note -on `INTERPOLATION` below. - -Unknown keys are a hard error rather than a silent skip, matching `xcattest`, -where an unrecognised `check:` fails the case instead of passing it. -""" - -import configparser -import os - -from . import assertions as assertions_mod -from . import machine, netutil, subst -from .errors import ConfigError -from .model import Scenario, Step - -#: Keys accepted in a `[scenario ...]` section. -SCENARIO_KEYS = frozenset(["description", "interface"]) - -#: Keys that control how a step is run. -STEP_CONTROL_KEYS = frozenset([ - "type", "expect", "timeout", "retries", "mac", "xid", "broadcast_flag", - "min_size", "collect_extra", "select", "arp_respond", "assert", - "assert_all", "duration", "server", "dest_mac", -]) - -#: Keys that put something into the packet. -STEP_FIELD_KEYS = frozenset([ - "ciaddr", "giaddr", "requested_address", "server_id", "lease_time", - "hostname", "fqdn", "client_id", "vendor_class", "user_class", - "user_class_form", - "client_arch", "client_ndi", "client_uuid", "max_message_size", - "request_options", "vendor_specific", "ipxe_options", "relay", - "bootfile", "sname", -]) - -STEP_KEYS = STEP_CONTROL_KEYS | STEP_FIELD_KEYS - -#: Keys `[defaults]` may carry: step keys, minus the ones that must be per-step. -DEFAULTS_KEYS = (STEP_KEYS - frozenset(["type", "assert", "assert_all"])) | \ - frozenset(["interface"]) - -BOOLEAN_TRUE = frozenset(["1", "yes", "true", "on"]) -BOOLEAN_FALSE = frozenset(["0", "no", "false", "off"]) - - -def load(paths, overrides=None, interface=None, raw=False): - """Read every path and return `(scenarios, variables)`. - - With `raw`, `%(name)s` is left in place instead of being interpolated, so - `validate` can check a file in a checkout where no value has been supplied - for it yet. - """ - scenarios = [] - variables = {} - for path in paths: - file_scenarios, file_vars = load_file(path, overrides, raw) - scenarios.extend(file_scenarios) - variables.update(file_vars) - variables.update(overrides or {}) - - if interface: - scenarios = [ - Scenario(s.name, s.description, interface, s.steps, s.source) - for s in scenarios - ] - - _reject_duplicate_scenarios(scenarios) - return scenarios, variables - - -def load_file(path, overrides=None, raw=False): - """Read one .conf file into `(scenarios, variables)`.""" - if not os.path.isfile(path): - raise ConfigError("no such file", path) - - parser = configparser.ConfigParser( - interpolation=None, - inline_comment_prefixes=(";",), - delimiters=("=",), - ) - parser.optionxform = str.lower - try: - with open(path) as handle: - parser.read_file(handle, source=path) - except configparser.Error as exc: - raise ConfigError(str(exc), path) - - # `[vars]` supplies the values, `--set` overrides them, so one file can - # carry sensible defaults and a run can point it at another network. - variables = dict(parser["vars"]) if parser.has_section("vars") else {} - variables.update(overrides or {}) - - scenarios = [] - defaults = {} - current = None - steps = [] - - for section in parser.sections(): - items = _items(parser, section, path, raw, variables) - kind, _, name = section.partition(" ") - kind = kind.strip().lower() - name = name.strip() - where = "%s [%s]" % (path, section) - - if section.lower() == "vars": - continue # read above, into `variables` - - if section.lower() == "defaults": - _reject_unknown(items, DEFAULTS_KEYS, where) - defaults = items - continue - - if kind == "scenario": - if not name: - raise ConfigError("a [scenario] section needs a name", where) - if current is not None: - scenarios.append(_build_scenario(current, steps, defaults, path)) - _reject_unknown(items, SCENARIO_KEYS, where) - current = (name, items) - steps = [] - continue - - if kind == "step": - if current is None: - raise ConfigError( - "a [step] section must follow a [scenario] section", where) - if not name: - raise ConfigError("a [step] section needs a name", where) - steps.append(_build_step(name, items, defaults, where)) - continue - - raise ConfigError( - "unknown section; expected [vars], [defaults], [scenario ] " - "or [step ]", where) - - if current is not None: - scenarios.append(_build_scenario(current, steps, defaults, path)) - elif steps: - raise ConfigError("steps without a scenario", path) - - if not scenarios: - raise ConfigError("no [scenario ...] section found", path) - - return scenarios, variables - - -def _build_scenario(current, steps, defaults, path): - name, items = current - if not steps: - raise ConfigError("scenario %r has no steps" % (name,), path) - interface = items.get("interface") or defaults.get("interface") - return Scenario( - name=name, - description=items.get("description", ""), - interface=interface, - steps=steps, - source=path, - ) - - -def _build_step(name, items, defaults, where): - _reject_unknown(items, STEP_KEYS, where) - - merged = {} - for key, value in defaults.items(): - if key in STEP_KEYS: - merged[key] = value - merged.update(items) - - step_type = merged.get("type") - if not step_type: - raise ConfigError("step %r has no type=" % (name,), where) - step_type = step_type.strip().lower() - if step_type not in machine.known_step_types(): - raise ConfigError( - "unknown step type %r (known: %s)" - % (step_type, ", ".join(sorted(machine.known_step_types()))), where) - - expect = (merged.get("expect") or "").strip().lower() or None - timeout = _float(merged.get("timeout"), 2.0, "timeout", where) - retries = _int(merged.get("retries"), 3, "retries", where) - - # Validate a literal MAC now so a typo fails offline. A $reference is a - # reply that has not arrived; a %(name)s survives only a raw load, where - # the values were deliberately not supplied. Neither can be checked here. - mac = merged.get("mac", "") - if mac not in ("random", "iface", "") and "$" not in mac \ - and not subst.variables(mac): - netutil.normalise_mac(mac) - - parsed = assertions_mod.parse_block(merged.pop("assert", ""), where) - parsed_all = assertions_mod.parse_block(merged.pop("assert_all", ""), where) - - for key in ("type", "expect", "timeout", "retries"): - merged.pop(key, None) - - return Step( - name=name, - type=step_type, - expect=expect, - params=merged, - assertions=parsed, - assertions_all=parsed_all, - timeout=timeout, - retries=retries, - source=where, - ) - - -#: configparser's own variable syntax, `%(name)s`. The parser is built with -#: `interpolation=None` and this is applied by hand, per value, because both of -#: the built-in ways of supplying the values -- DEFAULTSECT and `get(vars=...)` -#: -- also make each variable an *option*, so `--set user_class=xNBA` would -#: overwrite the step key of the same name. Interpolating explicitly keeps the -#: two namespaces apart: variables are only ever read through `%(...)s`. -INTERPOLATION = configparser.BasicInterpolation() - - -def _items(parser, section, path, raw, variables=None): - """One section's keys, with `%(name)s` resolved out of `variables`.""" - items = {} - for key in parser.options(section): - value = parser.get(section, key, raw=True) - if raw: - items[key] = value - continue - try: - items[key] = INTERPOLATION.before_get( - parser, section, key, value, variables or {}) - except configparser.InterpolationMissingOptionError as exc: - raise ConfigError( - "%%(%s)s is not defined; pass --set %s=" - % (exc.reference, exc.reference), "%s [%s]" % (path, section)) - except configparser.Error as exc: - raise ConfigError(str(exc), "%s [%s]" % (path, section)) - return items - - -def _reject_unknown(items, allowed, where): - unknown = sorted(set(items) - set(allowed)) - if unknown: - raise ConfigError( - "unknown key(s): %s (allowed: %s)" - % (", ".join(unknown), ", ".join(sorted(allowed))), where) - - -def _reject_duplicate_scenarios(scenarios): - seen = {} - for scenario in scenarios: - if scenario.name in seen: - raise ConfigError( - "scenario %r is defined in both %s and %s" - % (scenario.name, seen[scenario.name], scenario.source)) - seen[scenario.name] = scenario.source - - -def _float(value, default, key, where): - if value is None or value == "": - return default - try: - return float(value) - except ValueError: - raise ConfigError("%s= must be a number, got %r" % (key, value), where) - - -def _int(value, default, key, where): - if value is None or value == "": - return default - try: - return int(value, 0) if isinstance(value, str) else int(value) - except ValueError: - raise ConfigError("%s= must be an integer, got %r" % (key, value), where) - - -def to_bool(value, default=False): - if value is None or value == "": - return default - text = str(value).strip().lower() - if text in BOOLEAN_TRUE: - return True - if text in BOOLEAN_FALSE: - return False - raise ConfigError("expected a yes/no value, got %r" % (value,)) diff --git a/xCAT-test/dhcptest/src/dhcptest_lib/errors.py b/xCAT-test/dhcptest/src/dhcptest_lib/errors.py deleted file mode 100644 index 100bb7a3d..000000000 --- a/xCAT-test/dhcptest/src/dhcptest_lib/errors.py +++ /dev/null @@ -1,39 +0,0 @@ -"""Exception types and the exit codes they map to. - -Exit codes are part of the tool's contract, because callers such as `xcattest` -check them: - - 0 every assertion passed (or everything was skipped) - 1 at least one assertion failed - 2 the configuration or command line was wrong - 3 the environment was wrong (no such interface, not root, no scapy) -""" - -EXIT_OK = 0 -EXIT_FAILED = 1 -EXIT_CONFIG = 2 -EXIT_ENVIRONMENT = 3 - - -class DhcpTestError(Exception): - """Base class for every error this tool raises deliberately.""" - - exit_code = EXIT_CONFIG - - -class ConfigError(DhcpTestError): - """A .conf file is malformed, or references something that cannot exist.""" - - exit_code = EXIT_CONFIG - - def __init__(self, message, source=None): - if source: - message = "%s: %s" % (source, message) - Exception.__init__(self, message) - self.source = source - - -class EnvironmentError_(DhcpTestError): - """The host cannot run this test: no interface, no privileges, no scapy.""" - - exit_code = EXIT_ENVIRONMENT diff --git a/xCAT-test/dhcptest/src/dhcptest_lib/machine.py b/xCAT-test/dhcptest/src/dhcptest_lib/machine.py deleted file mode 100644 index 1099a4d13..000000000 --- a/xCAT-test/dhcptest/src/dhcptest_lib/machine.py +++ /dev/null @@ -1,259 +0,0 @@ -"""The DHCP client state machine. - -The transition table below is a subset of the RFC 2131 client FSM. Keeping it -as data rather than as control flow buys two things: - - * `dhcptest validate` can walk a scenario without touching the network, so a - `request` step with no preceding `discover` is a configuration error caught - in CI rather than a mystery in a lab; - * the table is the protocol documentation, and is trivially unit testable. - -This module stays importable without scapy. `run_step` imports the packet and -transport layers lazily, so `validate` and `list` work on a host that has no -scapy and no privileges. -""" - -from . import assertions as assertions_mod -from . import subst -from .errors import ConfigError - -# States -INIT = "INIT" -SELECTING = "SELECTING" -REQUESTING = "REQUESTING" -BOUND = "BOUND" -RENEWING = "RENEWING" -REBINDING = "REBINDING" - -ANY = "*" - -#: Step types that send nothing and therefore have no transition. -PASSIVE_TYPES = frozenset(["noop", "sleep"]) - - -class Transition(object): - """One legal (state, step type) pair and what it does.""" - - __slots__ = ("expects", "default_expect", "next_state", "nak_state", - "requires", "binds", "unicast") - - def __init__(self, expects, default_expect, next_state, nak_state=None, - requires=(), binds=(), unicast=False): - self.expects = frozenset(expects) - self.default_expect = default_expect - self.next_state = next_state - self.nak_state = nak_state - self.requires = tuple(requires) - self.binds = tuple(binds) - self.unicast = unicast - - -TRANSITIONS = { - (INIT, "discover"): Transition( - expects=["offer"], default_expect="offer", next_state=SELECTING, - binds=("offer",)), - - # A BOOTREQUEST is what a client that predates DHCP sends: the same BOOTP - # header with no option 53 at all. There is no handshake to follow it -- - # the single BOOTREPLY is the whole exchange -- so it leaves the session in - # INIT rather than moving on to SELECTING. - (INIT, "bootrequest"): Transition( - expects=["bootreply"], default_expect="bootreply", next_state=INIT, - binds=("reply",)), - - (SELECTING, "request"): Transition( - expects=["ack", "nak"], default_expect="ack", next_state=BOUND, - nak_state=INIT, binds=("ack", "lease")), - - # INIT-REBOOT: a REQUEST with option 50 and no option 54. - (INIT, "request"): Transition( - expects=["ack", "nak"], default_expect="ack", next_state=BOUND, - nak_state=INIT, requires=("requested_address",), binds=("ack", "lease")), - - (BOUND, "renew"): Transition( - expects=["ack", "nak"], default_expect="ack", next_state=BOUND, - nak_state=INIT, binds=("ack", "lease"), unicast=True), - - (RENEWING, "renew"): Transition( - expects=["ack", "nak"], default_expect="ack", next_state=BOUND, - nak_state=INIT, binds=("ack", "lease"), unicast=True), - - (BOUND, "rebind"): Transition( - expects=["ack", "nak"], default_expect="ack", next_state=BOUND, - nak_state=INIT, binds=("ack", "lease")), - - (RENEWING, "rebind"): Transition( - expects=["ack", "nak"], default_expect="ack", next_state=BOUND, - nak_state=INIT, binds=("ack", "lease")), - - (BOUND, "release"): Transition( - expects=[], default_expect="none", next_state=INIT, unicast=True), - - (BOUND, "decline"): Transition( - expects=[], default_expect="none", next_state=INIT), - - # INFORM asks for configuration without asking for an address, and is legal - # from any state. - (ANY, "inform"): Transition( - expects=["ack"], default_expect="ack", next_state=None, - binds=("ack",)), -} - -#: Bindings that exist before any step runs. -INITIAL_BINDINGS = frozenset() - - -def transition_for(state, step_type): - """The transition for `(state, step_type)`, or None.""" - if (state, step_type) in TRANSITIONS: - return TRANSITIONS[(state, step_type)] - return TRANSITIONS.get((ANY, step_type)) - - -def known_step_types(): - types = set(PASSIVE_TYPES) - types.update(step_type for _, step_type in TRANSITIONS) - return types - - -def states_allowing(step_type): - """Which states a step type is legal in -- used to explain a rejection.""" - states = [state for state, kind in TRANSITIONS if kind == step_type] - return sorted(states) - - -# --------------------------------------------------------------------------- -# offline validation - - -def validate_scenario(scenario): - """Walk a scenario without sending anything. - - Returns a list of human-readable problems; empty means the scenario is - internally consistent. Nothing here needs root, a network, or scapy. - """ - problems = [] - state = INIT - available = set() # no binding exists until a step produces one - seen_names = set() - - for step in scenario.steps: - where = "%s/%s" % (scenario.name, step.name) - - if step.name in seen_names: - problems.append("%s: duplicate step name %r" % (where, step.name)) - seen_names.add(step.name) - - problems.extend(_check_references(step, available, where)) - - if step.type in PASSIVE_TYPES: - available.add(step.name) - continue - - if step.type not in known_step_types(): - problems.append( - "%s: unknown step type %r (known: %s)" - % (where, step.type, ", ".join(sorted(known_step_types())))) - continue - - transition = transition_for(state, step.type) - if transition is None: - allowed = states_allowing(step.type) - problems.append( - "%s: a %r step is not legal in state %s (legal in: %s)" - % (where, step.type, state, ", ".join(allowed) or "no state")) - continue - - for required in transition.requires: - if not step.param(required): - problems.append( - "%s: a %r step in state %s needs %s=" - % (where, step.type, state, required)) - - expect = step.expect or transition.default_expect - if expect not in ("none", "any") and expect not in transition.expects: - problems.append( - "%s: expect=%s is not a possible reply to %r in state %s " - "(possible: %s)" - % (where, expect, step.type, state, - ", ".join(sorted(transition.expects)) or "nothing")) - - available.add(step.name) - if expect not in ("none",): - available.add("reply") - for name in transition.binds: - available.add(name) - - # A step that expected a reply and got none leaves the client where it - # was: a DISCOVER nobody answers does not put it in SELECTING. RELEASE - # and DECLINE are the other kind of silence -- no reply is ever sent to - # them, and giving the lease up is the whole point of the step -- so - # they move the client on regardless. - if expect == "none" and transition.expects: - continue - if transition.next_state: - state = transition.next_state - - return problems - - -def _check_references(step, available, where): - """Every `$binding.field` a step uses must already be bound.""" - problems = [] - values = list(step.params.values()) - values.extend(a.value for a in step.assertions if a.value is not None) - values.extend(a.target for a in step.assertions) - values.extend(a.value for a in step.assertions_all if a.value is not None) - values.extend(a.target for a in step.assertions_all) - - for value in values: - for name, field in subst.references(value): - if name not in available: - problems.append( - "%s: $%s.%s is used before anything binds $%s" - % (where, name, field, name)) - elif not subst.check_field(field): - problems.append( - "%s: $%s.%s is not a field of a DHCP reply" - % (where, name, field)) - return problems - - -def required_variables(scenarios): - """Every `%(name)s` the given scenarios still need. - - Only meaningful on scenarios loaded raw: once configparser has - interpolated a value, the name it came from is gone. - """ - names = set() - for scenario in scenarios: - for step in scenario.steps: - values = list(step.params.values()) - values.extend(a.value for a in step.assertions if a.value is not None) - values.extend(a.value for a in step.assertions_all - if a.value is not None) - for value in values: - names.update(subst.variables(value)) - return names - - -def expected_types(step, state): - """The DHCP message types a step will accept as its reply.""" - transition = transition_for(state, step.type) - if transition is None: - raise ConfigError("no transition for %s in state %s" % (step.type, state)) - expect = step.expect or transition.default_expect - if expect == "none": - return frozenset() - if expect == "any": - return transition.expects - return frozenset([expect]) - - -def evaluate_step(step, reply, context, extras=None): - """Run a step's assertions against its reply. Returns a list of Results.""" - results = [] - for assertion in step.assertions: - results.append( - assertions_mod.evaluate(assertion, reply, context, extras)) - return results diff --git a/xCAT-test/dhcptest/src/dhcptest_lib/model.py b/xCAT-test/dhcptest/src/dhcptest_lib/model.py deleted file mode 100644 index d2cd2eac5..000000000 --- a/xCAT-test/dhcptest/src/dhcptest_lib/model.py +++ /dev/null @@ -1,273 +0,0 @@ -"""Plain data structures shared across the tool. - -No network, no scapy, no I/O. Written for Python 3.6, so no dataclasses. -""" - -from .errors import ConfigError - - -class Assertion(object): - """One line of the assertion mini-language. - - `target op value`, for example:: - - msgtype == OFFER - yiaddr in 10.0.0.0/24 - option:54 present - file matches ^http:// - """ - - __slots__ = ("target", "op", "value", "source", "text") - - def __init__(self, target, op, value, source="", text=""): - self.target = target - self.op = op - self.value = value - self.source = source - self.text = text or self.render() - - def render(self): - if self.value is None: - return "%s %s" % (self.target, self.op) - return "%s %s %s" % (self.target, self.op, self.value) - - def __repr__(self): - return "Assertion(%r)" % (self.render(),) - - -class Step(object): - """One protocol action plus the assertions made about its reply.""" - - __slots__ = ( - "name", - "type", - "expect", - "params", - "assertions", - "assertions_all", - "timeout", - "retries", - "source", - ) - - def __init__(self, name, type, expect, params, assertions, - assertions_all=(), timeout=2.0, retries=3, source=""): - self.name = name - self.type = type - self.expect = expect - self.params = dict(params) - self.assertions = tuple(assertions) - self.assertions_all = tuple(assertions_all) - self.timeout = timeout - self.retries = retries - self.source = source - - def param(self, key, default=None): - return self.params.get(key, default) - - def __repr__(self): - return "Step(%r, type=%r, expect=%r)" % (self.name, self.type, self.expect) - - -class Scenario(object): - """A named sequence of steps run against one interface, in order.""" - - __slots__ = ("name", "description", "interface", "steps", "source") - - def __init__(self, name, description="", interface=None, steps=(), source=""): - self.name = name - self.description = description - self.interface = interface - self.steps = tuple(steps) - self.source = source - - def step(self, name): - for step in self.steps: - if step.name == name: - return step - raise ConfigError("no such step %r in scenario %r" % (name, self.name)) - - def __repr__(self): - return "Scenario(%r, steps=%d)" % (self.name, len(self.steps)) - - -class Reply(object): - """A decoded DHCP reply. - - Built by `options.decode()` from a scapy packet, but the class itself - knows nothing about scapy, so assertions can be tested against synthetic - replies with no capture and no root. - """ - - __slots__ = ( - "msgtype", - "xid", - "yiaddr", - "siaddr", - "ciaddr", - "giaddr", - "chaddr", - "file", - "sname", - "secs", - "flags", - "options", - "raw_options", - "src_ip", - "src_mac", - "attempt", - "raw", - ) - - def __init__(self, msgtype="", xid=0, yiaddr="0.0.0.0", siaddr="0.0.0.0", - ciaddr="0.0.0.0", giaddr="0.0.0.0", chaddr="", file="", - sname="", secs=0, flags=0, options=None, src_ip="", - src_mac="", attempt=0, raw=None, raw_options=b""): - self.msgtype = msgtype - self.xid = xid - self.yiaddr = yiaddr - self.siaddr = siaddr - self.ciaddr = ciaddr - self.giaddr = giaddr - self.chaddr = chaddr - self.file = file - self.sname = sname - self.secs = secs - self.flags = flags - self.options = dict(options or {}) - # The option area as it arrived. Option 81 is decoded to its name, so - # the flag bits are only recoverable from here. - self.raw_options = raw_options - self.src_ip = src_ip - self.src_mac = src_mac - self.attempt = attempt - self.raw = raw - - # -- the fields reachable through $step. in a .conf --------------- - - #: Aliases so a .conf can say `$offer.address` rather than `$offer.yiaddr`. - FIELD_ALIASES = { - "address": "yiaddr", - "mac": "chaddr", - "next_server": "siaddr", - } - - #: Fields a client derives from more than one place in the message. - DERIVED_FIELDS = frozenset(["bootfile", "fqdn_flags", "dns_name"]) - - #: Fields that come from an option rather than the BOOTP header. - FIELD_OPTIONS = { - "server_id": 54, - "lease_time": 51, - "renewal_time": 58, - "rebinding_time": 59, - "subnet_mask": 1, - "router": 3, - "domain_name": 15, - "hostname": 12, - "vendor_class": 60, - "user_class": 77, - } - - def bootfile(self): - """What this reply would actually make a client boot. - - RFC 2132 option 67 also carries a boot file name, and a server that was - asked for option 67 may answer there and leave the BOOTP `file` header - empty -- dnsmasq does exactly that, while ISC dhcpd fills the header. - Firmware reads whichever arrived, so a test that looked only at the - header would pass against one server and fail against another over a - difference no client can observe. - """ - return self.options.get(67) or self.file - - def dns_name(self): - """The name this reply says the node will be known by. - - Option 81 when the server sent one, option 12 otherwise. The two - backends put it in different places -- Kea answers a client that sent - option 81 with option 81, ISC with `ignore client-updates` sends only - option 12 -- and a scenario about the name should not have to know - which. A client reads whichever arrived, exactly as with `bootfile`. - """ - return self.options.get(81) or self.options.get(12) or "" - - def fqdn_flags(self): - """Option 81's flags as letters -- "SO" -- or "" when it is absent. - - S says the server will update the A record and O that it overrode what - the client asked for. The two together are how a reply says the client - does not get to name itself. - """ - from . import options # local import: avoids a cycle - - raw = self.options.get(81) - if raw is None: - return "" - if isinstance(raw, str): - # decode_value already reduced the option to its name, so the flag - # bits have to come off the packet again. - raw = self._raw_option(81) - if raw is None: - return "" - decoded = options.decode_fqdn(raw) - return options.fqdn_flag_letters(decoded[0]) if decoded else "" - - def _raw_option(self, code): - """The undecoded bytes of one option, straight off the wire.""" - from . import options # local import: avoids a cycle - - blob = self.raw_options - if not blob: - return None - return options.unpack_options(blob).get(code) - - def field(self, name): - """Resolve one `$step.` reference, or raise KeyError.""" - if name == "bootfile": - return self.bootfile() - if name == "fqdn_flags": - return self.fqdn_flags() - if name == "dns_name": - return self.dns_name() - if name in self.FIELD_OPTIONS: - code = self.FIELD_OPTIONS[name] - if code not in self.options: - raise KeyError( - "reply has no option %d, needed for .%s" % (code, name)) - return self.options[code] - if name.startswith("opt"): - try: - code = int(name[3:], 0) - except ValueError: - raise KeyError("bad option reference .%s" % (name,)) - if code not in self.options: - raise KeyError("reply has no option %d" % (code,)) - return self.options[code] - target = self.FIELD_ALIASES.get(name, name) - if target in self.__slots__ and target not in ("options", "raw"): - return getattr(self, target) - raise KeyError("unknown reply field .%s" % (name,)) - - @classmethod - def known_fields(cls): - """Every `$step.` name that can be validated offline.""" - names = set(cls.FIELD_OPTIONS) - names.update(cls.FIELD_ALIASES) - names.update(cls.DERIVED_FIELDS) - names.update( - f for f in cls.__slots__ - if f not in ("options", "raw_options", "raw", "attempt") - ) - return names - - def summary(self): - who = self.src_ip or "?" - if self.src_mac: - who = "%s (%s)" % (who, self.src_mac) - return "%s from %s xid=0x%08x yiaddr=%s siaddr=%s file=%r" % ( - self.msgtype or "?", who, self.xid, self.yiaddr, self.siaddr, - self.file) - - def __repr__(self): - return "Reply(%s)" % (self.summary(),) diff --git a/xCAT-test/dhcptest/src/dhcptest_lib/netutil.py b/xCAT-test/dhcptest/src/dhcptest_lib/netutil.py deleted file mode 100644 index a96587fc9..000000000 --- a/xCAT-test/dhcptest/src/dhcptest_lib/netutil.py +++ /dev/null @@ -1,107 +0,0 @@ -"""Address and interface helpers built on the standard library only.""" - -import ipaddress -import os -import random -import re - -from .errors import ConfigError, EnvironmentError_ - -MAC_RE = re.compile(r"^([0-9a-fA-F]{2}:){5}[0-9a-fA-F]{2}$") - -SYS_NET = "/sys/class/net" - - -def normalise_mac(text): - """Accept `aa:bb:cc:dd:ee:ff`, `aa-bb-...` or `aabbccddeeff`.""" - cleaned = str(text).strip().lower().replace("-", ":") - if ":" not in cleaned and len(cleaned) == 12: - cleaned = ":".join(cleaned[i:i + 2] for i in range(0, 12, 2)) - if not MAC_RE.match(cleaned): - raise ConfigError("not a MAC address: %r" % (text,)) - return cleaned - - -def random_mac(rng=None): - """A locally administered unicast MAC. - - Using a synthetic address rather than the interface's own is what keeps - the host's network stack out of the test: NetworkManager and any running - dhclient never see the replies as theirs, and the kernel drops the unicast - ones because neither the destination MAC nor the destination IP is local. - """ - rng = rng or random - octets = [0x02] + [rng.randint(0x00, 0xFF) for _ in range(5)] - return ":".join("%02x" % o for o in octets) - - -def mac_bytes(mac): - return bytes(bytearray(int(part, 16) for part in normalise_mac(mac).split(":"))) - - -def interface_exists(name): - return os.path.isdir(os.path.join(SYS_NET, name)) - - -def interface_mac(name): - """Read an interface's own MAC without shelling out to `ip`.""" - path = os.path.join(SYS_NET, name, "address") - try: - with open(path) as handle: - return normalise_mac(handle.read().strip()) - except (IOError, OSError): - raise EnvironmentError_("cannot read the MAC address of %r" % (name,)) - - -def parse_ip(text): - try: - return ipaddress.IPv4Address(str(text).strip()) - except (ipaddress.AddressValueError, ValueError): - return None - - -def parse_network(text): - try: - return ipaddress.IPv4Network(str(text).strip(), strict=False) - except (ipaddress.AddressValueError, ipaddress.NetmaskValueError, ValueError): - return None - - -def ip_in(address, network): - """True when `address` sits inside CIDR `network`.""" - addr = parse_ip(address) - net = parse_network(network) - if addr is None or net is None: - return False - return addr in net - - -def parse_range(text): - """`first-last` as a pair of addresses, or None if it is not one. - - A dynamic pool is usually written as two addresses rather than a CIDR -- - it rarely lines up on a prefix boundary -- so `yiaddr in 10.0.0.200-10.0.0.250` - has to mean what it says. - """ - first, sep, last = str(text).strip().partition("-") - if not sep: - return None - low = parse_ip(first) - high = parse_ip(last) - if low is None or high is None: - return None - return (low, high) if low <= high else (high, low) - - -def ip_in_range(address, text): - """True when `address` falls within the inclusive range `first-last`.""" - addr = parse_ip(address) - bounds = parse_range(text) - if addr is None or bounds is None: - return False - return bounds[0] <= addr <= bounds[1] - - -def random_xid(rng=None): - rng = rng or random - return rng.randint(1, 0xFFFFFFFF) diff --git a/xCAT-test/dhcptest/src/dhcptest_lib/report.py b/xCAT-test/dhcptest/src/dhcptest_lib/report.py deleted file mode 100644 index 26a79d9a7..000000000 --- a/xCAT-test/dhcptest/src/dhcptest_lib/report.py +++ /dev/null @@ -1,216 +0,0 @@ -"""Result reporting: TAP 13 by default, plus human and JSON forms. - -TAP is the default because `prove` consumes it directly, which is how this -repository already drives its Perl integration tests. The failure diagnostic -carries expected, received, the whole decoded reply and a summary of what was -sent, so a red test can be understood without re-running it. -""" - -import json -import sys - -from .errors import EXIT_FAILED, EXIT_OK -from . import options as options_mod - - -class Record(object): - """One assertion outcome, with everything needed to explain it.""" - - __slots__ = ("scenario", "step", "description", "ok", "skip_reason", - "expected", "actual", "detail", "reply", "sent", "attempt", - "attempts") - - def __init__(self, scenario, step, description, ok, skip_reason=None, - expected=None, actual=None, detail="", reply=None, sent="", - attempt=0, attempts=0): - self.scenario = scenario - self.step = step - self.description = description - self.ok = ok - self.skip_reason = skip_reason - self.expected = expected - self.actual = actual - self.detail = detail - self.reply = reply - self.sent = sent - self.attempt = attempt - self.attempts = attempts - - @property - def name(self): - return "%s/%s: %s" % (self.scenario, self.step, self.description) - - -class Reporter(object): - """Base reporter: counts, and an exit code.""" - - def __init__(self, stream=None): - self.stream = stream or sys.stdout - self.records = [] - self.passed = 0 - self.failed = 0 - self.skipped = 0 - - def add(self, record): - self.records.append(record) - if record.skip_reason is not None: - self.skipped += 1 - elif record.ok: - self.passed += 1 - else: - self.failed += 1 - self.emit(record) - - def emit(self, record): - raise NotImplementedError - - def start(self): - pass - - def finish(self): - pass - - def exit_code(self): - return EXIT_FAILED if self.failed else EXIT_OK - - def write(self, text): - self.stream.write(text) - self.stream.flush() - - -class TapReporter(Reporter): - """TAP version 13, with a trailing plan and YAML failure diagnostics.""" - - def start(self): - self.write("TAP version 13\n") - - def emit(self, record): - number = len(self.records) - status = "ok" if (record.ok or record.skip_reason is not None) else "not ok" - line = "%s %d - %s" % (status, number, record.name) - if record.skip_reason is not None: - line += " # SKIP %s" % (record.skip_reason,) - self.write(line + "\n") - if not record.ok and record.skip_reason is None: - self.write(_yaml_block(record)) - - def finish(self): - self.write("1..%d\n" % (len(self.records),)) - self.write("# passed %d, failed %d, skipped %d\n" - % (self.passed, self.failed, self.skipped)) - - -class PrettyReporter(Reporter): - """Human-first output carrying the same information as the TAP form.""" - - def __init__(self, stream=None): - Reporter.__init__(self, stream) - self._scenario = None - - def emit(self, record): - if record.scenario != self._scenario: - self._scenario = record.scenario - self.write("\n%s\n" % (record.scenario,)) - if record.skip_reason is not None: - self.write(" SKIP %s/%s -- %s\n" - % (record.step, record.description, record.skip_reason)) - return - mark = "PASS" if record.ok else "FAIL" - self.write(" %s %s: %s\n" % (mark, record.step, record.description)) - if not record.ok: - for line in _diagnostic_lines(record): - self.write(" %s\n" % (line,)) - - def finish(self): - self.write("\nDHCPTEST SUMMARY: total=%d passed=%d failed=%d skipped=%d\n" - % (len(self.records), self.passed, self.failed, self.skipped)) - - -class JsonReporter(Reporter): - """One JSON object per run, for callers that would rather not parse text.""" - - def emit(self, record): - pass - - def finish(self): - payload = { - "total": len(self.records), - "passed": self.passed, - "failed": self.failed, - "skipped": self.skipped, - "records": [_as_dict(record) for record in self.records], - } - self.write(json.dumps(payload, indent=2, sort_keys=True) + "\n") - - -REPORTERS = { - "tap": TapReporter, - "pretty": PrettyReporter, - "json": JsonReporter, -} - - -def make(name, stream=None): - try: - return REPORTERS[name](stream) - except KeyError: - raise ValueError("unknown report format %r" % (name,)) - - -# --------------------------------------------------------------------------- -# diagnostics - - -def _diagnostic_lines(record): - lines = [] - if record.expected is not None: - lines.append("expected: %r" % (record.expected,)) - lines.append("received: %r" % (record.actual,)) - if record.detail: - lines.append("detail: %s" % (record.detail,)) - if record.reply is not None: - lines.append("reply: %s" % (record.reply.summary(),)) - lines.append("options: %s" % (_render_options(record.reply),)) - elif record.skip_reason is None: - lines.append("reply: none received") - if record.sent: - lines.append("sent: %s" % (record.sent,)) - if record.attempts: - lines.append("attempt: %d of %d" % (record.attempt, record.attempts)) - return lines - - -def _yaml_block(record): - out = [" ---\n"] - for line in _diagnostic_lines(record): - key, _, value = line.partition(":") - out.append(" %s: %s\n" % (key.strip(), value.strip())) - out.append(" ...\n") - return "".join(out) - - -def _render_options(reply): - parts = [] - for code in sorted(reply.options): - value = reply.options[code] - parts.append("%s=%s" % (options_mod.option_label(code), value)) - return "{" + ", ".join(parts) + "}" - - -def _as_dict(record): - return { - "scenario": record.scenario, - "step": record.step, - "assertion": record.description, - "ok": bool(record.ok), - "skip": record.skip_reason, - "expected": record.expected, - "actual": record.actual, - "detail": record.detail, - "reply": record.reply.summary() if record.reply is not None else None, - "options": (dict((str(k), v) for k, v in record.reply.options.items()) - if record.reply is not None else None), - "sent": record.sent, - "attempt": record.attempt, - "attempts": record.attempts, - } diff --git a/xCAT-test/dhcptest/src/dhcptest_lib/runner.py b/xCAT-test/dhcptest/src/dhcptest_lib/runner.py deleted file mode 100644 index 3e5bf33c8..000000000 --- a/xCAT-test/dhcptest/src/dhcptest_lib/runner.py +++ /dev/null @@ -1,512 +0,0 @@ -"""Sending, receiving and running scenarios. The only module that uses scapy. - -Everything is raw Layer 2: the tool builds the whole frame, so it works on an -interface that has no address yet, which is the normal state of a provisioning -NIC and the reason the existing Perl probe cannot be used there. -""" - -import ipaddress -import os -import random -import select -import time - -from . import assertions as assertions_mod -from . import machine, netutil, report, subst -from . import options as opt -from .errors import (ConfigError, EnvironmentError_, EXIT_OK) -from .subst import Context - -BPF = "(udp and (port 67 or port 68)) or arp" - - -def _scapy(): - """Import scapy on demand, with a message a human can act on.""" - try: - from scapy.all import (ARP, BOOTP, Ether, IP, UDP, Raw, conf, wrpcap) - except ImportError: - raise EnvironmentError_( - "scapy is not installed; install the python3-scapy package") - return dict(ARP=ARP, BOOTP=BOOTP, Ether=Ether, IP=IP, UDP=UDP, Raw=Raw, - conf=conf, wrpcap=wrpcap) - - -class RunOptions(object): - """Command-line overrides that apply to every scenario in a run.""" - - def __init__(self, interface=None, mac=None, timeout=None, retries=None, - pcap=None, verbose=0): - self.interface = interface - self.mac = mac - self.timeout = timeout - self.retries = retries - self.pcap = pcap - self.verbose = verbose - - -class Session(object): - """One client's state across the steps of a scenario.""" - - def __init__(self, mac, interface): - self.mac = mac - self.mac_bytes = netutil.mac_bytes(mac) - self.interface = interface - self.state = machine.INIT - self.xid = netutil.random_xid() - self.context = Context() - self.leases = set() # addresses we may answer ARP for - - -# --------------------------------------------------------------------------- -# building packets - - -def _encode(code, text): - """Turn a .conf value into the bytes of one DHCP option.""" - text = "" if text is None else str(text) - if code in opt.IP_OPTIONS: - return ipaddress.IPv4Address(text.strip()).packed - if code in opt.IP_LIST_OPTIONS: - blob = b"" - for item in text.split(","): - blob += ipaddress.IPv4Address(item.strip()).packed - return blob - if code in opt.U32_OPTIONS: - return int(text, 0).to_bytes(4, "big") - if code in opt.U16_OPTIONS: - return int(text, 0).to_bytes(2, "big") - if code in opt.U8_OPTIONS: - return int(text, 0).to_bytes(1, "big") - if code == 55: - return bytes(bytearray(opt.option_code(item.strip()) - for item in text.split(",") if item.strip())) - if code in opt.TEXT_OPTIONS: - return text.encode("utf-8") - if code == 81: - return opt.encode_fqdn(text) - return opt.parse_hex(text) - - -def build_options(step, session, context, msgtype): - """The option area for one outgoing message.""" - def value(key, default=None): - raw = step.param(key, default) - return subst.resolve(raw, context) if raw is not None else None - - # A BOOTREQUEST carries no option 53, and no option 61 either: a client - # that predates DHCP has neither to send. Anything the step asks for beyond - # that is still encoded, so a BOOTP client can still name a vendor class. - if msgtype == "bootrequest": - items = [] - else: - items = [(53, bytes([opt.msgtype_code(msgtype)]))] - - client_id = value("client_id", "none" if msgtype == "bootrequest" else "auto") - if client_id and client_id != "none": - if client_id == "auto": - items.append((61, b"\x01" + session.mac_bytes)) - else: - items.append((61, _encode(61, client_id))) - - simple = [ - ("requested_address", 50), ("server_id", 54), ("lease_time", 51), - ("hostname", 12), ("fqdn", 81), ("max_message_size", 57), - ("vendor_class", 60), - ("client_arch", 93), ("client_ndi", 94), ("client_uuid", 97), - ("vendor_specific", 43), ("ipxe_options", 175), - ("request_options", 55), - ] - for key, code in simple: - text = value(key) - if text not in (None, ""): - items.append((code, _encode(code, text))) - - user_class = value("user_class") - if user_class: - form = (value("user_class_form") or "raw").lower() - raw = user_class.encode("utf-8") - if form == "rfc3004": - raw = bytes([len(raw)]) + raw - elif form != "raw": - raise ConfigError("user_class_form must be 'raw' or 'rfc3004'") - items.append((77, raw)) - - for key, text in step.params.items(): - if key.startswith("option:"): - code = opt.option_code(key) - items.append((code, _encode(code, subst.resolve(text, context)))) - - return items - - -def build_packet(step, session, context, msgtype, scapy): - """One complete Ethernet frame carrying a DHCP message.""" - def value(key, default=None): - raw = step.param(key, default) - return subst.resolve(raw, context) if raw is not None else None - - ciaddr = value("ciaddr", "0.0.0.0") - giaddr = value("giaddr", "0.0.0.0") - transition = machine.transition_for(session.state, step.type) - unicast = bool(transition and transition.unicast) and value("server_id") - - broadcast_flag = value("broadcast_flag") - if broadcast_flag is None: - flags = 0 if unicast or ciaddr != "0.0.0.0" else 0x8000 - else: - flags = 0x8000 if broadcast_flag.strip().lower() in ("1", "yes", "true") else 0 - - if unicast: - dst_ip = value("server_id") - dst_mac = value("dest_mac") or "ff:ff:ff:ff:ff:ff" - src_ip = ciaddr - else: - dst_ip = "255.255.255.255" - dst_mac = value("dest_mac") or "ff:ff:ff:ff:ff:ff" - src_ip = ciaddr - - blob = opt.MAGIC + opt.pack_options(build_options(step, session, context, msgtype)) - min_size = int(value("min_size") or 300) - if len(blob) < min_size - 240: - blob += b"\x00" * (min_size - 240 - len(blob)) - - frame = ( - scapy["Ether"](src=session.mac, dst=dst_mac) / - scapy["IP"](src=src_ip, dst=dst_ip) / - scapy["UDP"](sport=68, dport=67) / - scapy["BOOTP"](op=1, htype=1, hlen=6, xid=session.xid, flags=flags, - ciaddr=ciaddr, giaddr=giaddr, chaddr=session.mac_bytes) / - scapy["Raw"](load=blob) - ) - return frame - - -def describe_sent(step, session, context, msgtype): - """A one-line summary of what went out, for failure diagnostics.""" - parts = ["%s mac=%s xid=0x%08x" % (msgtype.upper(), session.mac, session.xid)] - for key in sorted(step.params): - if key in ("assert", "assert_all", "select", "collect_extra"): - continue - value = subst.resolve(step.params[key], context) - if value not in (None, ""): - parts.append("%s=%s" % (key, value)) - return " ".join(parts) - - -# --------------------------------------------------------------------------- -# the wire - - -class Wire(object): - """A Layer 2 socket, opened before the first send so no reply is missed.""" - - def __init__(self, interface, pcap=None, verbose=0): - self.scapy = _scapy() - if not netutil.interface_exists(interface): - raise EnvironmentError_("no such interface: %s" % (interface,)) - if os.geteuid() != 0: - raise EnvironmentError_("raw sockets need root; re-run with sudo") - try: - self.socket = self.scapy["conf"].L2socket(iface=interface, filter=BPF) - except Exception as exc: # scapy raises many types - raise EnvironmentError_( - "cannot open a raw socket on %s: %s" % (interface, exc)) - self.pcap_path = pcap - self.frames = [] - self.verbose = verbose - - def close(self): - try: - self.socket.close() - finally: - if self.pcap_path and self.frames: - self.scapy["wrpcap"](self.pcap_path, self.frames) - - def send(self, frame): - self.frames.append(frame) - self.socket.send(frame) - - def collect(self, session, deadline, arp_for=()): - """Every DHCP reply for this session until `deadline`. - - Frames that fail the cheap checks are dropped silently. A frame that - is ours but carries an unexpected message type is still returned, so a - NAK where an ACK was wanted is reported as a NAK rather than as a - timeout. - """ - found = [] - while True: - remaining = deadline - time.monotonic() - if remaining <= 0: - break - ready, _, _ = select.select([self.socket], [], [], remaining) - if not ready: - break - frame = self.socket.recv(2048) - if frame is None: - continue - self.frames.append(frame) - - if arp_for and frame.haslayer("ARP"): - self._answer_arp(frame, session, arp_for) - continue - - if not frame.haslayer("BOOTP"): - continue - bootp = frame["BOOTP"] - if int(bootp.op) != 2: - continue - if int(bootp.xid) != session.xid: - continue - if bytes(bootp.chaddr)[:6] != session.mac_bytes: - continue - if frame.haslayer("Ether") and frame["Ether"].src == session.mac: - continue - found.append(opt.decode(frame)) - return found - - def _answer_arp(self, frame, session, addresses): - """Answer ARP only for addresses this session was actually granted. - - A server may ARP for the leased address before it unicasts a renewal - reply. Answering here keeps the lease testable without ever putting the - address on the host. - """ - arp = frame["ARP"] - if int(arp.op) != 1 or arp.pdst not in addresses: - return - reply = ( - self.scapy["Ether"](src=session.mac, dst=frame["Ether"].src) / - self.scapy["ARP"](op=2, hwsrc=session.mac, psrc=arp.pdst, - hwdst=arp.hwsrc, pdst=arp.psrc) - ) - self.send(reply) - - -# --------------------------------------------------------------------------- -# running - - -def run(scenarios, reporter, run_options): - """Run every scenario and return a process exit code.""" - # A host with no scapy or no root cannot test anything. Reporting that as a - # run of skipped tests would be a green result that proves nothing, so it - # is raised before a single test point is emitted. - _scapy() - if os.geteuid() != 0: - raise EnvironmentError_("raw sockets need root; re-run with sudo") - - reporter.start() - for scenario in scenarios: - interface = run_options.interface or scenario.interface - if not interface: - _skip(reporter, scenario, "no interface; pass -i") - continue - # An absent interface is a property of this host, not of the run: a - # shared .conf may name a NIC that only some machines have. - if not netutil.interface_exists(interface): - _skip(reporter, scenario, "no such interface: %s" % (interface,)) - continue - wire = Wire(interface, run_options.pcap, run_options.verbose) - try: - run_scenario(scenario, reporter, wire, run_options) - finally: - wire.close() - reporter.finish() - return reporter.exit_code() - - -def _skip(reporter, scenario, reason): - for step in scenario.steps: - for assertion in step.assertions: - reporter.add(report.Record( - scenario.name, step.name, assertion.render(), True, - skip_reason=reason)) - - -def run_scenario(scenario, reporter, wire, run_options): - interface = run_options.interface or scenario.interface - mac = run_options.mac - if mac in (None, "", "random"): - mac = netutil.random_mac() - elif mac == "iface": - mac = netutil.interface_mac(interface) - else: - mac = netutil.normalise_mac(mac) - - session = Session(mac, interface) - - for step in scenario.steps: - run_step(scenario, step, session, reporter, wire, run_options) - - -def run_step(scenario, step, session, reporter, wire, run_options): - context = session.context - timeout = run_options.timeout or step.timeout - retries = run_options.retries or step.retries - - if step.type == "sleep": - time.sleep(float(subst.resolve(step.param("duration", "1"), context))) - return - - if step.type == "noop": - _record(reporter, scenario, step, step.assertions, None, context, "", 0, 0) - return - - if step.param("mac"): - session.mac = netutil.normalise_mac(subst.resolve(step.param("mac"), context)) - session.mac_bytes = netutil.mac_bytes(session.mac) - - transition = machine.transition_for(session.state, step.type) - if transition is None: - raise ConfigError("%s/%s: %r is not legal in state %s" - % (scenario.name, step.name, step.type, session.state)) - - msgtype = _message_type(step.type) - wanted = machine.expected_types(step, session.state) - session.xid = netutil.random_xid() - - replies = [] - attempt = 0 - scapy = wire.scapy - sent = "" - for attempt in range(1, retries + 1): - frame = build_packet(step, session, context, msgtype, scapy) - sent = describe_sent(step, session, context, msgtype) - if run_options.verbose: - print("# send %s" % (sent,)) - wire.send(frame) - deadline = time.monotonic() + timeout - replies = wire.collect(session, deadline, arp_for=session.leases) - if _pick(replies, wanted) is not None: - break - if not wanted and replies: - break - - chosen = _pick(replies, wanted) - if chosen is None and replies: - chosen = replies[0] # wrong type: report it, do not time out - - offers = _distinct(r for r in replies if r.msgtype == "OFFER") - if chosen is not None and chosen.msgtype == "OFFER": - chosen = _select_offer(offers, step, context) or chosen - - extras = {"offers": len(offers), "attempt": attempt} - if chosen is not None: - chosen.attempt = attempt - context.bind(step.name, chosen) - context.bind("reply", chosen) - for name in transition.binds: - if name == "offer" and chosen.msgtype != "OFFER": - continue - if name in ("ack", "lease") and chosen.msgtype != "ACK": - continue - context.bind(name, chosen) - if chosen.msgtype == "ACK" and chosen.yiaddr != "0.0.0.0": - session.leases.add(chosen.yiaddr) - - _record(reporter, scenario, step, step.assertions, chosen, context, sent, - attempt, retries, extras) - for offer in offers: - _record(reporter, scenario, step, step.assertions_all, offer, context, - sent, attempt, retries, extras, suffix=" [offer %s]" % offer.src_ip) - - expect = step.expect or transition.default_expect - # Silence after a step that expected an answer leaves the client where it - # was; silence after a RELEASE or a DECLINE, which are never answered, is - # the step working as intended. machine.validate_scenario draws the same - # line, so what a run does matches what validate accepted. - if transition.expects and (expect == "none" or chosen is None): - return - if chosen is not None and chosen.msgtype == "NAK" and transition.nak_state: - session.state = transition.nak_state - elif transition.next_state: - session.state = transition.next_state - - -def _distinct(replies): - """One entry per answering server. - - Retransmits keep the same xid, as RFC 2131 requires, so an answer to an - earlier attempt still matches a later one and the same server can be heard - twice. `offers` is meant to count servers, not packets, so a second answer - from a server already heard from is dropped. - """ - seen = {} - for reply in replies: - key = reply.options.get(54) or reply.src_ip or reply.src_mac - if key not in seen: - seen[key] = reply - return list(seen.values()) - - -def _record(reporter, scenario, step, assertion_list, reply, context, sent, - attempt, attempts, extras=None, suffix=""): - for assertion in assertion_list: - result = assertions_mod.evaluate(assertion, reply, context, extras) - reporter.add(report.Record( - scenario.name, step.name, assertion.render() + suffix, - result.ok, expected=result.expected, actual=result.actual, - detail=result.detail, reply=reply, sent=sent, - attempt=attempt, attempts=attempts)) - - -def _message_type(step_type): - return { - "discover": "discover", "request": "request", "renew": "request", - "rebind": "request", "release": "release", "decline": "decline", - "inform": "inform", "bootrequest": "bootrequest", - }[step_type] - - -def _pick(replies, wanted): - if not wanted: - return None - for reply in replies: - if reply.msgtype.lower() in set(w.lower() for w in wanted): - return reply - return None - - -def _select_offer(offers, step, context): - """Which OFFER drives the following REQUEST.""" - if not offers: - return None - rule = subst.resolve(step.param("select", "first"), context) - if rule == "first": - return offers[0] - if rule == "with-bootfile": - for offer in offers: - if offer.file: - return offer - return offers[0] - if rule.startswith("server-id:"): - wanted = rule.split(":", 1)[1].strip() - for offer in offers: - if offer.options.get(54) == wanted: - return offer - return None - raise ConfigError("unknown select=%r" % (rule,)) - - -def ad_hoc_discover(interface, client_arch=None, vendor_class=None, - user_class=None, request_options=None): - """A one-step scenario for `dhcptest discover`, built without a .conf.""" - from .model import Scenario, Step - from . import assertions as assertions_mod - - params = {} - if client_arch: - params["client_arch"] = client_arch - if vendor_class: - params["vendor_class"] = vendor_class - if user_class: - params["user_class"] = user_class - params["request_options"] = request_options or "1,3,6,15,28,51,54,66,67" - - step = Step(name="discover", type="discover", expect="offer", params=params, - assertions=assertions_mod.parse_block( - "msgtype == OFFER\nyiaddr present", ""), - source="") - return Scenario(name="discover", description="ad-hoc DISCOVER", - interface=interface, steps=[step], source="") diff --git a/xCAT-test/dhcptest/src/dhcptest_lib/subst.py b/xCAT-test/dhcptest/src/dhcptest_lib/subst.py deleted file mode 100644 index e7c01cade..000000000 --- a/xCAT-test/dhcptest/src/dhcptest_lib/subst.py +++ /dev/null @@ -1,109 +0,0 @@ -"""Runtime references to earlier replies. - -Variables are configparser's job: `%(name)s` is interpolated by -`BasicInterpolation` out of the `[vars]` section and `--set`, before a value -ever reaches this module. - -What configparser cannot do is `$offer.address`, because that value is a reply -that has not arrived when the file is read. So exactly one form is resolved -here, at step-execution time, when the earlier replies exist: - - $step.field a field of an earlier step's reply - -`BasicInterpolation` gives `$` no meaning of its own, which is why the two -schemes can share a file without escaping. -""" - -import re - -from .errors import ConfigError -from .model import Reply - -REF_RE = re.compile(r"\$([A-Za-z_][A-Za-z0-9_-]*)\.([A-Za-z_][A-Za-z0-9_]*)") - -#: configparser's variable syntax, matched only to report what a file needs. -VAR_RE = re.compile(r"%\(([^)]+)\)s") - -#: Bindings the tool maintains itself, in addition to one per named step. -ALIASES = ("offer", "ack", "nak", "lease", "reply") - - -class Context(object): - """Everything a `$reference` can resolve against, at one point in a run. - - Only replies: variables were interpolated by configparser at load time and - are gone by the time a step runs. - """ - - def __init__(self): - self.bindings = {} - - def bind(self, name, reply): - self.bindings[name] = reply - - def get_binding(self, name): - if name not in self.bindings: - raise ConfigError( - "$%s is not available here; nothing has bound it yet" % (name,)) - return self.bindings[name] - - def __repr__(self): - return "Context(bindings=%r)" % (sorted(self.bindings),) - - -def references(text): - """Every `$step.field` in a value, as `[(step, field), ...]`. - - Static analysis: `validate` uses this to reject a typo like - `$offer.addres` without sending a single packet. - """ - if text is None: - return [] - return REF_RE.findall(str(text)) - - -def variables(text): - """Every `%(name)s` in a value. - - Only used to tell an operator what a file needs. Resolving them is - configparser's job, not ours. - """ - if text is None: - return set() - return set(VAR_RE.findall(str(text))) - - -def resolve(text, context): - """Substitute every `$step.field` in `text`.""" - if text is None: - return None - - def replace(match): - name, field = match.group(1), match.group(2) - reply = context.get_binding(name) - try: - return format_value(reply.field(field)) - except KeyError as exc: - raise ConfigError(str(exc)) - - return REF_RE.sub(replace, str(text)) - - -def format_value(value): - """Render a resolved value the way a .conf author would have typed it.""" - if isinstance(value, bool): - return "yes" if value else "no" - if isinstance(value, (list, tuple)): - return ", ".join(format_value(item) for item in value) - return str(value) - - -def check_field(field): - """True when `field` is a name `Reply.field()` could ever resolve.""" - if field.startswith("opt") and field[3:]: - try: - int(field[3:], 0) - except ValueError: - return False - return True - return field in Reply.known_fields() diff --git a/xCAT-test/dhcptest/tests/helper.py b/xCAT-test/dhcptest/tests/helper.py deleted file mode 100644 index 4f5e3502a..000000000 --- a/xCAT-test/dhcptest/tests/helper.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Put src/ on sys.path so the tests run straight from a checkout.""" - -import os -import sys - -HERE = os.path.dirname(os.path.abspath(__file__)) -ROOT = os.path.dirname(HERE) -SRC = os.path.join(ROOT, "src") -CONF = os.path.join(ROOT, "conf") - -if SRC not in sys.path: - sys.path.insert(0, SRC) diff --git a/xCAT-test/dhcptest/tests/test_assertions.py b/xCAT-test/dhcptest/tests/test_assertions.py deleted file mode 100644 index 4c6dbd64b..000000000 --- a/xCAT-test/dhcptest/tests/test_assertions.py +++ /dev/null @@ -1,197 +0,0 @@ -import unittest - -import helper # noqa: F401 - -from dhcptest_lib import assertions -from dhcptest_lib.errors import ConfigError -from dhcptest_lib.model import Reply -from dhcptest_lib.subst import Context - - -#: Distinguishes "caller passed no reply argument" from "there was no reply". -NOTHING = object() - - -def offer(**kwargs): - fields = dict( - msgtype="OFFER", - yiaddr="10.0.0.101", - siaddr="10.0.0.1", - file="xcat/xnba.kpxe", - options={54: "10.0.0.1", 51: 43200, 1: "255.255.255.0", 60: "PXEClient"}, - ) - fields.update(kwargs) - return Reply(**fields) - - -class Parsing(unittest.TestCase): - - def test_parses_the_three_shapes(self): - self.assertEqual(assertions.parse("msgtype == OFFER").op, "==") - self.assertEqual(assertions.parse("option:54 present").value, None) - self.assertEqual(assertions.parse("yiaddr in 10.0.0.0/24").value, - "10.0.0.0/24") - - def test_a_value_may_contain_spaces(self): - parsed = assertions.parse("file matches ^http://.* end$") - self.assertEqual(parsed.value, "^http://.* end$") - - def test_empty_value_means_non_empty(self): - self.assertEqual(assertions.parse("file !=").value, "") - - def test_rejects_unknown_operator_target_and_regex(self): - self.assertRaises(ConfigError, assertions.parse, "msgtype === OFFER") - self.assertRaises(ConfigError, assertions.parse, "nonsense == 1") - self.assertRaises(ConfigError, assertions.parse, "file matches (") - self.assertRaises(ConfigError, assertions.parse, "option:54 present now") - self.assertRaises(ConfigError, assertions.parse, "msgtype") - - def test_parse_block_skips_blanks_and_comments(self): - block = "\nmsgtype == OFFER\n# a comment\n\noffers == 1\n" - self.assertEqual(len(assertions.parse_block(block)), 2) - - def test_reference_targets_are_checked(self): - assertions.parse("$offer.address in 10.0.0.0/24") - self.assertRaises(ConfigError, assertions.parse, - "$offer.addres in 10.0.0.0/24") - - -class Evaluation(unittest.TestCase): - - def setUp(self): - self.context = Context() - - def check(self, line, reply=NOTHING, extras=None): - return assertions.evaluate(assertions.parse(line), - offer() if reply is NOTHING else reply, - self.context, extras) - - def test_message_type_and_header_fields(self): - self.assertTrue(self.check("msgtype == OFFER").ok) - self.assertTrue(self.check("msgtype != ACK").ok) - self.assertTrue(self.check("siaddr == 10.0.0.1").ok) - self.assertFalse(self.check("siaddr == 10.0.0.2").ok) - - def test_a_reply_naming_no_boot_file_is_absent_not_empty(self): - # A server that hands out no boot file sends an empty `file` header and - # no option 67. "Nothing to fetch" is therefore written `bootfile - # absent`: a regex against an empty string can never hold, so a - # scenario written that way would fail whatever the server did. - nothing = offer(file="", options={54: "10.0.0.1"}) - self.assertTrue(self.check("bootfile absent", nothing).ok) - self.assertFalse(self.check("bootfile present", nothing).ok) - self.assertFalse(self.check("bootfile matches ^$", nothing).ok) - self.assertTrue(self.check("bootfile present").ok) - - def test_a_negative_comparison_holds_against_a_target_that_is_absent(self): - # "is not the node's install script" is satisfied by a reply that names - # no boot file at all -- which is the strongest way a server can - # satisfy it. The same reading applies to `not-in`. Everything else - # still fails on an absent target, because there is nothing to compare. - nothing = offer(file="", options={54: "10.0.0.1"}) - self.assertTrue(self.check("bootfile != pxelinux.0", nothing).ok) - self.assertTrue(self.check("option:67 != pxelinux.0", nothing).ok) - self.assertTrue(self.check("option:66 not-in 10.0.0.0/24", nothing).ok) - self.assertFalse(self.check("bootfile == pxelinux.0", nothing).ok) - self.assertFalse(self.check("bootfile starts-with http://", nothing).ok) - - def test_options_by_number_and_name(self): - self.assertTrue(self.check("option:54 == 10.0.0.1").ok) - self.assertTrue(self.check("option:server_id == 10.0.0.1").ok) - self.assertTrue(self.check("51 == 43200").ok) - - def test_presence(self): - self.assertTrue(self.check("option:54 present").ok) - self.assertTrue(self.check("option:66 absent").ok) - self.assertFalse(self.check("option:66 present").ok) - - def test_subnet_membership(self): - self.assertTrue(self.check("yiaddr in 10.0.0.0/24").ok) - self.assertFalse(self.check("yiaddr in 192.168.0.0/24").ok) - self.assertTrue(self.check("yiaddr not-in 192.168.0.0/24").ok) - - def test_list_membership(self): - self.assertTrue(self.check("siaddr in 10.0.0.1, 10.0.0.2").ok) - self.assertFalse(self.check("siaddr in 10.0.0.3, 10.0.0.2").ok) - - def test_range_membership(self): - # A dynamic pool is written as two addresses, not as a CIDR: it rarely - # lines up on a prefix boundary. - self.assertTrue(self.check("yiaddr in 10.0.0.100-10.0.0.200").ok) - self.assertFalse(self.check("yiaddr in 10.0.0.102-10.0.0.200").ok) - self.assertTrue(self.check("yiaddr not-in 10.0.0.1-10.0.0.50").ok) - # The bounds are inclusive. - self.assertTrue(self.check("yiaddr in 10.0.0.101-10.0.0.101").ok) - # A value with a hyphen that is not two addresses is still a list entry. - self.assertTrue(self.check("option:60 in PXEClient, some-other").ok) - - def test_string_operators(self): - self.assertTrue(self.check("file starts-with xcat/").ok) - self.assertTrue(self.check("file ends-with .kpxe").ok) - self.assertTrue(self.check("file contains xnba").ok) - self.assertTrue(self.check("file matches ^xcat/.*\\.kpxe$").ok) - - def test_numeric_operators(self): - self.assertTrue(self.check("option:51 > 600").ok) - self.assertFalse(self.check("option:51 < 600").ok) - - def test_missing_field_fails_with_an_explanation(self): - result = self.check("option:67 == pxelinux.0") - self.assertFalse(result.ok) - self.assertIn("not present", result.detail) - - def test_offers_count_comes_from_extras(self): - self.assertTrue(self.check("offers == 1", extras={"offers": 1}).ok) - self.assertFalse(self.check("offers == 1", extras={"offers": 2}).ok) - - def test_a_failure_reports_expected_and_received(self): - result = self.check("file == pxelinux.0") - self.assertFalse(result.ok) - self.assertEqual(result.expected, "pxelinux.0") - self.assertEqual(result.actual, "xcat/xnba.kpxe") - - def test_no_reply_is_a_failure_not_a_crash(self): - result = self.check("msgtype == OFFER", reply=None) - self.assertFalse(result.ok) - - -class FqdnFlagsTarget(unittest.TestCase): - """`fqdn_flags` is a target of its own, because option 81 holds two facts.""" - - def _reply(self, raw): - from dhcptest_lib import options - blob = options.pack_options([(53, b"\x02"), (81, raw)]) - decoded = dict((code, options.decode_value(code, value)) - for code, value in options.unpack_options(blob).items()) - return Reply(msgtype="OFFER", options=decoded, raw_options=blob) - - def test_the_flags_and_the_name_are_asserted_separately(self): - reply = self._reply(b"\x03\x00\x00node01.cluster.local.") - self.assertTrue(self._ok("fqdn_flags == SO", reply)) - self.assertTrue(self._ok("option:81 == node01.cluster.local.", reply)) - - def test_the_dns_name_prefers_option_81_and_falls_back_to_12(self): - # The two servers put the name in different options. A scenario about - # the name asserts on one target and holds for both. - both = self._reply(b"\x03\x00\x00node01.cluster.local.") - both.options[12] = "node01" - self.assertEqual(both.dns_name(), "node01.cluster.local.") - only12 = Reply(msgtype="OFFER", options={12: "node01"}) - self.assertEqual(only12.dns_name(), "node01") - self.assertTrue(self._ok("dns_name matches ^node01", only12)) - - def test_a_reply_naming_the_node_nowhere_has_no_dns_name(self): - self.assertTrue( - self._ok("dns_name absent", Reply(msgtype="OFFER", options={53: 2}))) - - def test_a_reply_with_no_option_81_has_no_flags(self): - reply = Reply(msgtype="OFFER", options={53: 2}) - self.assertEqual(reply.fqdn_flags(), "") - self.assertTrue(self._ok("fqdn_flags absent", reply)) - - def _ok(self, text, reply): - return assertions.evaluate(assertions.parse(text), reply, {}).ok - - -if __name__ == "__main__": - unittest.main() diff --git a/xCAT-test/dhcptest/tests/test_boundaries.py b/xCAT-test/dhcptest/tests/test_boundaries.py deleted file mode 100644 index e0ceef1d0..000000000 --- a/xCAT-test/dhcptest/tests/test_boundaries.py +++ /dev/null @@ -1,150 +0,0 @@ -"""The two boundaries this tool promises to keep. - -Both are checked by reading the source rather than by convention, because both -are easy to break with one convenient import and neither breaks visibly. -""" - -import ast -import os -import unittest - -import helper - -LIB = os.path.join(helper.SRC, "dhcptest_lib") - -#: The one module allowed to touch scapy. -SCAPY_MODULE = "runner.py" - -#: Modules that must import on a host with no scapy and no privileges, because -#: `dhcptest validate` and `dhcptest list` run in CI on exactly such a host. -OFFLINE_MODULES = [ - "assertions", "cli", "config", "errors", "machine", "model", "netutil", - "options", "report", "subst", -] - - -def sources(): - for name in sorted(os.listdir(LIB)): - if name.endswith(".py"): - path = os.path.join(LIB, name) - with open(path) as handle: - yield name, ast.parse(handle.read(), filename=path) - - -def imported_roots(tree): - roots = set() - for node in ast.walk(tree): - if isinstance(node, ast.Import): - roots.update(alias.name.split(".")[0] for alias in node.names) - elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module: - roots.add(node.module.split(".")[0]) - return roots - - -def strings(tree): - """Every string literal that is not a docstring. - - Docstrings are prose about the design, and prose is allowed to explain what - the tool deliberately does not do. Only strings the code actually uses are - evidence of coupling. - """ - docstrings = set() - for node in ast.walk(tree): - if isinstance(node, ast.Expr) and isinstance(node.value, ast.Constant) \ - and isinstance(node.value.value, str): - docstrings.add(id(node.value)) - for node in ast.walk(tree): - if isinstance(node, ast.Constant) and isinstance(node.value, str) \ - and id(node) not in docstrings: - yield node.value - - -class Layering(unittest.TestCase): - """Only runner.py may import scapy.""" - - def test_scapy_is_confined_to_one_module(self): - for name, tree in sources(): - if name == SCAPY_MODULE: - continue - self.assertNotIn("scapy", imported_roots(tree), - "%s imports scapy; only %s may" - % (name, SCAPY_MODULE)) - - def test_the_offline_modules_import_without_scapy(self): - # They are already imported by the rest of the suite, but importing - # them here states the requirement rather than relying on it. - for name in OFFLINE_MODULES: - __import__("dhcptest_lib." + name) - - def test_runner_is_imported_lazily(self): - # cli.py must not pull scapy in at start-up, or `validate` would need it. - for name, tree in sources(): - if name not in ("cli.py", "config.py", "machine.py"): - continue - for node in ast.walk(tree): - if isinstance(node, ast.ImportFrom) and node.col_offset == 0: - names = [alias.name for alias in node.names] - self.assertNotIn("runner", names, - "%s imports runner at module level" % (name,)) - - -class NoXcatCoupling(unittest.TestCase): - """The tool talks to a DHCP server over the wire and to nothing else. - - No xCAT database, no xCAT commands, no xCAT paths -- so it can be pointed - at any DHCP server, and so a change inside xCAT cannot silently change what - a test means. - """ - - #: Anything that would mean shelling out or reading a local database. - FORBIDDEN_IMPORTS = frozenset(["subprocess", "sqlite3", "xcat", "xCAT"]) - - #: Substrings that would mean the tool knows what is answering it. Checked - #: against string literals only, so the AST has already dropped comments. - FORBIDDEN_TEXT = ("xcat", "makedhcp", "lsdef", "tabdump", "nodels", - "dhcpd.conf", "kea", "dnsmasq") - - def test_no_module_shells_out_or_opens_a_database(self): - for name, tree in sources(): - clash = imported_roots(tree) & self.FORBIDDEN_IMPORTS - self.assertFalse(clash, "%s imports %s" % (name, sorted(clash))) - - def test_no_module_names_a_server_implementation_or_xcat(self): - for name, tree in sources(): - for text in strings(tree): - lowered = text.lower() - for bad in self.FORBIDDEN_TEXT: - self.assertNotIn( - bad, lowered, - "%s has a string mentioning %r: %r" % (name, bad, text)) - - def test_no_module_calls_os_system_or_popen(self): - for name, tree in sources(): - for node in ast.walk(tree): - if not isinstance(node, ast.Attribute): - continue - self.assertNotIn( - node.attr, ("system", "popen", "spawnv", "execv"), - "%s calls os.%s" % (name, node.attr)) - - def test_the_shipped_scenarios_name_no_implementation(self): - # A .conf may state what an address or a loader must be -- that is the - # operator's knowledge -- but never who is meant to be serving it. - # - # Comments are skipped: naming two servers to explain how they differ - # on the wire is exactly the prose a reader needs. What must stay - # implementation-free is the part that decides whether a test passes. - for name in sorted(os.listdir(helper.CONF)): - if not name.endswith(".conf"): - continue - with open(os.path.join(helper.CONF, name)) as handle: - text = "\n".join( - line for line in handle.read().lower().splitlines() - if not line.lstrip().startswith("#")) - for bad in ("makedhcp", "lsdef", "tabdump", "dhcpd.conf", - "kea-dhcp4", "dnsmasq"): - self.assertNotIn(bad, text, "%s mentions %r" % (name, bad)) - - -if __name__ == "__main__": - unittest.main() diff --git a/xCAT-test/dhcptest/tests/test_config.py b/xCAT-test/dhcptest/tests/test_config.py deleted file mode 100644 index b763d2aa8..000000000 --- a/xCAT-test/dhcptest/tests/test_config.py +++ /dev/null @@ -1,166 +0,0 @@ -import os -import tempfile -import unittest - -import helper - -from dhcptest_lib import config -from dhcptest_lib.errors import ConfigError - -BASIC = """ -[vars] -net = 10.0.0.0/24 - -[defaults] -interface = eth0 -retries = 2 - -[scenario basic] -description = one discover - -[step d] -type = discover -expect = offer -assert = - msgtype == OFFER - yiaddr in %(net)s -""" - - -class Loading(unittest.TestCase): - - def setUp(self): - self.dir = tempfile.mkdtemp(prefix="dhcptest-") - - def write(self, text, name="t.conf"): - path = os.path.join(self.dir, name) - with open(path, "w") as handle: - handle.write(text) - return path - - def load(self, text, **kwargs): - return config.load([self.write(text)], **kwargs) - - def test_sections_become_scenarios_steps_and_vars(self): - scenarios, variables = self.load(BASIC) - self.assertEqual(len(scenarios), 1) - self.assertEqual(variables["net"], "10.0.0.0/24") - scenario = scenarios[0] - self.assertEqual(scenario.name, "basic") - self.assertEqual(scenario.interface, "eth0") - self.assertEqual(len(scenario.steps), 1) - self.assertEqual(len(scenario.step("d").assertions), 2) - - def test_defaults_apply_and_a_step_overrides_them(self): - scenarios, _ = self.load(BASIC) - self.assertEqual(scenarios[0].step("d").retries, 2) - scenarios, _ = self.load(BASIC.replace( - "type = discover", "type = discover\nretries = 7")) - self.assertEqual(scenarios[0].step("d").retries, 7) - - def test_command_line_beats_the_file(self): - scenarios, variables = self.load( - BASIC, overrides={"net": "192.168.0.0/24"}, interface="eth9") - self.assertEqual(variables["net"], "192.168.0.0/24") - self.assertEqual(scenarios[0].interface, "eth9") - - def test_a_literal_mac_is_checked_at_load_time(self): - self.assertRaises(ConfigError, self.load, - BASIC.replace("expect = offer", - "expect = offer\nmac = nonsense")) - - def test_a_mac_carrying_a_reference_is_left_for_runtime(self): - # $offer.chaddr is a reply that has not arrived yet, so it cannot be - # checked as a MAC until the step runs. - scenarios, _ = self.load(BASIC.replace( - "expect = offer", "expect = offer\nmac = $offer.chaddr")) - self.assertEqual(scenarios[0].step("d").param("mac"), "$offer.chaddr") - - def test_a_variable_is_interpolated_by_configparser(self): - scenarios, _ = self.load(BASIC.replace( - "net = 10.0.0.0/24", "net = 10.0.0.0/24\nwho = 02:00:de:ad:be:ef") - .replace("expect = offer", "expect = offer\nmac = %(who)s")) - self.assertEqual(scenarios[0].step("d").param("mac"), - "02:00:de:ad:be:ef") - self.assertEqual(scenarios[0].step("d").assertions[1].value, - "10.0.0.0/24") - - def test_a_variable_may_share_a_name_with_a_step_key(self): - # A variable is only ever read through %(...)s. Letting the two share a - # namespace made `--set user_class=xNBA` delete the step's user_class. - scenarios, _ = self.load( - BASIC.replace("expect = offer", - "expect = offer\nuser_class = %(user_class)s"), - overrides={"net": "10.0.0.0/24", "user_class": "xNBA"}) - self.assertEqual(scenarios[0].step("d").param("user_class"), "xNBA") - - def test_an_undefined_variable_names_the_flag_that_fixes_it(self): - with self.assertRaises(ConfigError) as caught: - self.load(BASIC.replace("%(net)s", "%(nowhere)s")) - self.assertIn("--set nowhere=", str(caught.exception)) - - def test_raw_loading_leaves_the_variables_in_place(self): - # What `validate` and `list` do: check the shape of a file that has - # not been given its values yet. - scenarios, _ = self.load(BASIC.replace("%(net)s", "%(nowhere)s"), - raw=True) - self.assertEqual(scenarios[0].step("d").assertions[1].value, - "%(nowhere)s") - - def test_unknown_keys_and_sections_are_hard_errors(self): - self.assertRaises(ConfigError, self.load, - BASIC.replace("expect = offer", "expct = offer")) - self.assertRaises(ConfigError, self.load, BASIC + "\n[nonsense]\na = b\n") - - def test_a_step_needs_a_type_and_a_scenario(self): - self.assertRaises(ConfigError, self.load, - BASIC.replace("type = discover", "timeout = 1")) - self.assertRaises(ConfigError, self.load, - "[step orphan]\ntype = discover\n") - - def test_an_unknown_step_type_is_rejected(self): - self.assertRaises(ConfigError, self.load, - BASIC.replace("type = discover", "type = telepathy")) - - def test_a_scenario_needs_steps_and_a_file_needs_a_scenario(self): - self.assertRaises(ConfigError, self.load, "[scenario empty]\n") - self.assertRaises(ConfigError, self.load, "[vars]\na = b\n") - - def test_a_scenario_name_may_not_be_reused_across_files(self): - first = self.write(BASIC, "a.conf") - second = self.write(BASIC, "b.conf") - self.assertRaises(ConfigError, config.load, [first, second]) - - def test_a_missing_file_names_itself(self): - with self.assertRaises(ConfigError) as caught: - config.load([os.path.join(self.dir, "absent.conf")]) - self.assertIn("absent.conf", str(caught.exception)) - - def test_numbers_must_be_numbers(self): - self.assertRaises(ConfigError, self.load, - BASIC.replace("retries = 2", "retries = soon")) - self.assertRaises(ConfigError, self.load, - BASIC.replace("retries = 2", "timeout = soon")) - - def test_a_semicolon_starts_an_inline_comment(self): - scenarios, _ = self.load( - BASIC.replace("type = discover", "type = discover ; the only one")) - self.assertEqual(scenarios[0].step("d").type, "discover") - - -class ShippedFiles(unittest.TestCase): - """Every .conf in conf/ must load and be internally consistent.""" - - def test_all_shipped_conf_files_load(self): - # raw=True is what `validate` does: a shipped file states which - # variables it needs, and only a run supplies them. - names = sorted(n for n in os.listdir(helper.CONF) if n.endswith(".conf")) - self.assertTrue(names, "conf/ has no .conf files") - for name in names: - scenarios, _ = config.load([os.path.join(helper.CONF, name)], - raw=True) - self.assertTrue(scenarios, "%s defines no scenario" % (name,)) - - -if __name__ == "__main__": - unittest.main() diff --git a/xCAT-test/dhcptest/tests/test_machine.py b/xCAT-test/dhcptest/tests/test_machine.py deleted file mode 100644 index 4a02d8fd1..000000000 --- a/xCAT-test/dhcptest/tests/test_machine.py +++ /dev/null @@ -1,152 +0,0 @@ -import os -import unittest - -import helper - -from dhcptest_lib import config, machine -from dhcptest_lib.model import Scenario, Step - - -def step(name, type, expect=None, params=None, assertions=()): - return Step(name=name, type=type, expect=expect, params=params or {}, - assertions=assertions) - - -def scenario(*steps): - return Scenario(name="s", steps=steps) - - -class Table(unittest.TestCase): - - def test_known_step_types_covers_passive_and_protocol_types(self): - types = machine.known_step_types() - for expected in ("discover", "request", "renew", "rebind", "release", - "decline", "inform", "noop", "sleep"): - self.assertIn(expected, types) - - def test_inform_is_legal_from_any_state(self): - self.assertIsNotNone(machine.transition_for(machine.INIT, "inform")) - self.assertIsNotNone(machine.transition_for(machine.BOUND, "inform")) - - def test_discover_is_not_legal_once_bound(self): - self.assertIsNone(machine.transition_for(machine.BOUND, "discover")) - - def test_a_selecting_request_expects_ack_or_nak(self): - transition = machine.transition_for(machine.SELECTING, "request") - self.assertEqual(transition.expects, frozenset(["ack", "nak"])) - self.assertEqual(transition.default_expect, "ack") - self.assertEqual(transition.next_state, machine.BOUND) - - def test_renew_is_unicast_and_rebind_is_not(self): - self.assertTrue(machine.transition_for(machine.BOUND, "renew").unicast) - self.assertFalse(machine.transition_for(machine.BOUND, "rebind").unicast) - - def test_a_bootrequest_is_a_single_exchange_from_init(self): - transition = machine.transition_for(machine.INIT, "bootrequest") - self.assertEqual(transition.expects, frozenset(["bootreply"])) - self.assertEqual(transition.default_expect, "bootreply") - # No handshake follows a BOOTREPLY, so the session stays where it was - # and a second BOOTREQUEST in the same scenario is still legal. - self.assertEqual(transition.next_state, machine.INIT) - - -class Validation(unittest.TestCase): - - def test_a_clean_lease_scenario_has_no_problems(self): - self.assertEqual(machine.validate_scenario(scenario( - step("d", "discover"), - step("r", "request", params={"requested_address": "$offer.address", - "server_id": "$offer.server_id"}), - )), []) - - def test_a_request_without_a_discover_is_rejected(self): - problems = machine.validate_scenario(scenario(step("r", "request"))) - self.assertEqual(len(problems), 1) - self.assertIn("needs requested_address=", problems[0]) - - def test_a_renew_before_bound_is_rejected(self): - problems = machine.validate_scenario(scenario(step("x", "renew"))) - self.assertIn("not legal in state INIT", problems[0]) - - def test_a_reference_to_a_later_step_is_rejected(self): - problems = machine.validate_scenario(scenario( - step("d", "discover", params={"ciaddr": "$ack.address"}), - )) - self.assertIn("used before anything binds $ack", problems[0]) - - def test_a_misspelt_reply_field_is_rejected(self): - problems = machine.validate_scenario(scenario( - step("d", "discover"), - step("r", "request", params={"requested_address": "$offer.addres"}), - )) - self.assertIn("not a field of a DHCP reply", problems[0]) - - def test_an_impossible_expectation_is_rejected(self): - problems = machine.validate_scenario(scenario( - step("d", "discover", expect="ack"), - )) - self.assertIn("is not a possible reply", problems[0]) - - def test_expect_none_leaves_the_state_alone_and_binds_nothing(self): - problems = machine.validate_scenario(scenario( - step("silent", "discover", expect="none"), - step("again", "discover"), - )) - self.assertEqual(problems, []) - - def test_a_release_gives_the_lease_up_even_though_nothing_answers_it(self): - # RELEASE is never answered, so its expect is "none" -- but unlike a - # DISCOVER nobody replied to, the step did its work, and the client is - # back in INIT and free to discover again. - self.assertEqual(machine.validate_scenario(scenario( - step("d", "discover"), - step("r", "request", params={"requested_address": "$offer.address", - "server_id": "$offer.server_id"}), - step("bye", "release"), - step("again", "discover"), - )), []) - - def test_a_duplicate_step_name_is_rejected(self): - problems = machine.validate_scenario(scenario( - step("d", "discover"), step("d", "discover", expect="none"), - )) - self.assertIn("duplicate step name", problems[0]) - - def test_expected_types_follows_expect(self): - self.assertEqual( - machine.expected_types(step("d", "discover"), machine.INIT), - frozenset(["offer"])) - self.assertEqual( - machine.expected_types(step("d", "discover", expect="none"), - machine.INIT), - frozenset()) - self.assertEqual( - machine.expected_types(step("r", "request", expect="any"), - machine.SELECTING), - frozenset(["ack", "nak"])) - - def test_required_variables_gathers_every_set_name(self): - names = machine.required_variables([scenario( - step("d", "discover", params={"mac": "%(who)s"}, - assertions=(),), - )]) - self.assertEqual(names, {"who"}) - - -class ShippedFiles(unittest.TestCase): - """conf/ must stay valid: this is what `dhcptest validate` runs in CI.""" - - def test_every_shipped_scenario_validates(self): - for name in sorted(os.listdir(helper.CONF)): - if not name.endswith(".conf"): - continue - scenarios, _ = config.load([os.path.join(helper.CONF, name)], - raw=True) - for one in scenarios: - problems = machine.validate_scenario(one) - self.assertEqual(problems, [], - "%s/%s: %s" % (name, one.name, problems)) - - -if __name__ == "__main__": - unittest.main() diff --git a/xCAT-test/dhcptest/tests/test_options.py b/xCAT-test/dhcptest/tests/test_options.py deleted file mode 100644 index 9f3ed6651..000000000 --- a/xCAT-test/dhcptest/tests/test_options.py +++ /dev/null @@ -1,179 +0,0 @@ -import unittest - -import helper # noqa: F401 (sets sys.path) - -from dhcptest_lib import options -from dhcptest_lib.errors import ConfigError - - -class OptionNames(unittest.TestCase): - - def test_resolves_numbers_names_and_prefixed_forms(self): - self.assertEqual(options.option_code(54), 54) - self.assertEqual(options.option_code("54"), 54) - self.assertEqual(options.option_code("0x36"), 54) - self.assertEqual(options.option_code("server_id"), 54) - self.assertEqual(options.option_code("server-id"), 54) - self.assertEqual(options.option_code("option:server_id"), 54) - - def test_rejects_nonsense(self): - self.assertRaises(ConfigError, options.option_code, "not_an_option") - self.assertRaises(ConfigError, options.option_code, "256") - - def test_message_type_round_trip(self): - self.assertEqual(options.msgtype_code("offer"), 2) - self.assertEqual(options.msgtype_name(2), "OFFER") - self.assertEqual(options.msgtype_name(99), "TYPE-99") - - -class ClientFqdn(unittest.TestCase): - """Option 81, RFC 4702: who names the node, and who updates DNS.""" - - def test_a_bare_name_is_sent_with_no_flags(self): - self.assertEqual(options.encode_fqdn("node01"), - b"\x00\x00\x00node01") - - def test_flags_are_taken_from_the_prefix(self): - self.assertEqual(options.encode_fqdn("S:node01")[0], 0x01) - self.assertEqual(options.encode_fqdn("N:node01")[0], 0x08) - self.assertEqual(options.encode_fqdn("SO:node01")[0], 0x03) - - def test_a_name_with_a_colon_is_not_read_as_flags(self): - # The prefix is flags only when every character before the colon is a - # flag letter. Otherwise a name is a name. - flags, name = options.decode_fqdn(options.encode_fqdn("host:1")) - self.assertEqual((flags, name), (0, "host:1")) - - def test_a_prefix_that_is_not_all_flag_letters_is_part_of_the_name(self): - # There is no way to tell a typo'd flag from a name that holds a colon, - # so the rule is mechanical: flags only when every character before the - # first colon is one. A mistyped flag then sends a name no server - # answers with, which fails the step rather than passing it quietly. - flags, name = options.decode_fqdn(options.encode_fqdn("SX:node01")) - self.assertEqual((flags, name), (0, "SX:node01")) - - def test_an_unknown_flag_letter_is_refused_when_it_is_read_as_flags(self): - self.assertRaises(ConfigError, options.fqdn_flag_bits, "SX") - - def test_the_e_flag_sends_the_name_in_wire_format(self): - raw = options.encode_fqdn("SE:node01.cluster.local") - self.assertEqual(raw[0], 0x05) - self.assertEqual(raw[3:], b"\x06node01\x07cluster\x05local\x00") - self.assertEqual(options.decode_fqdn(raw), - (0x05, "node01.cluster.local")) - - def test_the_option_decodes_to_the_name_a_scenario_asserts_on(self): - # The flags are reached through `fqdn_flags`, so the option itself is - # the name and `option:81 == node01` reads as it should. - raw = b"\x03\x00\x00node01.cluster.local." - self.assertEqual(options.decode_value(81, raw), - "node01.cluster.local.") - - def test_flag_letters_round_trip(self): - self.assertEqual(options.fqdn_flag_letters(0x03), "SO") - self.assertEqual(options.fqdn_flag_letters(0), "-") - self.assertEqual(options.fqdn_flag_bits("so"), 0x03) - - def test_a_truncated_option_is_left_as_hex(self): - self.assertIsNone(options.decode_fqdn(b"\x03\x00")) - self.assertEqual(options.decode_value(81, b"\x03\x00"), "03:00") - - -class OptionArea(unittest.TestCase): - - def test_pack_then_unpack(self): - blob = options.pack_options([(53, b"\x02"), (54, b"\x0a\x00\x00\x01")]) - self.assertTrue(blob.endswith(b"\xff")) - found = options.unpack_options(blob) - self.assertEqual(found[53], b"\x02") - self.assertEqual(found[54], b"\x0a\x00\x00\x01") - - def test_unpack_tolerates_magic_padding_and_missing_end(self): - blob = options.MAGIC + b"\x00\x00" + bytes([53, 1, 5]) - self.assertEqual(options.unpack_options(blob), {53: b"\x05"}) - - def test_unpack_concatenates_a_split_option(self): - # RFC 3396: a long option may arrive as several instances. - blob = bytes([60, 2]) + b"PX" + bytes([60, 7]) + b"EClient" - self.assertEqual(options.unpack_options(blob)[60], b"PXEClient") - - def test_decode_value_types(self): - self.assertEqual(options.decode_value(1, b"\xff\xff\xff\x00"), - "255.255.255.0") - self.assertEqual(options.decode_value(51, b"\x00\x00\xa8\xc0"), 43200) - self.assertEqual(options.decode_value(93, b"\x00\x0b"), 11) - self.assertEqual(options.decode_value(60, b"PXEClient"), "PXEClient") - self.assertEqual(options.decode_value(55, b"\x01\x03\x06"), [1, 3, 6]) - # An address-list option is a list even when it holds one address: - # option 3 arriving as raw hex was a real bug. - self.assertEqual(options.decode_value(3, b"\x0a\x63\x00\x01"), - ["10.99.0.1"]) - self.assertEqual( - options.decode_value(6, b"\x0a\x00\x00\x01\x0a\x00\x00\x02"), - ["10.0.0.1", "10.0.0.2"]) - self.assertEqual(options.decode_value(175, b"\xab\xcd"), "ab:cd") - - -class NameLists(unittest.TestCase): - """Option 119, the domain search list: RFC 1035 labels, RFC 3397 pointers.""" - - def test_uncompressed_names(self): - raw = (b"\x03foo\x03com\x00" b"\x03bar\x03com\x00") - self.assertEqual(options.decode_value(119, raw), ["foo.com", "bar.com"]) - - def test_a_compression_pointer_is_followed(self): - # "bar.com" written as the label "bar" plus a pointer to the "com" at - # offset 4 -- which is what a server that compresses actually sends. - raw = b"\x03foo\x03com\x00" + b"\x03bar" + b"\xc0\x04" - self.assertEqual(options.decode_value(119, raw), ["foo.com", "bar.com"]) - - def test_a_pointer_loop_falls_back_to_hex_instead_of_hanging(self): - raw = b"\x03foo" + b"\xc0\x00" - self.assertEqual(options.decode_value(119, raw), "03:66:6f:6f:c0:00") - - def test_a_truncated_name_falls_back_to_hex(self): - raw = b"\x09foo" - self.assertEqual(options.decode_value(119, raw), "09:66:6f:6f") - - -class _FakeBootp(object): - """Enough of a BOOTP layer for `decode`, without needing scapy.""" - - def __init__(self, option_area): - self.payload = option_area - self.xid = 0x1234 - self.yiaddr = "10.0.0.2" - self.siaddr = "10.0.0.1" - self.ciaddr = "0.0.0.0" - self.giaddr = "0.0.0.0" - self.chaddr = b"\x02\x00\x11\x22\x33\x44" + b"\x00" * 10 - self.file = b"pxelinux.0" + b"\x00" * 118 - self.sname = b"\x00" * 64 - self.secs = 0 - self.flags = 0 - - def __getitem__(self, _layer): - return self - - -class MessageTypeOfAReply(unittest.TestCase): - - def _reply(self, option_area): - return options.decode(_FakeBootp(option_area)) - - def test_a_dhcp_reply_is_named_by_option_53(self): - reply = self._reply(options.pack_options([(53, b"\x02")])) - self.assertEqual(reply.msgtype, "OFFER") - - def test_a_reply_with_no_option_53_is_plain_bootp(self): - # Hardware predating DHCP sends a BOOTREQUEST with no option 53, and - # the answer carries none either. Reporting that as a missing message - # type would read as a malformed packet rather than as BOOTP working, - # and there would be nothing for a step to assert on. - reply = self._reply(options.MAGIC + b"\xff") - self.assertEqual(reply.msgtype, "BOOTREPLY") - self.assertEqual(reply.bootfile(), "pxelinux.0") - - -if __name__ == "__main__": - unittest.main() diff --git a/xCAT-test/dhcptest/tests/test_subst.py b/xCAT-test/dhcptest/tests/test_subst.py deleted file mode 100644 index 2439a5318..000000000 --- a/xCAT-test/dhcptest/tests/test_subst.py +++ /dev/null @@ -1,82 +0,0 @@ -import unittest - -import helper # noqa: F401 - -from dhcptest_lib import subst -from dhcptest_lib.errors import ConfigError -from dhcptest_lib.model import Reply - - -class References(unittest.TestCase): - """Static analysis, the part `validate` leans on.""" - - def test_finds_references(self): - self.assertEqual(subst.references("$offer.address and $ack.server_id"), - [("offer", "address"), ("ack", "server_id")]) - - def test_configparser_syntax_is_not_a_reference(self): - # %(net)s is interpolated before a value ever reaches this module. - self.assertEqual(subst.references("%(net)s"), []) - - def test_variables_reports_what_a_raw_file_still_needs(self): - self.assertEqual(subst.variables("%(net)s and %(server)s"), - {"net", "server"}) - self.assertEqual(subst.variables("$offer.address"), set()) - - def test_none_has_no_references(self): - self.assertEqual(subst.references(None), []) - self.assertEqual(subst.variables(None), set()) - - def test_check_field_accepts_names_aliases_and_opt_numbers(self): - self.assertTrue(subst.check_field("yiaddr")) - self.assertTrue(subst.check_field("address")) # alias - self.assertTrue(subst.check_field("server_id")) # option-backed - self.assertTrue(subst.check_field("opt54")) - self.assertTrue(subst.check_field("opt0x36")) - - def test_check_field_rejects_typos(self): - self.assertFalse(subst.check_field("addres")) - self.assertFalse(subst.check_field("optfifty")) - self.assertFalse(subst.check_field("opt")) - - -class Resolution(unittest.TestCase): - - def setUp(self): - self.context = subst.Context() - self.context.bind("offer", Reply( - msgtype="OFFER", yiaddr="10.0.0.101", file="xcat/xnba.kpxe", - options={54: "10.0.0.1", 51: 43200, 55: [1, 3, 6]})) - - def test_a_reference_resolves_to_a_field_of_the_reply(self): - self.assertEqual(subst.resolve("$offer.address", self.context), - "10.0.0.101") - self.assertEqual(subst.resolve("$offer.server_id", self.context), - "10.0.0.1") - - def test_mixed_text_is_substituted_in_place(self): - self.assertEqual( - subst.resolve("host-$offer.address.local", self.context), - "host-10.0.0.101.local") - - def test_configparser_syntax_is_left_alone(self): - # It cannot appear here -- configparser already ate it -- but if it - # does, passing it through beats mangling it. - self.assertEqual(subst.resolve("%(net)s", self.context), "%(net)s") - - def test_unbound_reference_is_an_error(self): - self.assertRaises(ConfigError, subst.resolve, "$ack.address", - self.context) - - def test_reference_to_a_missing_option_is_an_error(self): - self.assertRaises(ConfigError, subst.resolve, "$offer.router", - self.context) - - def test_format_value_renders_lists_and_booleans(self): - self.assertEqual(subst.format_value([1, 3, 6]), "1, 3, 6") - self.assertEqual(subst.format_value(True), "yes") - self.assertEqual(subst.format_value(43200), "43200") - - -if __name__ == "__main__": - unittest.main() diff --git a/xCAT-test/dhcptest/tests/test_wire.py b/xCAT-test/dhcptest/tests/test_wire.py deleted file mode 100644 index 64eb485a7..000000000 --- a/xCAT-test/dhcptest/tests/test_wire.py +++ /dev/null @@ -1,145 +0,0 @@ -"""Build a real frame and read it back. - -This is the one test that needs scapy, so it skips where scapy is absent. It -needs neither root nor a network: the frame is serialised, reparsed and -decoded in memory, which is enough to catch a mis-encoded option or a scapy -API that moved under us. -""" - -import os -import tempfile -import unittest - -import helper - -from dhcptest_lib import config -from dhcptest_lib import options as opt - -try: - from scapy.all import Ether - from dhcptest_lib import runner - HAVE_SCAPY = True -except ImportError: # pragma: no cover - HAVE_SCAPY = False - -CONF = """ -[scenario wire] -description = a PXE discover carrying every option shape - -[step d] -type = discover -mac = 02:00:de:ad:be:ef -vendor_class = PXEClient:Arch:00011:UNDI:003000 -client_arch = 0x000b -user_class = xNBA -user_class_form = rfc3004 -request_options = 1, 3, 6, 43, 54, 60, 66, 67 -max_message_size = 1260 -hostname = probe -fqdn = N:probe.cluster.local -expect = offer -""" - - -@unittest.skipUnless(HAVE_SCAPY, "scapy is not installed") -class RoundTrip(unittest.TestCase): - - def setUp(self): - directory = tempfile.mkdtemp(prefix="dhcptest-") - path = os.path.join(directory, "wire.conf") - with open(path, "w") as handle: - handle.write(CONF) - scenarios, _ = config.load([path]) - self.step = scenarios[0].step("d") - self.session = runner.Session("02:00:de:ad:be:ef", "lo") - self.context = runner.Context() - self.frame = runner.build_packet( - self.step, self.session, self.context, "discover", runner._scapy()) - self.reply = opt.decode(Ether(bytes(self.frame))) - - def test_the_frame_survives_serialisation(self): - self.assertEqual(self.reply.msgtype, "DISCOVER") - self.assertEqual(self.reply.xid, self.session.xid) - self.assertEqual(self.reply.chaddr.lower(), "02:00:de:ad:be:ef") - - def test_it_is_broadcast_from_an_unconfigured_client(self): - frame = self.frame - self.assertEqual(frame["Ether"].dst, "ff:ff:ff:ff:ff:ff") - self.assertEqual(frame["IP"].src, "0.0.0.0") - self.assertEqual(frame["IP"].dst, "255.255.255.255") - self.assertEqual(frame["UDP"].sport, 68) - self.assertEqual(frame["UDP"].dport, 67) - self.assertEqual(int(frame["BOOTP"].flags), 0x8000) - - def test_every_option_shape_encodes_and_decodes(self): - options = self.reply.options - self.assertEqual(options[60], "PXEClient:Arch:00011:UNDI:003000") # text - self.assertEqual(options[93], 11) # u16 - self.assertEqual(options[57], 1260) # u16 - self.assertEqual(options[12], "probe") # text - self.assertEqual(options[55], [1, 3, 6, 43, 54, 60, 66, 67]) # list - self.assertEqual(options[61], "01:02:00:de:ad:be:ef") # hex - self.assertEqual(options[81], "probe.cluster.local") # fqdn - - def test_the_client_fqdn_carries_the_flags_the_step_asked_for(self): - # "N" is a client saying it wants no DNS update at all. A server xCAT - # configures overrides that, so the step has to be able to send it. - raw = opt.unpack_options(bytes(self.frame["BOOTP"].payload))[81] - self.assertEqual(raw[:3], b"\x08\x00\x00") - self.assertEqual(self.reply.fqdn_flags(), "N") - - def test_rfc3004_user_class_is_length_prefixed(self): - raw = opt.unpack_options(bytes(self.frame["BOOTP"].payload))[77] - self.assertEqual(raw, b"\x04xNBA") - - def test_a_raw_user_class_is_not_prefixed(self): - self.step.params["user_class_form"] = "raw" - frame = runner.build_packet(self.step, self.session, self.context, - "discover", runner._scapy()) - raw = opt.unpack_options(bytes(frame["BOOTP"].payload))[77] - self.assertEqual(raw, b"xNBA") - - def test_the_message_is_padded_to_the_minimum_size(self): - # Some relays and older servers drop a short BOOTP frame. - payload = bytes(self.frame["BOOTP"]) - self.assertGreaterEqual(len(payload), 300) - - def test_a_unicast_renew_is_addressed_to_the_server(self): - step = config.load([self._write(""" -[scenario r] -[step renew] -type = renew -ciaddr = 10.0.0.101 -server_id = 10.0.0.1 -""")])[0][0].step("renew") - self.session.state = "BOUND" - frame = runner.build_packet(step, self.session, self.context, "request", - runner._scapy()) - self.assertEqual(frame["IP"].dst, "10.0.0.1") - self.assertEqual(frame["IP"].src, "10.0.0.101") - self.assertEqual(int(frame["BOOTP"].flags), 0) - self.assertEqual(frame["BOOTP"].ciaddr, "10.0.0.101") - - def test_one_server_heard_twice_counts_once(self): - # A retransmit reuses the xid, so an answer to the first attempt still - # matches the second. `offers` counts servers, not packets. - from dhcptest_lib.model import Reply - first = Reply(msgtype="OFFER", yiaddr="10.0.0.101", src_ip="10.0.0.1", - options={54: "10.0.0.1"}) - again = Reply(msgtype="OFFER", yiaddr="10.0.0.101", src_ip="10.0.0.1", - options={54: "10.0.0.1"}) - other = Reply(msgtype="OFFER", yiaddr="10.0.0.7", src_ip="10.0.0.2", - options={54: "10.0.0.2"}) - self.assertEqual(len(runner._distinct([first, again])), 1) - self.assertEqual(len(runner._distinct([first, again, other])), 2) - - def _write(self, text): - directory = tempfile.mkdtemp(prefix="dhcptest-") - path = os.path.join(directory, "t.conf") - with open(path, "w") as handle: - handle.write(text) - return path - - -if __name__ == "__main__": - unittest.main() diff --git a/xCAT-test/provtest/.gitignore b/xCAT-test/provtest/.gitignore deleted file mode 100644 index 9b4490b34..000000000 --- a/xCAT-test/provtest/.gitignore +++ /dev/null @@ -1,6 +0,0 @@ -# Python leaves these next to the source it imports, and provtest is run in -# place -- from a checkout during development and from -# /opt/xcat/share/xcat/tools/autotest after packaging -- so they appear in the -# tree rather than in a build directory. -__pycache__/ -*.pyc diff --git a/xCAT-test/provtest/README.md b/xCAT-test/provtest/README.md deleted file mode 100644 index 9023daddd..000000000 --- a/xCAT-test/provtest/README.md +++ /dev/null @@ -1,337 +0,0 @@ -# provtest - -A wire-level test of the xCAT provision chain. It asks the management node the -questions a booting node asks -- a DNS lookup, a TFTP fetch, an HTTP GET, an -XML request to `xcatd` -- and asserts on the answers that come back. - -Scenarios are `.conf` files, not Perl. The tool knows nothing about xCAT's -database and never runs an xCAT command to decide what to expect: everything it -expects arrives on the command line, from the fixture that set the cluster up. - -Every scenario carries the `@P-nn` tag of the clause it covers, so a failure -names the clause without the specification being at hand. - -## Why - -`ci_test` proves that xCAT's commands accept their arguments and write the rows -they are supposed to write. It does not prove that a node can boot. Between -`nodeset osimage` returning zero and a node reaching its installer there are six -daemons, four file trees and two authentication models, and every one of them -fails silently: a `grub.cfg` named with one wrong hex digit produces no error -anywhere, just a loader prompt nobody is watching. - -The gap this fills is the one between "the database says the node will boot" and -"a client on the provisioning network can fetch what it needs to boot". Only a -client can answer the second question, so provtest is a client. - -`dhcptest` covers DHCP the same way, and provtest deliberately does not: the two -suites meet at one assertion, `@P-70`, where the boot file name DHCP handed out -is fetched over TFTP under exactly that name. - -## Commands - - provtest run [options] CONF [CONF...] # execute scenarios - provtest validate CONF [CONF...] # check offline - provtest list CONF [CONF...] # show what a file does - -`validate` and `list` open no socket, spawn no client and need no root, so they -run in a checkout with no management node anywhere near. They are the check that -catches a misspelt reply field or a reference to a step that has not run yet. - - provtest run --set server=10.99.1.1 --set client=10.99.1.11 \ - --set node=provtestcn --set domain=provtest.cluster \ - --set nodeip=10.99.1.11 --set revname=11.1.99.10 \ - --set alias=provtestcn-alias --set forwarded=example.com \ - --set missing=nosuchnode --set master=10.99.1.1 --set net=10.99.1.0 \ - conf/dns.conf - -Options to `run`: - -| Option | Meaning | -| --- | --- | -| `-b`, `--bind ADDR` | source address for every step that does not set its own | -| `--timeout SECS` | per-attempt timeout, overriding the files | -| `--retries N` | attempts per step, overriding the files | -| `--format tap\|pretty\|json` | output format (default `tap`) | -| `-s`, `--scenario NAME` | run only this scenario; repeatable | -| `-v`, `--verbose` | print each reply as it arrives | - -Exit codes, which are dhcptest's: - -| Code | Meaning | -| --- | --- | -| 0 | every assertion held | -| 1 | at least one assertion failed | -| 2 | the configuration or the command line is wrong | -| 3 | this host cannot run the test at all | - -3 is not 0 on purpose. A host with no `dig` reports that it tested nothing, -because a pass that proves nothing is worse than a failure that says why. - -## Configuration - -INI syntax, read by `configparser`. Three kinds of section: - - [vars] - domain = provtest.cluster - - [defaults] - server = %(server)s - bind = %(client)s - timeout = 8 - - [scenario node-reverse] - description = The node's address resolves to its fully qualified name - - [step ptr] - type = dns - name = %(revname)s.in-addr.arpa - rrtype = PTR - assert = - status == NOERROR - data contains %(node)s.%(domain)s - -`[vars]` and `--set` feed `%(name)s` substitution; `--set` wins. `[defaults]` -supplies any key a later step does not set for itself. A `[step]` belongs to the -`[scenario]` above it, and steps run in file order. - -Section names are unique per file, which `configparser` enforces and not merely -per scenario -- so two scenarios in one file may not both have a `[step config]`. - -## Variables and references - -Two substitutions, resolved at different times. - -`%(name)s` is resolved when the file is loaded, from `[vars]` and `--set`. It is -resolved before `-s` selects anything, so every variable a file mentions must be -supplied even for scenarios that will not run. Each file's header comment lists -the `--set` values it needs, and `validate` prints the union for a set of files. - -`$step.field` is resolved at run time, from the reply of an earlier step in the -same scenario. This is what makes a scenario a chain rather than a list: - - [step config] - type = tftp - path = boot/grub2/grub.cfg-%(hexip)s - - [step kernelpath] - type = extract - from = $config.text - pattern = ^\s*linux(?:efi|16)?\s+/?(?:tftpboot/)?(\S+) - - [step kernel] - type = tftp - path = $kernelpath.value - -Nothing in that scenario says what the kernel is called. The config is fetched, -the name is read out of it, and *that* name is fetched -- which is the difference -between testing xCAT and comparing xCAT's output to a copy of xCAT's output. - -A reference to a step that has not run yet is a configuration error, reported by -`validate`. - -## Assertions - -One per line, under a single multi-line `assert` key, as `target op value`: - - assert = - status == 200 - size > 0 - sha256 == $overtftp.sha256 - text contains destiny=install - error absent - -| Operator | Holds when | -| --- | --- | -| `==`, `!=` | the field equals, or does not equal, the value | -| `<`, `<=`, `>`, `>=` | numeric comparison | -| `in`, `not-in` | the field is, or is not, one of a space-separated list | -| `contains`, `starts-with`, `ends-with` | substring tests | -| `matches` | the field matches the regular expression | -| `present`, `absent` | the field is, or is not, non-empty; no value | - -`not-in` is membership, not negative containment. "This text must not contain -X" is written as an `extract` step asserting `matched == no`. - -A field that is a list is addressed by index (`lines.0`), a field that is a -mapping by key (`header.content-type`, `elements.destiny`). - -Each step also carries `expect`, which says what the transport itself must do: - -| `expect` | Meaning | -| --- | --- | -| `ok` (default) | the request must have got an answer | -| `fail` | the request must not have got one | -| `any` | either; only the assertions decide | - -The default is `ok` so that a step whose server never answered fails even with -no assertions at all. Silence is not a pass. - -## Step keys - -Every step takes `type`, `expect`, `timeout`, `retries`, `assert` and `bind`. - -`bind` is the client's own address, and it is the most important key in the -file. `xcatd` names a client by the reverse lookup of the address the connection -arrived from, so which end of the veth pair a request leaves by decides who the -server thinks is asking. It is never left to the kernel's source selection. - -| `type` | Driven by | Keys | Reply fields | -| --- | --- | --- | --- | -| `dns` | `dig` | `server`, `port`, `name`, `rrtype`, `recursion` | `status`, `flags`, `count`, `data`, `type`, `ttl`, `name`, `question`, `answers`, `authority` | -| `tftp` | `tftp` | `server`, `port`, `path`, `mode` | `ok`, `size`, `sha256`, `text`, `error`, `path` | -| `http` | `curl` | `server`, `port`, `path`, `url`, `method`, `header`, `insecure` | `status`, `size`, `sha256`, `text`, `url`, `header`, `content_type`, `error`, `ok` | -| `xcatreq` | socket | `server`, `port`, `command`, `element`, `raw`, `source_port`, `callback_port`, `callback_listen`, `callback_reply`, `callback_wait`, `cert`, `key` | `destiny`, `kernel`, `initrd`, `kcmdline`, `imgserver`, `name`, `error`, `serverdone`, `elements`, `data`, `text`, `handshake`, `ok`, `callback_seen`, `callback_data` | -| `monitor` | socket | `server`, `port`, `send`, `source_port` | `greeting`, `lines`, `text`, `ok`, `closed`, `error` | -| `flowrequest` | socket | `server`, `port`, `message`, `replies`, `source_port` | `replies`, `count`, `ok`, `error` | -| `findme` | socket | `server`, `port`, `payload`, `encoding`, `source_port`, `callback_listen`, `callback_wait` | `callbacks`, `count`, `ok`, `error` | -| `extract` | -- | `from`, `pattern`, `group` | `value`, `groups`, `matched`, `count` | -| `sleep` | -- | `duration` | -- | -| `noop` | -- | -- | -- | - -`dig`, `curl` and `tftp` are the clients an operator would reach for, which -removes "the test's own protocol parser was wrong" from every result they -produce. The xCAT stages use sockets instead, because binding a source address -and holding a callback listener open while a request is in flight is not -something those programs will do. - -## Shipped scenarios - -| File | Stage | Scenarios | Clauses | -| --- | --- | --- | --- | -| `dns.conf` | DNS | `node-forward`, `node-reverse`, `node-alias`, `forwarded-name`, `local-nxdomain`, `master-resolves` | P-01..P-07 | -| `dns-removal.conf` | DNS | `removed-node` | P-08 | -| `tftp-grub2.conf` | TFTP | `grub2-binary`, `grub2-node-config`, `grub2-config-by-mac`, `grub2-kernel-and-initrd`, `tftp-escape`, `dhcp-named-file-must-exist` | P-09, P-11..P-17, P-24, P-70, P-71 | -| `tftp-pxelinux.conf` | TFTP | `pxelinux-node-config`, `pxelinux-boot-from-disk` | P-18..P-20 | -| `tftp-xnba.conf` | TFTP | `xnba-script`, `xnba-kernel` | P-21, P-22 | -| `tftp-petitboot.conf` | TFTP | `petitboot-config` | P-23 | -| `discovery-artefacts.conf` | TFTP | `grub2-discovery-config`, `pxelinux-discovery-config`, `xnba-discovery-config`, `genesis-images-pxelinux`, `genesis-images-grub2` | P-25..P-31 | -| `http.conf` | HTTP | `install-tree`, `tftp-tree-over-http`, `urls-the-node-was-given`, `boot-images-over-http`, `outside-the-aliases`, `postscripts-listing`, `port-the-node-was-told`, `default-port-the-node-was-told` | P-32..P-39 | -| `flowcontrol.conf` | UDP 3001 | `acknowledged`, `granted` | P-40, P-41 | -| `findme.conf` | UDP 3001 | `findme-callbacks`, `findme-plain-xml`, `findme-unprivileged-port`, `findme-foreign-address` | P-42..P-46, P-48, P-73 | -| `xcatd-destiny.conf` | TLS 3001 | `certless-handshake`, `unknown-client`, `known-node`, `chain-advances` | P-49..P-56 | -| `xcatd-postscript.conf` | TLS 3001, TCP 3002 | `postscript-terminated`, `postscript-unknown-client`, `postscript-both-transports` | P-57, P-58, P-67 | -| `xcatd-credentials.conf` | TLS 3001 | `credentials-granted`, `credentials-refused`, `credentials-unprivileged-callback`, `credentials-nameless` | P-61..P-63, P-76 | -| `xcatd-policy.conf` | TLS 3001 | `command-outside-policy`, `malformed-request` | P-59, P-60 | -| `monitor.conf` | TCP 3002 | `monitor-accepts-status`, `monitor-unknown-client`, `monitor-unknown-verb`, `monitor-is-not-tls` | P-64..P-66, P-68, P-69 | -| `ordering.conf` | whole chain | `unreachable-master`, `missing-ptr`, `state-replaced` | P-72, P-74, P-75 | - -A node has exactly one netboot method, so the four `tftp-*.conf` files are -alternatives, not a set: running all four against one node fails three of them. -The fixture selects the one the node is defined with. - -### What is deliberately not covered - -`getcredentials` has a second form: genesis asks for `x509cert` and encloses a -certificate signing request, and `xcatd` signs it and returns the certificate. -None of these scenarios exercise it. A CSR is not something a wire test can -fabricate meaningfully — a signature over a key that belongs to nothing proves -only that OpenSSL works — and the part of the exchange that decides whether a -node gets a credential at all is the callback on port 300, which is the same for -both forms and is asserted three ways here. A cluster where signing itself is -broken fails at `credentials-granted`; one where the x509 path alone is broken -is not caught, and that is the gap. - -Nothing here installs an operating system either. The install tree the nodes -point at is fabricated, not produced by `copycds`: the scenarios fetch the files -a node fetches and assert what they contain, and the media that would have to be -staged to do more is measured in gigabytes. - -## What it does to the host - -`provtest` itself changes nothing. It opens sockets and spawns `dig`, `curl` and -`tftp`; it writes no file outside a temporary directory it removes, touches no -xCAT table and restarts no daemon. - -What it does need is an address to bind to. Every step binds a source address, -because the address is the credential; that address has to exist on the host -before the run. Two things also need privilege: a `findme` from a source port -below 1024, which `xcatd` requires before it will dispatch the request, and the -callback listener on port 300 that `getcredentials` connects back to. - -Setting those addresses up, defining the node, and putting them all back -afterwards is the fixture's job, not the tool's: -`xCAT-test/autotest/testcase/provtest/provfixture.sh`. - -## Failure output - -TAP 13, so `prove` and the xcattest harness both read it: - - TAP version 13 - 1..6 - ok 1 - node-forward - not ok 2 - node-reverse - --- - scenario: node-reverse - source: conf/dns.conf - step: ptr - sent: dig @10.99.1.1 -b 10.99.1.11 PTR 11.1.99.10.in-addr.arpa - assertion: data contains provtestcn.provtest.cluster - expected: provtestcn.provtest.cluster - actual: (no answer records) - status: NXDOMAIN - ... - -The command that was sent is printed as it was sent, so the first thing an -operator does after a failure -- run it again by hand -- is a copy and a paste. - -## Running under xcattest - -The cases live in `xCAT-test/autotest/testcase/provtest/cases0`. - -Two labels, because they cost different things: - -| Label | What runs | Needs | -| --- | --- | --- | -| `mn_only,ci_test,prov` | `validate`, `list`, and the unit tests | a checkout | -| `mn_only,prov,prov_wire` | the scenarios above | a management node and root | - - xcattest -t ci_test # includes the offline half - xcattest -t prov_wire # the wire half - -Each wire case sets its fixture up, runs one stage, and tears it down from a -`trap` so an interrupted run does not leave a veth pair and half a zone file -behind. - -In CI, `github_action_xcat_test.pl` runs the offline cases with the rest of -`ci_test`, then `prov_wire`, then `dhcp_wire` -- provision chain before DHCP, -because a DHCP failure on a cluster whose TFTP tree is empty is the less -interesting of the two findings. - -## Tests - - python3 -m unittest discover -s tests -v - -Standard library only, no network, no root. They cover the config parser, the -assertion language, substitution, the offline machine, and the decoding of -captured `dig`, `curl` and `tftp` output. - -One of them, `test_boundaries.py`, is a test about the code rather than about -xCAT: it asserts that `proc.py` is the only module that starts a process, and -that no module opens a database or runs an xCAT command. That rule is the reason -the results mean anything -- a suite that asked xCAT what to expect would pass -on a cluster that could not boot a single node. - -## Layout - - provtest/ - README.md this file - src/provtest entry point; runs from a checkout, no install - src/provtest_lib/ - cli.py argument parsing and the three subcommands - config.py INI parsing, %(name)s substitution - model.py Scenario, Step, Assertion, Reply - machine.py what each step type accepts and returns - assertions.py the assertion mini-language - subst.py $step.field resolution - proc.py the only module that spawns a process - dnsc.py dig - httpc.py curl - tftpc.py tftp - xcatc.py the xCAT client: 3001 TLS, 3001 UDP, 3002 plain - netutil.py address and hex-name helpers - runner.py executes scenarios, retries, collects results - report.py TAP, pretty and JSON output - errors.py exceptions and exit codes - conf/ the scenarios - tests/ unit tests diff --git a/xCAT-test/provtest/src/provtest b/xCAT-test/provtest/src/provtest deleted file mode 100755 index 2846b30fd..000000000 --- a/xCAT-test/provtest/src/provtest +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env python3 -"""provtest -- drive the xCAT provision chain the way a booting node does. - -Runs straight out of a checkout: this script puts its own directory on -sys.path, so no installation, no virtualenv and no pip are involved. Nothing -beyond the standard library is imported; the DNS, HTTP and TFTP stages are -driven by `dig`, `curl` and `tftp`, and the xCAT stages by a socket. -""" - -import os -import sys - -sys.path.insert(0, os.path.dirname(os.path.realpath(__file__))) - -from provtest_lib.cli import main # noqa: E402 (path set up above) - -if __name__ == "__main__": - sys.exit(main()) diff --git a/xCAT-test/provtest/src/provtest_lib/__init__.py b/xCAT-test/provtest/src/provtest_lib/__init__.py deleted file mode 100644 index 753608dd0..000000000 --- a/xCAT-test/provtest/src/provtest_lib/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -"""provtest -- drive the xCAT provision chain the way a booting node does. - -A node that is being provisioned speaks five protocols in turn: it resolves -names, fetches a loader and a configuration over TFTP, fetches a kernel and a -kickstart over HTTP, asks xcatd where it is going, and reports back while it -installs. This tool speaks all five, from a client's side, and asserts on what -comes back. - -It never reads the xCAT database and never runs an xCAT command. Every value it -expects is supplied to it, so a test cannot be satisfied by asking xCAT to -confirm its own output. -""" - -__version__ = "1.0" diff --git a/xCAT-test/provtest/src/provtest_lib/assertions.py b/xCAT-test/provtest/src/provtest_lib/assertions.py deleted file mode 100644 index 2b918338b..000000000 --- a/xCAT-test/provtest/src/provtest_lib/assertions.py +++ /dev/null @@ -1,229 +0,0 @@ -"""The assertion mini-language: `target op value`. - -One assertion per line, under a single multi-line `assert` key. A single key -holding many lines sidesteps configparser's one-value-per-key rule and keeps -the grammar uniform, so `status == NXDOMAIN` and `text contains destiny=install` -parse through exactly the same path. - -The operator table is dhcptest's, so an operator reads the same in both suites. -""" - -import re - -from . import netutil, subst -from .errors import ConfigError -from .model import Assertion - -#: Targets that describe the step rather than the reply. -META_TARGETS = frozenset(["attempt"]) - -#: Operators that take no value. -NULLARY_OPS = frozenset(["present", "absent"]) - -#: Operators asserting that the reply is *not* something, which a reply that -#: says nothing on the subject satisfies. -NEGATIVE_OPS = frozenset(["!=", "not-in"]) - -OPS = frozenset([ - "==", "!=", "in", "not-in", "present", "absent", - "matches", "contains", "starts-with", "ends-with", - "<", "<=", ">", ">=", -]) - - -class Result(object): - """The outcome of one assertion, carrying enough to explain a failure.""" - - __slots__ = ("assertion", "ok", "expected", "actual", "detail") - - def __init__(self, assertion, ok, expected=None, actual=None, detail=""): - self.assertion = assertion - self.ok = ok - self.expected = expected - self.actual = actual - self.detail = detail - - def __repr__(self): - return "Result(%r, ok=%r, expected=%r, actual=%r)" % ( - self.assertion.render(), self.ok, self.expected, self.actual) - - -def parse(line, source=""): - """Parse one assertion line into an `Assertion`.""" - text = line.strip() - if not text: - raise ConfigError("empty assertion", source) - - parts = text.split(None, 2) - if len(parts) < 2: - raise ConfigError("assertion needs at least a target and an operator: %r" - % (text,), source) - - target, op = parts[0], parts[1] - value = parts[2] if len(parts) > 2 else None - - if op not in OPS: - raise ConfigError( - "unknown operator %r in %r (expected one of: %s)" - % (op, text, ", ".join(sorted(OPS))), source) - - if op in NULLARY_OPS: - if value: - raise ConfigError("operator %r takes no value: %r" % (op, text), source) - value = None - elif value is None: - # `error != ` with nothing after it is a legitimate "must be non-empty". - value = "" - - if op == "matches": - try: - re.compile(value) - except re.error as exc: - raise ConfigError("bad regex %r: %s" % (value, exc), source) - - return Assertion(target=target, op=op, value=value, source=source, text=text) - - -def parse_block(block, source=""): - """Parse a multi-line `assert = ...` value into a tuple of assertions.""" - result = [] - for number, raw in enumerate(str(block or "").splitlines(), start=1): - line = raw.strip() - if not line or line.startswith("#"): - continue - where = "%s line %d of assert block" % (source, number) if source else "" - result.append(parse(line, where)) - return tuple(result) - - -# --------------------------------------------------------------------------- -# evaluation - - -def evaluate(assertion, reply, context, extras=None): - """Check one assertion against one reply.""" - extras = extras or {} - expected = None - if assertion.value is not None: - expected = subst.resolve(assertion.value, context) - - present, actual = _actual(assertion.target, reply, context, extras) - - if assertion.op == "present": - return Result(assertion, present, expected="present", - actual="present" if present else "absent") - if assertion.op == "absent": - return Result(assertion, not present, expected="absent", - actual="present" if present else "absent") - - if not present: - # A negative comparison is satisfied by a target that is not there at - # all: a config that names no kernel has certainly not named the wrong - # one. `absent` remains the way to assert the absence itself. - if assertion.op in NEGATIVE_OPS: - return Result(assertion, True, expected=expected, actual=None, - detail="%s is not present in the reply" - % (assertion.target,)) - return Result(assertion, False, expected=expected, actual=None, - detail="%s is not present in the reply" % (assertion.target,)) - - ok, detail = _compare(assertion.op, actual, expected) - return Result(assertion, ok, expected=expected, - actual=subst.format_value(actual), detail=detail) - - -def _actual(target, reply, context, extras): - """Return `(present, value)` for one assertion target.""" - if target in META_TARGETS: - if target not in extras: - return False, None - return True, extras[target] - - if target.startswith("$"): - try: - return True, subst.resolve(target, context) - except ConfigError: - return False, None - - if reply is None: - return False, None - if not reply.has(target): - return False, None - return True, reply.whole(target) - - -def _compare(op, actual, expected): - """Type-aware comparison. Returns `(ok, detail)`.""" - if op in ("==", "!="): - same = _equal(actual, expected) - return (same if op == "==" else not same), "" - - if op in ("in", "not-in"): - inside = _member(actual, expected) - return (inside if op == "in" else not inside), "" - - text = subst.format_value(actual) - if op == "matches": - return bool(re.search(expected, text, re.MULTILINE)), "" - if op == "contains": - return expected in text, "" - if op == "starts-with": - return text.startswith(expected), "" - if op == "ends-with": - return text.endswith(expected), "" - - left, right = _as_numbers(actual, expected) - if left is None: - return False, "%r and %r are not both numbers" % (actual, expected) - if op == "<": - return left < right, "" - if op == "<=": - return left <= right, "" - if op == ">": - return left > right, "" - if op == ">=": - return left >= right, "" - raise ConfigError("unhandled operator %r" % (op,)) - - -def _equal(actual, expected): - """Compare as addresses, then as numbers, then as text. - - A list on the left means "any of these", so a name with two A records - satisfies `data == `. Writing the whole set is `data.0` and - `data.1`, or `count == 2`. - """ - if isinstance(actual, (list, tuple)): - return any(_equal(item, expected) for item in actual) - - left_ip = netutil.parse_ip(subst.format_value(actual)) - right_ip = netutil.parse_ip(expected) - if left_ip is not None and right_ip is not None: - return left_ip == right_ip - - left, right = _as_numbers(actual, expected) - if left is not None: - return left == right - - return subst.format_value(actual) == str(expected) - - -def _member(actual, expected): - """`in` means membership: of a CIDR, of a `first-last` range, or of a list.""" - text = subst.format_value(actual) - if "/" in expected and netutil.parse_ip(text) is not None: - return netutil.ip_in(text, expected) - if netutil.parse_range(expected) is not None: - return netutil.ip_in_range(text, expected) - candidates = [item.strip() for item in expected.split(",") if item.strip()] - return any(_equal(actual, candidate) for candidate in candidates) - - -def _as_numbers(actual, expected): - """Both sides as ints, or `(None, None)` when either is not numeric.""" - try: - left = actual if isinstance(actual, int) else int(str(actual).strip(), 0) - right = expected if isinstance(expected, int) else int(str(expected).strip(), 0) - except (TypeError, ValueError): - return None, None - return left, right diff --git a/xCAT-test/provtest/src/provtest_lib/cli.py b/xCAT-test/provtest/src/provtest_lib/cli.py deleted file mode 100644 index 8b1f7dc29..000000000 --- a/xCAT-test/provtest/src/provtest_lib/cli.py +++ /dev/null @@ -1,160 +0,0 @@ -"""Command line interface. - - provtest run [options] CONF [CONF...] # execute scenarios - provtest validate CONF [CONF...] # check offline - provtest list CONF [CONF...] # show what a file does - -`validate` and `list` need no root, no network and no client programs, so they -run in a checkout-only CI job -- which is where a typo in a field name or a -reference to a step that has not run yet should be caught, long before a -management node is involved. -""" - -import argparse -import sys - -from . import config, machine, report -from .errors import EXIT_CONFIG, EXIT_OK, ProvTestError - - -def build_parser(): - parser = argparse.ArgumentParser( - prog="provtest", - description="Drive the xCAT provision chain the way a booting node " - "does, one declarative .conf file at a time.") - subparsers = parser.add_subparsers(dest="command") - - run = subparsers.add_parser( - "run", help="execute scenarios against a real management node") - _add_conf_arguments(run) - run.add_argument("-b", "--bind", metavar="ADDR", - help="source address for every request; xcatd names a " - "client by the reverse lookup of this address") - run.add_argument("--timeout", type=float, metavar="SECS", - help="per-attempt timeout (default: 5)") - run.add_argument("--retries", type=int, metavar="N", - help="attempts per step (default: 2)") - run.add_argument("--format", default="tap", choices=sorted(report.REPORTERS), - help="output format (default: tap)") - run.add_argument("-s", "--scenario", action="append", default=[], - metavar="NAME", help="run only this scenario (repeatable)") - run.add_argument("-v", "--verbose", action="count", default=0, - help="print each reply as it arrives") - - validate = subparsers.add_parser( - "validate", help="parse and check .conf files without any network I/O") - _add_conf_arguments(validate) - - listing = subparsers.add_parser( - "list", help="list the scenarios and steps in .conf files") - _add_conf_arguments(listing) - - return parser - - -def _add_conf_arguments(parser): - parser.add_argument("conf", nargs="+", metavar="CONF", - help="a .conf file in INI syntax") - parser.add_argument("--set", action="append", default=[], metavar="KEY=VALUE", - dest="settings", - help="define %%(KEY)s for substitution (repeatable)") - - -def parse_settings(pairs): - settings = {} - for pair in pairs: - if "=" not in pair: - raise ProvTestError("--set needs KEY=VALUE, got %r" % (pair,)) - key, _, value = pair.partition("=") - settings[key.strip()] = value - return settings - - -def main(argv=None): - parser = build_parser() - args = parser.parse_args(argv) - if not args.command: - parser.print_help() - return EXIT_CONFIG - - try: - if args.command == "validate": - return command_validate(args) - if args.command == "list": - return command_list(args) - if args.command == "run": - return command_run(args) - except ProvTestError as exc: - sys.stderr.write("provtest: %s\n" % (exc,)) - return exc.exit_code - parser.print_help() - return EXIT_CONFIG - - -def command_validate(args): - # Raw: a file is checked in a checkout, where no value has been supplied - # for %(name)s yet, so interpolating it here would fail every time. - settings = parse_settings(args.settings) - scenarios, _ = config.load(args.conf, settings, raw=True) - problems = [] - for scenario in scenarios: - problems.extend(machine.validate_scenario(scenario)) - - for problem in problems: - sys.stderr.write("provtest: %s\n" % (problem,)) - if problems: - return EXIT_CONFIG - - needed = sorted(machine.required_variables(scenarios) - set(settings)) - sys.stdout.write("%d scenario(s) OK in %d file(s)\n" - % (len(scenarios), len(args.conf))) - if needed: - sys.stdout.write("required variables: %s\n" % (", ".join(needed),)) - return EXIT_OK - - -def command_list(args): - scenarios, _ = config.load(args.conf, parse_settings(args.settings), raw=True) - for scenario in scenarios: - sys.stdout.write("%s\n" % (scenario.name,)) - if scenario.description: - sys.stdout.write(" %s\n" % (scenario.description,)) - sys.stdout.write(" source: %s\n" % (scenario.source,)) - for step in scenario.steps: - sys.stdout.write(" step %-16s type=%-12s expect=%-4s asserts=%d\n" - % (step.name, step.type, step.expect or "ok", - len(step.assertions))) - needed = sorted(machine.required_variables([scenario])) - if needed: - sys.stdout.write(" requires: %s\n" % (", ".join(needed),)) - return EXIT_OK - - -def command_run(args): - from . import runner # spawns clients and opens sockets; not for list - - scenarios, _ = config.load(args.conf, parse_settings(args.settings)) - if args.scenario: - wanted = set(args.scenario) - missing = wanted - set(s.name for s in scenarios) - if missing: - raise ProvTestError( - "no such scenario: %s" % (", ".join(sorted(missing)),)) - scenarios = [s for s in scenarios if s.name in wanted] - - problems = [] - for scenario in scenarios: - problems.extend(machine.validate_scenario(scenario)) - if problems: - for problem in problems: - sys.stderr.write("provtest: %s\n" % (problem,)) - return EXIT_CONFIG - - reporter = report.make(args.format) - options = runner.RunOptions( - bind=args.bind, - timeout=args.timeout, - retries=args.retries, - verbose=args.verbose, - ) - return runner.run(scenarios, reporter, options) diff --git a/xCAT-test/provtest/src/provtest_lib/config.py b/xCAT-test/provtest/src/provtest_lib/config.py deleted file mode 100644 index c0fce5eed..000000000 --- a/xCAT-test/provtest/src/provtest_lib/config.py +++ /dev/null @@ -1,242 +0,0 @@ -"""Reading declarative .conf files with configparser. - -Layout of a file:: - - [vars] optional; %(name)s values, overridable with --set - [defaults] optional; step keys applied to every step - [scenario ] starts a scenario - [step ] belongs to the scenario above it - -Variables use configparser's own `BasicInterpolation`: `%(name)s` is resolved -by the library, out of `[vars]` and of anything passed with `--set`. `$` has no -meaning to it, so `$grubcfg.text` reaches the runtime untouched -- and it has -to, because its value is a reply that has not arrived when the file is read. - -Unknown keys are a hard error rather than a silent skip, matching `xcattest`, -where an unrecognised `check:` fails the case instead of passing it. -""" - -import configparser -import os - -from . import assertions as assertions_mod -from . import machine -from .errors import ConfigError -from .model import Scenario, Step - -#: Keys accepted in a `[scenario ...]` section. -SCENARIO_KEYS = frozenset(["description"]) - -#: Keys `[defaults]` may carry. `type` and `assert` are per-step by nature: a -#: default type would make a mistyped `type=` silently become another protocol, -#: and a default assertion would be made about replies it was never written for. -DEFAULTS_KEYS = frozenset( - key - for step_type in machine.STEP_KEYS - for key in machine.allowed_keys(step_type) -) - frozenset(["type", "assert", "description"]) - - -def load(paths, overrides=None, raw=False): - """Read every path and return `(scenarios, variables)`. - - With `raw`, `%(name)s` is left in place instead of being interpolated, so - `validate` can check a file in a checkout where no value has been supplied - for it yet. - """ - scenarios = [] - variables = {} - for path in paths: - file_scenarios, file_vars = load_file(path, overrides, raw) - scenarios.extend(file_scenarios) - variables.update(file_vars) - variables.update(overrides or {}) - _reject_duplicate_scenarios(scenarios) - return scenarios, variables - - -def load_file(path, overrides=None, raw=False): - """Read one .conf file into `(scenarios, variables)`.""" - if not os.path.isfile(path): - raise ConfigError("no such file", path) - - parser = configparser.ConfigParser( - interpolation=None, - inline_comment_prefixes=(";",), - delimiters=("=",), - ) - parser.optionxform = str.lower - try: - with open(path) as handle: - parser.read_file(handle, source=path) - except configparser.Error as exc: - raise ConfigError(str(exc), path) - - variables = dict(parser["vars"]) if parser.has_section("vars") else {} - variables.update(overrides or {}) - - scenarios = [] - defaults = {} - current = None - steps = [] - - for section in parser.sections(): - items = _items(parser, section, path, raw, variables) - kind, _, name = section.partition(" ") - kind = kind.strip().lower() - name = name.strip() - where = "%s [%s]" % (path, section) - - if section.lower() == "vars": - continue # read above, into `variables` - - if section.lower() == "defaults": - _reject_unknown(items, DEFAULTS_KEYS, where) - defaults = items - continue - - if kind == "scenario": - if not name: - raise ConfigError("a [scenario] section needs a name", where) - if current is not None: - scenarios.append(_build_scenario(current, steps, path)) - _reject_unknown(items, SCENARIO_KEYS, where) - current = (name, items) - steps = [] - continue - - if kind == "step": - if current is None: - raise ConfigError( - "a [step] section must follow a [scenario] section", where) - if not name: - raise ConfigError("a [step] section needs a name", where) - steps.append(_build_step(name, items, defaults, where)) - continue - - raise ConfigError( - "unknown section; expected [vars], [defaults], [scenario ] " - "or [step ]", where) - - if current is not None: - scenarios.append(_build_scenario(current, steps, path)) - elif steps: - raise ConfigError("steps without a scenario", path) - - if not scenarios: - raise ConfigError("no [scenario ...] section found", path) - - return scenarios, variables - - -def _build_scenario(current, steps, path): - name, items = current - if not steps: - raise ConfigError("scenario %r has no steps" % (name,), path) - return Scenario( - name=name, - description=items.get("description", ""), - steps=steps, - source=path, - ) - - -def _build_step(name, items, defaults, where): - step_type = (items.get("type") or "").strip().lower() - if not step_type: - raise ConfigError("step %r has no type=" % (name,), where) - machine.check_type(step_type, where) - - allowed = machine.allowed_keys(step_type) - _reject_unknown(items, allowed, where) - - # A default only applies where the type in hand can use it, so one - # `[defaults] server =` serves dns, tftp, http and xcatreq steps alike - # without making `path =` illegal for the dns step in the same file. - merged = dict((key, value) for key, value in defaults.items() - if key in allowed) - merged.update(items) - - expect = (merged.get("expect") or "").strip().lower() or None - timeout = _float(merged.get("timeout"), 5.0, "timeout", where) - retries = _int(merged.get("retries"), 2, "retries", where) - parsed = assertions_mod.parse_block(merged.pop("assert", ""), where) - - for key in ("type", "expect", "timeout", "retries"): - merged.pop(key, None) - - return Step( - name=name, - type=step_type, - expect=expect, - params=merged, - assertions=parsed, - timeout=timeout, - retries=retries, - source=where, - ) - - -#: configparser's own variable syntax, `%(name)s`. The parser is built with -#: `interpolation=None` and this is applied by hand, per value, because both of -#: the built-in ways of supplying the values -- DEFAULTSECT and `get(vars=...)` -#: -- also make each variable an *option*, so `--set path=/install` would -#: overwrite the step key of the same name. Interpolating explicitly keeps the -#: two namespaces apart: variables are only ever read through `%(...)s`. -INTERPOLATION = configparser.BasicInterpolation() - - -def _items(parser, section, path, raw, variables=None): - """One section's keys, with `%(name)s` resolved out of `variables`.""" - items = {} - for key in parser.options(section): - value = parser.get(section, key, raw=True) - if raw: - items[key] = value - continue - try: - items[key] = INTERPOLATION.before_get( - parser, section, key, value, variables or {}) - except configparser.InterpolationMissingOptionError as exc: - raise ConfigError( - "%%(%s)s is not defined; pass --set %s=" - % (exc.reference, exc.reference), "%s [%s]" % (path, section)) - except configparser.Error as exc: - raise ConfigError(str(exc), "%s [%s]" % (path, section)) - return items - - -def _reject_unknown(items, allowed, where): - unknown = sorted(set(items) - set(allowed)) - if unknown: - raise ConfigError( - "unknown key(s): %s (allowed: %s)" - % (", ".join(unknown), ", ".join(sorted(allowed))), where) - - -def _reject_duplicate_scenarios(scenarios): - seen = {} - for scenario in scenarios: - if scenario.name in seen: - raise ConfigError( - "scenario %r is defined in both %s and %s" - % (scenario.name, seen[scenario.name], scenario.source)) - seen[scenario.name] = scenario.source - - -def _float(value, default, key, where): - if value is None or value == "": - return default - try: - return float(value) - except ValueError: - raise ConfigError("%s= must be a number, got %r" % (key, value), where) - - -def _int(value, default, key, where): - if value is None or value == "": - return default - try: - return int(value, 0) if isinstance(value, str) else int(value) - except ValueError: - raise ConfigError("%s= must be an integer, got %r" % (key, value), where) diff --git a/xCAT-test/provtest/src/provtest_lib/errors.py b/xCAT-test/provtest/src/provtest_lib/errors.py deleted file mode 100644 index 552e22eaf..000000000 --- a/xCAT-test/provtest/src/provtest_lib/errors.py +++ /dev/null @@ -1,46 +0,0 @@ -"""Exceptions and exit codes. - -The codes are dhcptest's, so a caller driving both suites reads one table: - - 0 every assertion held - 1 at least one assertion failed - 2 the configuration or the command line is wrong - 3 this host cannot run the test at all - -3 is deliberately not 0. A host with no `dig` reports that it tested nothing -rather than reporting a green run, because a pass that proves nothing is worse -than a failure that says why. -""" - -EXIT_OK = 0 -EXIT_FAILED = 1 -EXIT_CONFIG = 2 -EXIT_UNSUPPORTED = 3 - - -class ProvTestError(Exception): - """Base class: every failure this tool raises on purpose.""" - - exit_code = EXIT_CONFIG - - -class ConfigError(ProvTestError): - """A .conf file says something that cannot be carried out.""" - - exit_code = EXIT_CONFIG - - def __init__(self, message, source=""): - self.message = message - self.source = source - Exception.__init__(self, self.__str__()) - - def __str__(self): - if self.source: - return "%s: %s" % (self.source, self.message) - return self.message - - -class UnsupportedError(ProvTestError): - """The host is missing something the run cannot go on without.""" - - exit_code = EXIT_UNSUPPORTED diff --git a/xCAT-test/provtest/src/provtest_lib/machine.py b/xCAT-test/provtest/src/provtest_lib/machine.py deleted file mode 100644 index dfd5efb64..000000000 --- a/xCAT-test/provtest/src/provtest_lib/machine.py +++ /dev/null @@ -1,234 +0,0 @@ -"""What a step of each type may say, and what its reply will hold. - -This is the offline half of the tool. `validate` runs it in a checkout with no -network, no root and no server, so a typo in a field name or a step that names -no server is a configuration error reported before anything is sent. - -Every table here is declarative on purpose: adding a protocol means adding a -row, not a branch in the runner. -""" - -from . import subst -from .errors import ConfigError - -#: Keys every step may carry, whatever its type. `bind` is the client's own -#: address: xcatd names a client by the reverse lookup of the address it -#: connected from, so which end of the veth pair a request leaves by is itself -#: under test, and cannot be left to the kernel's source selection. -COMMON_KEYS = frozenset(["type", "expect", "timeout", "retries", "assert", - "bind"]) - -#: Per type: the keys it accepts, beyond COMMON_KEYS. -STEP_KEYS = { - "dns": frozenset(["server", "port", "name", "rrtype", "recursion"]), - "tftp": frozenset(["server", "port", "path", "mode"]), - "http": frozenset(["server", "port", "path", "url", "method", "header", - "insecure"]), - "xcatreq": frozenset(["server", "port", "command", "element", "raw", - "callback_port", "callback_listen", "callback_reply", - "callback_wait", "cert", "key", "source_port"]), - "monitor": frozenset(["server", "port", "send", "source_port"]), - "flowrequest": frozenset(["server", "port", "message", "source_port", - "replies"]), - "findme": frozenset(["server", "port", "payload", "encoding", "source_port", - "callback_listen", "callback_wait"]), - "extract": frozenset(["from", "pattern", "group"]), - "sleep": frozenset(["duration"]), - "noop": frozenset([]), -} - -#: Keys naming a value that must be present for the step to mean anything. -REQUIRED_KEYS = { - "dns": ("server", "name"), - "tftp": ("server", "path"), - "http": (), # url, or server+path; checked below - "xcatreq": ("server", "command"), - "monitor": ("server", "send"), - "flowrequest": ("server",), - "findme": ("server",), - "extract": ("from", "pattern"), - "sleep": ("duration",), - "noop": (), -} - -#: Per type: the reply fields a `.conf` may name, as an assertion target or -#: through `$step.field`. A name not in this set is a typo, and is reported as -#: one rather than silently never matching. -REPLY_FIELDS = { - "dns": frozenset(["status", "flags", "count", "data", "type", "ttl", - "name", "question", "answers", "authority", "server", - "rcode", "raw"]), - "tftp": frozenset(["ok", "size", "sha256", "text", "error", "path", - "server", "raw"]), - "http": frozenset(["status", "size", "sha256", "text", "url", "header", - "content_type", "error", "ok", "raw"]), - "xcatreq": frozenset(["destiny", "kernel", "initrd", "kcmdline", - "imgserver", "name", "error", "serverdone", - "elements", "data", "text", "handshake", "ok", - "callback_seen", "callback_data", "raw", - # getcredentials wraps its payload one level deeper, - # as , and `desc` is - # how a reply carrying several credentials says - # which one each part is. - "content", "desc"]), - "monitor": frozenset(["greeting", "lines", "raw", "text", "ok", "closed", - "error"]), - "flowrequest": frozenset(["replies", "count", "ok", "error", "raw"]), - "findme": frozenset(["callbacks", "count", "ok", "error", "raw"]), - "extract": frozenset(["value", "groups", "matched", "count"]), - "sleep": frozenset([]), - "noop": frozenset([]), -} - -#: Reply fields that are a mapping rather than a scalar or a list, and so are -#: addressed as `header.content-type` or `elements.destiny`. -MAPPING_FIELDS = frozenset(["header", "elements"]) - -#: `expect=` values. `fail` is not an error: a scenario asserting that a path -#: outside the tftp root is refused wants the refusal to be the passing result. -EXPECTATIONS = frozenset(["ok", "fail", "any"]) - - -def known_step_types(): - return frozenset(STEP_KEYS) - - -def allowed_keys(step_type): - return COMMON_KEYS | STEP_KEYS.get(step_type, frozenset()) - - -def all_reply_fields(): - """Every field name any step type can produce.""" - names = set() - for fields in REPLY_FIELDS.values(): - names.update(fields) - return names - - -def check_field(field): - """True when `field` is a name some step type's reply could carry. - - Cross-type: a `$step.field` reference is resolved against whichever step - it names, and which step that is cannot be known from the text alone when - the reference is written in `[defaults]`. Validation is therefore that the - name exists *somewhere*; naming a field of the wrong type fails at run - time, where the reply is in hand and the message can say so. - """ - head, _, tail = field.rpartition(".") - if head: - if head in MAPPING_FIELDS: - return bool(tail) - if head not in all_reply_fields(): - return False - try: - int(tail) - except ValueError: - return False - return True - return field in all_reply_fields() - - -def validate_scenario(scenario): - """Every problem in one scenario, as a list of strings. - - Returns rather than raises, so one pass over a file reports everything - wrong with it instead of the first thing. - """ - problems = [] - seen = set() - for step in scenario.steps: - where = "%s scenario %r step %r" % (scenario.source, scenario.name, step.name) - if step.name in seen: - problems.append("%s: step %r is defined twice" % (where, step.name)) - seen.add(step.name) - - problems.extend(_validate_required(step, where)) - problems.extend(_validate_expect(step, where)) - problems.extend(_validate_targets(step, where)) - problems.extend(_validate_references(step, scenario, seen, where)) - return problems - - -def _validate_required(step, where): - problems = [] - for key in REQUIRED_KEYS.get(step.type, ()): - if not step.param(key): - problems.append("%s: a %s step needs %s=" % (where, step.type, key)) - if step.type == "http" and not step.param("url"): - if not (step.param("server") and step.param("path")): - problems.append( - "%s: an http step needs url=, or server= and path=" % (where,)) - if step.type == "extract": - group = step.param("group") - if group not in (None, ""): - try: - int(group) - except ValueError: - problems.append("%s: group= must be a number, got %r" - % (where, group)) - return problems - - -def _validate_expect(step, where): - if step.expect and step.expect not in EXPECTATIONS: - return ["%s: unknown expect=%r (known: %s)" - % (where, step.expect, ", ".join(sorted(EXPECTATIONS)))] - return [] - - -def _validate_targets(step, where): - problems = [] - fields = REPLY_FIELDS.get(step.type, frozenset()) - for assertion in step.assertions: - target = assertion.target - if target.startswith("$"): - continue - head, _, tail = target.rpartition(".") - if head in MAPPING_FIELDS: - if head not in fields: - problems.append("%s: a %s reply carries no %s" - % (where, step.type, head)) - continue - base = head or target - if base not in fields: - problems.append( - "%s: a %s reply has no field %r (it has: %s)" - % (where, step.type, target, ", ".join(sorted(fields)))) - return problems - - -def _validate_references(step, scenario, seen, where): - """Reject a `$step.field` naming a step that does not run before this one.""" - problems = [] - texts = [assertion.value for assertion in step.assertions] - texts.extend(str(value) for value in step.params.values()) - texts.extend(assertion.target for assertion in step.assertions) - for text in texts: - for name, field in subst.references(text): - if name not in seen: - problems.append( - "%s: $%s.%s refers to a step that has not run yet" - % (where, name, field)) - elif not check_field(field): - problems.append("%s: no reply field is called .%s" % (where, field)) - return problems - - -def required_variables(scenarios): - """Every `%(name)s` a run of these scenarios would have to be given.""" - names = set() - for scenario in scenarios: - for step in scenario.steps: - for value in step.params.values(): - names.update(subst.variables(value)) - for assertion in step.assertions: - names.update(subst.variables(assertion.value)) - names.update(subst.variables(assertion.target)) - return names - - -def check_type(step_type, where=""): - if step_type not in STEP_KEYS: - raise ConfigError( - "unknown step type %r (known: %s)" - % (step_type, ", ".join(sorted(STEP_KEYS))), where) diff --git a/xCAT-test/provtest/src/provtest_lib/model.py b/xCAT-test/provtest/src/provtest_lib/model.py deleted file mode 100644 index 4dd56ba28..000000000 --- a/xCAT-test/provtest/src/provtest_lib/model.py +++ /dev/null @@ -1,196 +0,0 @@ -"""Plain data structures shared across the tool. - -No network and no subprocess: everything here can be built by hand in a unit -test, which is how the assertion language is tested without a server. - -Written for Python 3.6, so no dataclasses. -""" - -from .errors import ConfigError - - -class Assertion(object): - """One line of the assertion mini-language: `target op value`. - - For example:: - - status == NOERROR - data == 10.99.1.11 - text contains destiny=install - sha256 == $tftpfetch.sha256 - """ - - __slots__ = ("target", "op", "value", "source", "text") - - def __init__(self, target, op, value, source="", text=""): - self.target = target - self.op = op - self.value = value - self.source = source - self.text = text or self.render() - - def render(self): - if self.value is None: - return "%s %s" % (self.target, self.op) - return "%s %s %s" % (self.target, self.op, self.value) - - def __repr__(self): - return "Assertion(%r)" % (self.render(),) - - -class Step(object): - """One client action plus the assertions made about what came back.""" - - __slots__ = ("name", "type", "expect", "params", "assertions", "timeout", - "retries", "source") - - def __init__(self, name, type, expect, params, assertions, - timeout=5.0, retries=2, source=""): - self.name = name - self.type = type - self.expect = expect - self.params = dict(params) - self.assertions = tuple(assertions) - self.timeout = timeout - self.retries = retries - self.source = source - - def param(self, key, default=None): - value = self.params.get(key, default) - if isinstance(value, str): - value = value.strip() - return value - - def __repr__(self): - return "Step(%r, type=%r, expect=%r)" % (self.name, self.type, self.expect) - - -class Scenario(object): - """A named sequence of steps, run in order against one server.""" - - __slots__ = ("name", "description", "steps", "source") - - def __init__(self, name, description="", steps=(), source=""): - self.name = name - self.description = description - self.steps = tuple(steps) - self.source = source - - def step(self, name): - for step in self.steps: - if step.name == name: - return step - raise ConfigError("no such step %r in scenario %r" % (name, self.name)) - - def __repr__(self): - return "Scenario(%r, steps=%d)" % (self.name, len(self.steps)) - - -class Reply(object): - """What one step got back, as a flat namespace of fields. - - Unlike DHCP, the five protocols here share no header, so a reply is a bag - of named values rather than a fixed record. What each step type puts in it - is declared in `machine.REPLY_FIELDS`, so a `.conf` naming a field no step - of that type ever produces is rejected before any socket is opened. - - A field may hold a list -- DNS answers, monitor lines, findme callbacks. - Indexing is written `data.0`, and the bare name of a list resolves to its - first element, so `data == 10.99.1.11` reads the way an operator expects - for the ordinary one-answer case. - """ - - __slots__ = ("kind", "fields", "ok", "error", "sent", "attempt", "raw") - - def __init__(self, kind="", fields=None, ok=True, error="", sent="", - attempt=0, raw=b""): - self.kind = kind - self.fields = dict(fields or {}) - self.ok = ok - self.error = error - self.sent = sent - self.attempt = attempt - self.raw = raw - - def field(self, name): - """Resolve one `$step.` reference, or raise KeyError. - - `a.b` is looked up whole first, so a header written into the reply as - `header.content-type` wins over any attempt to index a list called - `header`. Only then is the name split into container and index. - """ - if name in self.fields: - return self._first(self.fields[name]) - head, _, tail = name.rpartition(".") - if head and head in self.fields: - container = self.fields[head] - if isinstance(container, (list, tuple)): - try: - index = int(tail) - except ValueError: - raise KeyError("%s is not an index into .%s" % (tail, head)) - if index >= len(container) or index < -len(container): - raise KeyError( - "%s reply has %d item(s) in .%s, so .%s does not exist" - % (self.kind, len(container), head, name)) - return container[index] - if isinstance(container, dict): - if tail not in container: - raise KeyError("%s reply has no .%s" % (self.kind, name)) - return container[tail] - raise KeyError("%s reply has no field .%s" % (self.kind or "this", name)) - - def whole(self, name): - """Like `field`, but a list comes back entire instead of collapsed. - - `$step.data` wants one value to substitute into a command line, so - `field` gives it the first. An assertion wants all of them: a name with - two A records satisfies `data == `, and collapsing here would - make the second address unassertable without an index. - """ - if name in self.fields: - return self.fields[name] - return self.field(name) - - def has(self, name): - try: - value = self.whole(name) - except KeyError: - return False - # An empty list is "nothing came back", which is absence, not a value. - if isinstance(value, (list, tuple)): - return bool(value) - return value not in (None, "") - - @staticmethod - def _first(value): - if isinstance(value, (list, tuple)): - return value[0] if value else [] - return value - - def summary(self): - parts = [] - for key in sorted(self.fields): - value = self.fields[key] - if key in ("text", "raw", "body"): - continue - parts.append("%s=%s" % (key, _short(value))) - head = "%s %s" % (self.kind or "reply", "ok" if self.ok else "failed") - if self.error: - head += " (%s)" % (self.error,) - return "%s %s" % (head, " ".join(parts)) - - def __repr__(self): - return "Reply(%s)" % (self.summary(),) - - -def _short(value, limit=60): - if isinstance(value, (list, tuple)): - text = "[" + ", ".join(str(item) for item in value) + "]" - elif isinstance(value, dict): - text = "{" + ", ".join("%s=%s" % (k, v) for k, v in sorted(value.items())) + "}" - else: - text = str(value) - if len(text) > limit: - text = text[:limit - 3] + "..." - return text diff --git a/xCAT-test/provtest/src/provtest_lib/netutil.py b/xCAT-test/provtest/src/provtest_lib/netutil.py deleted file mode 100644 index 834c3a293..000000000 --- a/xCAT-test/provtest/src/provtest_lib/netutil.py +++ /dev/null @@ -1,111 +0,0 @@ -"""Address arithmetic, and nothing that touches a network. - -Kept separate from the clients so that the assertion language can compare -addresses in a unit test with no interface, no root and no server. -""" - -import ipaddress - - -def parse_ip(text): - """An `IPv4Address`, or None when `text` is not one.""" - try: - return ipaddress.IPv4Address(str(text).strip()) - except (ipaddress.AddressValueError, ValueError): - return None - - -def ip_in(address, network): - """True when `address` falls inside the CIDR block `network`.""" - ip = parse_ip(address) - if ip is None: - return False - try: - net = ipaddress.IPv4Network(str(network).strip(), strict=False) - except (ipaddress.AddressValueError, ipaddress.NetmaskValueError, ValueError): - return False - return ip in net - - -def parse_range(text): - """`first-last` as a pair of addresses, or None.""" - if text is None or "-" not in str(text): - return None - first, _, last = str(text).strip().partition("-") - low, high = parse_ip(first), parse_ip(last) - if low is None or high is None: - return None - return low, high - - -def ip_in_range(address, text): - """True when `address` is inside an inclusive `first-last` range.""" - bounds = parse_range(text) - ip = parse_ip(address) - if bounds is None or ip is None: - return False - low, high = bounds - return low <= ip <= high - - -def hex_ip(address, upper=True): - """The eight-digit hex a PXE or grub2 loader asks for its address by. - - This is the encoding the loader itself performs, not xCAT's: 10.99.1.11 - becomes 0A63010B. It is here so a scenario can say what the name should be - without the fixture having to compute it, and so the tool never has to ask - xCAT what name it wrote. - """ - ip = parse_ip(address) - if ip is None: - raise ValueError("not an IPv4 address: %r" % (address,)) - text = "%08X" % (int(ip),) - return text if upper else text.lower() - - -def hex_net(address, prefix, upper=True): - """The name a per-network configuration is written under. - - A loader that finds no file for its own address asks for ever shorter - prefixes of it, so the per-network file is named by as many hex digits as - the netmask covers, rounded up to a whole digit: 10.99.1.0/24 is 0A6301, - six digits and not eight. Anything defined for a network rather than for a - node -- which is everything a machine nobody has defined can reach -- is - named this way. - """ - text = hex_ip(address, upper=upper) - try: - bits = int(prefix) - except (TypeError, ValueError): - raise ValueError("not a prefix length: %r" % (prefix,)) - if not 0 <= bits <= 32: - raise ValueError("not a prefix length: %r" % (prefix,)) - return text[:(bits + 3) // 4] - - -def reverse_name(address): - """The IN-ADDR.ARPA name a resolver is asked for a PTR by.""" - ip = parse_ip(address) - if ip is None: - raise ValueError("not an IPv4 address: %r" % (address,)) - return ip.reverse_pointer - - -def normalise_mac(text): - """A MAC in the lowercase colon form, or ValueError.""" - raw = str(text).strip().lower().replace("-", ":").replace(".", ":") - parts = [part for part in raw.split(":") if part != ""] - if len(parts) != 6: - raise ValueError("not a MAC address: %r" % (text,)) - octets = [] - for part in parts: - if len(part) > 2: - raise ValueError("not a MAC address: %r" % (text,)) - value = int(part, 16) - octets.append("%02x" % (value,)) - return ":".join(octets) - - -def dashed_mac(text): - """The `01-aa-bb-...` form a grub2 or pxelinux per-MAC file is named by.""" - return normalise_mac(text).replace(":", "-") diff --git a/xCAT-test/provtest/src/provtest_lib/report.py b/xCAT-test/provtest/src/provtest_lib/report.py deleted file mode 100644 index 3df836196..000000000 --- a/xCAT-test/provtest/src/provtest_lib/report.py +++ /dev/null @@ -1,216 +0,0 @@ -"""Result reporting: TAP 13 by default, plus human and JSON forms. - -TAP is the default because `prove` consumes it directly, and because it is what -dhcptest emits, so a driver that already collects one suite's output collects -this one unchanged. - -The failure diagnostic carries expected, received, the reply's fields and the -command or request that produced it, so a red test can be understood from the -log without re-running it -- which matters most in CI, where the network it ran -on no longer exists by the time anyone reads the result. -""" - -import json -import sys - -from .errors import EXIT_FAILED, EXIT_OK - - -class Record(object): - """One assertion outcome, with everything needed to explain it.""" - - __slots__ = ("scenario", "step", "description", "ok", "skip_reason", - "expected", "actual", "detail", "reply", "sent", "attempt", - "attempts") - - def __init__(self, scenario, step, description, ok, skip_reason=None, - expected=None, actual=None, detail="", reply=None, sent="", - attempt=0, attempts=0): - self.scenario = scenario - self.step = step - self.description = description - self.ok = ok - self.skip_reason = skip_reason - self.expected = expected - self.actual = actual - self.detail = detail - self.reply = reply - self.sent = sent - self.attempt = attempt - self.attempts = attempts - - @property - def name(self): - return "%s/%s: %s" % (self.scenario, self.step, self.description) - - -class Reporter(object): - """Base reporter: counts, and an exit code.""" - - def __init__(self, stream=None): - self.stream = stream or sys.stdout - self.records = [] - self.passed = 0 - self.failed = 0 - self.skipped = 0 - - def add(self, record): - self.records.append(record) - if record.skip_reason is not None: - self.skipped += 1 - elif record.ok: - self.passed += 1 - else: - self.failed += 1 - self.emit(record) - - def emit(self, record): - raise NotImplementedError - - def start(self): - pass - - def finish(self): - pass - - def exit_code(self): - return EXIT_FAILED if self.failed else EXIT_OK - - def write(self, text): - self.stream.write(text) - self.stream.flush() - - -class TapReporter(Reporter): - """TAP version 13, with a trailing plan and YAML failure diagnostics.""" - - def start(self): - self.write("TAP version 13\n") - - def emit(self, record): - number = len(self.records) - status = "ok" if (record.ok or record.skip_reason is not None) else "not ok" - line = "%s %d - %s" % (status, number, record.name) - if record.skip_reason is not None: - line += " # SKIP %s" % (record.skip_reason,) - self.write(line + "\n") - if not record.ok and record.skip_reason is None: - self.write(_yaml_block(record)) - - def finish(self): - self.write("1..%d\n" % (len(self.records),)) - self.write("# passed %d, failed %d, skipped %d\n" - % (self.passed, self.failed, self.skipped)) - - -class PrettyReporter(Reporter): - """Human-first output carrying the same information as the TAP form.""" - - def __init__(self, stream=None): - Reporter.__init__(self, stream) - self._scenario = None - - def emit(self, record): - if record.scenario != self._scenario: - self._scenario = record.scenario - self.write("\n%s\n" % (record.scenario,)) - if record.skip_reason is not None: - self.write(" SKIP %s/%s -- %s\n" - % (record.step, record.description, record.skip_reason)) - return - mark = "PASS" if record.ok else "FAIL" - self.write(" %s %s: %s\n" % (mark, record.step, record.description)) - if not record.ok: - for line in _diagnostic_lines(record): - self.write(" %s\n" % (line,)) - - def finish(self): - self.write("\nPROVTEST SUMMARY: total=%d passed=%d failed=%d skipped=%d\n" - % (len(self.records), self.passed, self.failed, self.skipped)) - - -class JsonReporter(Reporter): - """One JSON object per run, for callers that would rather not parse text.""" - - def emit(self, record): - pass - - def finish(self): - payload = { - "total": len(self.records), - "passed": self.passed, - "failed": self.failed, - "skipped": self.skipped, - "records": [_as_dict(record) for record in self.records], - } - self.write(json.dumps(payload, indent=2, sort_keys=True) + "\n") - - -REPORTERS = { - "tap": TapReporter, - "pretty": PrettyReporter, - "json": JsonReporter, -} - - -def make(name, stream=None): - try: - return REPORTERS[name](stream) - except KeyError: - raise ValueError("unknown report format %r" % (name,)) - - -# --------------------------------------------------------------------------- -# diagnostics - - -def _diagnostic_lines(record): - lines = [] - if record.expected is not None: - lines.append("expected: %r" % (record.expected,)) - lines.append("received: %r" % (record.actual,)) - if record.detail: - lines.append("detail: %s" % (_oneline(record.detail),)) - if record.reply is not None: - lines.append("reply: %s" % (_oneline(record.reply.summary()),)) - else: - lines.append("reply: none received") - if record.sent: - lines.append("sent: %s" % (_oneline(record.sent),)) - if record.attempts: - lines.append("attempt: %d of %d" % (record.attempt, record.attempts)) - return lines - - -def _yaml_block(record): - out = [" ---\n"] - for line in _diagnostic_lines(record): - key, _, value = line.partition(":") - out.append(" %s: %s\n" % (key.strip(), value.strip())) - out.append(" ...\n") - return "".join(out) - - -def _oneline(text, limit=400): - """Collapse a value to one line, because TAP's diagnostics are per line.""" - flat = " ".join(str(text).split()) - if len(flat) > limit: - flat = flat[:limit - 3] + "..." - return flat - - -def _as_dict(record): - return { - "scenario": record.scenario, - "step": record.step, - "assertion": record.description, - "ok": bool(record.ok), - "skip": record.skip_reason, - "expected": record.expected, - "actual": record.actual, - "detail": record.detail, - "reply": record.reply.summary() if record.reply is not None else None, - "sent": record.sent, - "attempt": record.attempt, - "attempts": record.attempts, - } diff --git a/xCAT-test/provtest/src/provtest_lib/runner.py b/xCAT-test/provtest/src/provtest_lib/runner.py deleted file mode 100644 index 5a8a54121..000000000 --- a/xCAT-test/provtest/src/provtest_lib/runner.py +++ /dev/null @@ -1,354 +0,0 @@ -"""Executing scenarios: one step at a time, in the order they were written. - -A step is dispatched to the client for its protocol, its reply is bound under -the step's name so later steps can refer to it, and its assertions are then -evaluated against that reply. Binding before asserting is what makes the -cross-stage cases work: a TFTP step fetches a grub config, an extract step -pulls the kernel path out of it, and an HTTP step fetches exactly that path -- -none of which is possible if a step can only see values written in the file. -""" - -import re -import sys -import time - -from . import assertions as assertions_mod -from . import dnsc, httpc, machine, report, subst, tftpc, xcatc -from .errors import ConfigError, ProvTestError -from .model import Reply - -#: Step types whose "ok" is decided by the step itself rather than a server. -LOCAL_TYPES = frozenset(["extract", "sleep", "noop"]) - - -class RunOptions(object): - """Command-line overrides, applied on top of what the .conf says.""" - - __slots__ = ("bind", "timeout", "retries", "verbose") - - def __init__(self, bind=None, timeout=None, retries=None, verbose=0): - self.bind = bind - self.timeout = timeout - self.retries = retries - self.verbose = verbose - - -def run(scenarios, reporter, options): - """Run every scenario and return the process exit code.""" - reporter.start() - for scenario in scenarios: - _run_scenario(scenario, reporter, options) - reporter.finish() - return reporter.exit_code() - - -def _run_scenario(scenario, reporter, options): - context = subst.Context() - for step in scenario.steps: - try: - reply, attempt, attempts = _execute(step, context, options) - except ProvTestError as exc: - reporter.add(report.Record( - scenario.name, step.name, "step could not be run", ok=False, - detail=str(exc))) - return - context.bind(step.name, reply) - if options.verbose: - sys.stderr.write("# %s/%s: %s\n" - % (scenario.name, step.name, reply.summary())) - - extras = {"attempt": attempt} - _check_expectation(scenario, step, reply, reporter, attempt, attempts) - for assertion in step.assertions: - try: - result = assertions_mod.evaluate(assertion, reply, context, extras) - except ConfigError as exc: - reporter.add(report.Record( - scenario.name, step.name, assertion.render(), ok=False, - detail=str(exc), reply=reply, sent=reply.sent)) - continue - reporter.add(report.Record( - scenario.name, step.name, assertion.render(), ok=result.ok, - expected=result.expected, actual=result.actual, - detail=result.detail, reply=reply, sent=reply.sent, - attempt=attempt, attempts=attempts)) - - -def _check_expectation(scenario, step, reply, reporter, attempt, attempts): - """Report the transport outcome itself, unless the step waived it. - - The default is `ok`, so a step whose server never answered fails even when - the author wrote no assertion about it. Silence is not a pass. - """ - expect = step.expect or ("any" if step.type in LOCAL_TYPES else "ok") - if expect == "any": - return - wanted = (expect == "ok") - description = "expect %s" % (expect,) - reporter.add(report.Record( - scenario.name, step.name, description, ok=(reply.ok == wanted), - expected=expect, actual="ok" if reply.ok else "failed", - detail=reply.error, reply=reply, sent=reply.sent, - attempt=attempt, attempts=attempts)) - - -def _execute(step, context, options): - """Run one step, retrying only a transport that was expected to work.""" - retries = options.retries if options.retries is not None else step.retries - retries = max(1, int(retries)) - expect = step.expect or ("any" if step.type in LOCAL_TYPES else "ok") - - reply = None - for attempt in range(1, retries + 1): - reply = _dispatch(step, context, options) - reply.attempt = attempt - if expect != "ok" or reply.ok or attempt == retries: - return reply, attempt, retries - time.sleep(0.2) - return reply, retries, retries - - -def _dispatch(step, context, options): - handler = HANDLERS.get(step.type) - if handler is None: - raise ConfigError("no handler for step type %r" % (step.type,), step.source) - return handler(step, context, options) - - -# --------------------------------------------------------------------------- -# per-type handlers - - -def _dns(step, context, options): - param = _resolver(step, context) - return dnsc.query( - server=param("server"), - name=param("name"), - rrtype=param("rrtype") or "A", - port=_int(param("port"), 53), - bind=param("bind") or options.bind, - recursion=_bool(param("recursion"), True), - timeout=_timeout(step, options), - ) - - -def _tftp(step, context, options): - param = _resolver(step, context) - return tftpc.fetch( - server=param("server"), - path=param("path"), - port=_int(param("port"), 69), - mode=param("mode") or "octet", - timeout=_timeout(step, options), - ) - - -def _http(step, context, options): - param = _resolver(step, context) - url = param("url") - if not url: - url = httpc.url_for(param("server"), param("path"), - _int(param("port"), 80)) - headers = [line.strip() for line in (param("header") or "").splitlines() - if line.strip()] - return httpc.fetch( - url=url, - method=param("method") or "GET", - bind=param("bind") or options.bind, - headers=headers, - timeout=_timeout(step, options), - insecure=_bool(param("insecure"), False), - ) - - -def _xcatreq(step, context, options): - param = _resolver(step, context) - return xcatc.request( - server=param("server"), - command=param("command"), - port=_int(param("port"), 3001), - elements=_pairs(param("element")), - bind=param("bind") or options.bind, - timeout=_timeout(step, options), - cert=param("cert") or None, - key=param("key") or None, - callback_port=param("callback_port") or None, - callback_listen=param("callback_listen") or None, - callback_reply=param("callback_reply") or None, - callback_wait=_float(param("callback_wait"), 2.0), - raw=param("raw") or None, - source_port=_int(param("source_port"), 0), - ) - - -def _monitor(step, context, options): - param = _resolver(step, context) - send = [line.strip() for line in (param("send") or "").splitlines() - if line.strip()] - return xcatc.monitor( - server=param("server"), - send=send, - port=_int(param("port"), 3002), - bind=param("bind") or options.bind, - timeout=_timeout(step, options), - source_port=_int(param("source_port"), 0), - ) - - -def _flowrequest(step, context, options): - param = _resolver(step, context) - return xcatc.flowrequest( - server=param("server"), - port=_int(param("port"), 3001), - message=param("message") or "resourcerequest: xcatd", - bind=param("bind") or options.bind, - timeout=_timeout(step, options), - source_port=_int(param("source_port"), 0), - expected=_int(param("replies"), 1), - ) - - -def _findme(step, context, options): - param = _resolver(step, context) - payload = param("payload") - if not payload: - payload = xcatc.discovery_packet() - return xcatc.findme( - server=param("server"), - payload=payload, - port=_int(param("port"), 3001), - encoding=(param("encoding") or "gzip").lower(), - bind=param("bind") or options.bind, - source_port=_int(param("source_port"), 301), - timeout=_timeout(step, options), - callback_listen=_int(param("callback_listen"), 3001), - callback_wait=_float(param("callback_wait"), 5.0), - ) - - -def _extract(step, context, options): - """Pull a value out of an earlier reply, so a later step can fetch it. - - This is the join between stages. A grub2 config names a kernel; nothing in - the config file says what that name will be, and hard-coding it would make - the test assert xCAT's output against a copy of xCAT's output. Extracting - it and then fetching it asserts the only thing that matters: that what the - node was told to fetch can be fetched. - """ - param = _resolver(step, context) - source = param("from") or "" - pattern = step.param("pattern") or "" - try: - regex = re.compile(pattern, re.MULTILINE) - except re.error as exc: - raise ConfigError("bad regex %r: %s" % (pattern, exc), step.source) - - match = regex.search(source) - groups = list(match.groups()) if match else [] - index = _int(param("group"), 1 if groups else 0) - value = "" - if match: - if index > len(groups): - raise ConfigError( - "group=%d, but %r has %d capturing group(s)" - % (index, pattern, len(groups)), step.source) - value = match.group(index) or "" - - fields = { - "value": value, - "groups": [text or "" for text in groups], - "matched": bool(match), - "count": len(groups), - } - return Reply(kind="extract", fields=fields, ok=bool(match), - error="" if match else "%r matched nothing" % (pattern,), - sent="match %r" % (pattern,)) - - -def _sleep(step, context, options): - param = _resolver(step, context) - duration = _float(param("duration"), 1.0) - time.sleep(duration) - return Reply(kind="sleep", fields={}, ok=True, - sent="slept %.1fs" % (duration,)) - - -def _noop(step, context, options): - return Reply(kind="noop", fields={}, ok=True, sent="") - - -HANDLERS = { - "dns": _dns, - "tftp": _tftp, - "http": _http, - "xcatreq": _xcatreq, - "monitor": _monitor, - "flowrequest": _flowrequest, - "findme": _findme, - "extract": _extract, - "sleep": _sleep, - "noop": _noop, -} - -assert set(HANDLERS) == machine.known_step_types() - - -# --------------------------------------------------------------------------- -# parameter handling - - -def _resolver(step, context): - """A `param(key)` that resolves `$step.field` against what has run.""" - - def param(key, default=""): - value = step.param(key, default) - if value in (None, ""): - return value - return subst.resolve(value, context) - - return param - - -def _timeout(step, options): - return options.timeout if options.timeout is not None else step.timeout - - -def _int(value, default): - if value in (None, ""): - return default - try: - return int(str(value).strip(), 0) - except ValueError: - raise ConfigError("expected a number, got %r" % (value,)) - - -def _float(value, default): - if value in (None, ""): - return default - try: - return float(str(value).strip()) - except ValueError: - raise ConfigError("expected a number, got %r" % (value,)) - - -def _bool(value, default): - if value in (None, ""): - return default - text = str(value).strip().lower() - if text in ("1", "yes", "true", "on"): - return True - if text in ("0", "no", "false", "off"): - return False - raise ConfigError("expected a yes/no value, got %r" % (value,)) - - -def _pairs(block): - """`name = value` lines under an `element` key, as an ordered list.""" - pairs = [] - for line in (block or "").splitlines(): - text = line.strip() - if not text or text.startswith("#"): - continue - name, _, value = text.partition("=") - pairs.append((name.strip(), value.strip())) - return pairs diff --git a/xCAT-test/provtest/tests/context.py b/xCAT-test/provtest/tests/context.py deleted file mode 100644 index 310718b2c..000000000 --- a/xCAT-test/provtest/tests/context.py +++ /dev/null @@ -1,32 +0,0 @@ -"""Put `src/` on the path so the tests import the tool, not an installed copy. - -The suite runs out of a checkout with no install step, so the import has to be -made to work rather than assumed. Every test module imports this one first. -""" - -import os -import sys - -TESTS = os.path.dirname(os.path.realpath(__file__)) -ROOT = os.path.dirname(TESTS) -SRC = os.path.join(ROOT, "src") -LIB = os.path.join(SRC, "provtest_lib") -CONF = os.path.join(ROOT, "conf") - -if SRC not in sys.path: - sys.path.insert(0, SRC) - - -def conf_files(): - """Every shipped scenario file, sorted.""" - return sorted(os.path.join(CONF, name) for name in os.listdir(CONF) - if name.endswith(".conf")) - - -def modules(): - """Every module of the tool, as `(name, path)`.""" - found = [("provtest", os.path.join(SRC, "provtest"))] - for name in sorted(os.listdir(LIB)): - if name.endswith(".py"): - found.append((name[:-3], os.path.join(LIB, name))) - return found diff --git a/xCAT-test/provtest/tests/test_assertions.py b/xCAT-test/provtest/tests/test_assertions.py deleted file mode 100644 index 1c28e1f32..000000000 --- a/xCAT-test/provtest/tests/test_assertions.py +++ /dev/null @@ -1,165 +0,0 @@ -"""The assertion mini-language, checked without a server. - -Every case here is a line some shipped scenario writes, evaluated against a -reply built by hand. The point of the module is that a scenario's assertions -mean what their author thought they meant, and that is decidable offline. -""" - -import unittest - -import context # noqa: F401 (sys.path) - -from provtest_lib import assertions -from provtest_lib.errors import ConfigError -from provtest_lib.model import Reply -from provtest_lib.subst import Context - - -def reply(**fields): - return Reply(kind="test", fields=fields) - - -def check(line, target_reply, context=None, extras=None): - parsed = assertions.parse(line) - return assertions.evaluate(parsed, target_reply, context or Context(), - extras).ok - - -class ParseTests(unittest.TestCase): - - def test_target_operator_and_value(self): - parsed = assertions.parse("status == NOERROR") - self.assertEqual(parsed.target, "status") - self.assertEqual(parsed.op, "==") - self.assertEqual(parsed.value, "NOERROR") - - def test_value_keeps_its_spaces(self): - # `replies == resourcerequest: ok` is one value, not two. - parsed = assertions.parse("replies == resourcerequest: ok") - self.assertEqual(parsed.value, "resourcerequest: ok") - - def test_nullary_operators_take_no_value(self): - parsed = assertions.parse("error absent") - self.assertIsNone(parsed.value) - self.assertRaises(ConfigError, assertions.parse, "error absent junk") - - def test_unknown_operator_is_rejected(self): - self.assertRaises(ConfigError, assertions.parse, "status eq NOERROR") - - def test_bad_regex_is_rejected_at_parse_time(self): - self.assertRaises(ConfigError, assertions.parse, "text matches [") - - def test_block_skips_blanks_and_comments(self): - block = "\n# a comment\nstatus == 200\n\nsize > 0\n" - parsed = assertions.parse_block(block) - self.assertEqual([item.target for item in parsed], ["status", "size"]) - - -class ComparisonTests(unittest.TestCase): - - def test_equality_is_textual_by_default(self): - self.assertTrue(check("status == NOERROR", reply(status="NOERROR"))) - self.assertFalse(check("status == NOERROR", reply(status="NXDOMAIN"))) - - def test_equality_is_numeric_when_both_sides_are(self): - # curl reports a status as an int and a .conf writes it as text. - self.assertTrue(check("status == 200", reply(status=200))) - self.assertTrue(check("status == 200", reply(status="200"))) - - def test_equality_is_address_aware(self): - self.assertTrue(check("data == 10.99.1.11", reply(data="10.99.1.11 "))) - - def test_a_list_means_any_of_these(self): - # A name with two A records satisfies an assertion naming either. - two = reply(data=["10.99.1.11", "10.99.1.12"]) - self.assertTrue(check("data == 10.99.1.12", two)) - - def test_ordering_needs_numbers_on_both_sides(self): - self.assertTrue(check("size > 0", reply(size=512))) - self.assertFalse(check("size > 0", reply(size="not a number"))) - - def test_membership_of_a_cidr(self): - self.assertTrue(check("data in 10.99.1.0/24", reply(data="10.99.1.11"))) - self.assertFalse(check("data in 10.99.2.0/24", reply(data="10.99.1.11"))) - - def test_membership_of_a_range(self): - self.assertTrue(check("data in 10.99.1.1-10.99.1.20", - reply(data="10.99.1.11"))) - - def test_membership_of_a_list(self): - self.assertTrue(check("status in NOERROR, NXDOMAIN", - reply(status="NXDOMAIN"))) - - def test_substring_operators(self): - text = reply(text="linux /xcat/genesis.kernel destiny=install") - self.assertTrue(check("text contains destiny=install", text)) - self.assertTrue(check("text starts-with linux", text)) - self.assertTrue(check("text ends-with destiny=install", text)) - - def test_matches_is_multiline(self): - text = reply(text="first line\nlinux /boot/vmlinuz\n") - self.assertTrue(check(r"text matches ^linux\s+/boot", text)) - - -class PresenceTests(unittest.TestCase): - - def test_present_and_absent(self): - self.assertTrue(check("kernel present", reply(kernel="genesis.kernel"))) - self.assertFalse(check("kernel present", reply(kernel=""))) - self.assertTrue(check("error absent", reply(error=""))) - self.assertTrue(check("error absent", reply())) - - def test_an_empty_list_is_absence(self): - self.assertTrue(check("callbacks absent", reply(callbacks=[]))) - self.assertFalse(check("callbacks present", reply(callbacks=[]))) - - def test_a_positive_operator_fails_on_a_missing_field(self): - self.assertFalse(check("bootfile == pxelinux.0", reply())) - - def test_a_negative_operator_holds_on_a_missing_field(self): - # A reply naming no boot file has certainly not named the wrong one. - self.assertTrue(check("bootfile != pxelinux.0", reply())) - self.assertTrue(check("bootfile not-in a, b", reply())) - - -class IndexingTests(unittest.TestCase): - - def test_a_bare_list_name_is_its_first_element(self): - self.assertTrue(check("lines == done", reply(lines=["done", "ready"]))) - - def test_an_index_selects_an_element(self): - self.assertTrue(check("lines.1 == ready", reply(lines=["done", "ready"]))) - - def test_an_index_past_the_end_is_absence_not_an_error(self): - self.assertFalse(check("lines.5 == ready", reply(lines=["done"]))) - - def test_a_mapping_is_addressed_by_key(self): - headers = reply(header={"content-type": "text/plain"}) - self.assertTrue(check("header.content-type == text/plain", headers)) - - -class ReferenceTests(unittest.TestCase): - - def test_a_reference_resolves_against_an_earlier_reply(self): - ctx = Context() - ctx.bind("overtftp", reply(sha256="abc123")) - self.assertTrue(check("sha256 == $overtftp.sha256", - reply(sha256="abc123"), ctx)) - self.assertFalse(check("sha256 == $overtftp.sha256", - reply(sha256="different"), ctx)) - - def test_a_reference_to_a_step_that_has_not_run_is_an_error(self): - parsed = assertions.parse("sha256 == $nosuchstep.sha256") - self.assertRaises(ConfigError, assertions.evaluate, parsed, - reply(sha256="abc"), Context()) - - -class MetaTargetTests(unittest.TestCase): - - def test_attempt_comes_from_the_runner_not_the_reply(self): - self.assertTrue(check("attempt == 1", reply(), None, {"attempt": 1})) - self.assertFalse(check("attempt == 1", reply())) - - -if __name__ == "__main__": - unittest.main() diff --git a/xCAT-test/provtest/tests/test_boundaries.py b/xCAT-test/provtest/tests/test_boundaries.py deleted file mode 100644 index eb6e143ac..000000000 --- a/xCAT-test/provtest/tests/test_boundaries.py +++ /dev/null @@ -1,220 +0,0 @@ -"""Rules about this tool's own code, not about xCAT. - -Two of them, and both are the reason the results mean anything: - - * only `proc.py` starts a process, so nothing a `.conf` supplies can become a - command; - * no module reads the xCAT database or runs an xCAT command, so what the - suite expects never comes from the same place as what it is testing. - -A suite that asked xCAT what to expect would pass on a cluster that could not -boot a single node. That is not a style rule, so it is asserted rather than -written down in the README and hoped for. -""" - -import ast -import re -import unittest - -import context # noqa: F401 (sys.path) - - -#: The only module allowed to start a child process. -SPAWNER = "proc" - -#: Modules and callables that start one. -SPAWNING_MODULES = frozenset(["subprocess", "pty", "multiprocessing"]) -SPAWNING_CALLS = frozenset(["system", "popen", "spawn", "spawnl", "spawnv", - "execv", "execve", "execvp", "fork", "forkpty"]) - -#: Database drivers. None of them belongs here: the tool is a client of the -#: wire, not a reader of the rows that produced it. -DATABASE_MODULES = frozenset([ - "sqlite3", "psycopg2", "psycopg", "pymysql", "MySQLdb", "mysql", - "sqlalchemy", "dbm", "shelve", -]) - -#: xCAT commands. A string naming one in executable code means the tool asked -#: xCAT what to expect instead of being told by the fixture. `xcatd` is not in -#: the list: it is the daemon under test, and naming it is unavoidable. -XCAT_NAMES = re.compile( - r"\b(lsdef|mkdef|chdef|rmdef|nodels|nodech|nodeadd|tabdump|tabedit|" - r"chtab|gettab|nodeset|makedns|makedhcp|makehosts|makeknownhosts|" - r"xdsh|xdcp|rinstall|rpower|rcons|copycds|genimage|packimage|" - r"xcatprobe|xcatconfig)\b") - -#: Where those commands and the database live. -XCAT_PATHS = re.compile(r"/opt/xcat/(bin|sbin)|/etc/xcat/|xcatdb|site\.tab") - - -def parse(path): - with open(path) as handle: - return ast.parse(handle.read(), filename=path) - - -def imported_names(tree): - """Every module name an import statement brings in.""" - names = set() - for node in ast.walk(tree): - if isinstance(node, ast.Import): - for alias in node.names: - names.add(alias.name.split(".")[0]) - elif isinstance(node, ast.ImportFrom): - # `from . import model` is this package importing itself, which is - # not a dependency on anything. - if node.level == 0 and node.module: - names.add(node.module.split(".")[0]) - return names - - -def strings_in_code(tree): - """Every string literal that is not a docstring. - - Docstrings are excluded because they explain the behaviour under test and - routinely name the commands whose output this tool must not depend on -- - saying so is the opposite of doing it. - """ - docstrings = set() - for node in ast.walk(tree): - if isinstance(node, (ast.Module, ast.ClassDef, ast.FunctionDef, - ast.AsyncFunctionDef)): - body = getattr(node, "body", None) - if body and isinstance(body[0], ast.Expr) and \ - isinstance(body[0].value, ast.Constant) and \ - isinstance(body[0].value.value, str): - docstrings.add(id(body[0].value)) - - found = [] - for node in ast.walk(tree): - if isinstance(node, ast.Constant) and isinstance(node.value, str): - if id(node) not in docstrings: - found.append(node.value) - return found - - -def called_attributes(tree): - """Every `something.name(...)` called, as its attribute name.""" - names = [] - for node in ast.walk(tree): - if isinstance(node, ast.Call): - func = node.func - if isinstance(func, ast.Attribute): - names.append(func.attr) - elif isinstance(func, ast.Name): - names.append(func.id) - return names - - -class SpawningTests(unittest.TestCase): - - def test_only_proc_imports_a_process_module(self): - for name, path in context.modules(): - found = imported_names(parse(path)) & SPAWNING_MODULES - if name == SPAWNER: - continue - self.assertEqual(found, set(), - "%s imports %s; only %s.py may start a process" - % (name, ", ".join(sorted(found)), SPAWNER)) - - def test_no_module_calls_a_spawning_function(self): - for name, path in context.modules(): - if name == SPAWNER: - continue - found = set(called_attributes(parse(path))) & SPAWNING_CALLS - self.assertEqual(found, set(), - "%s calls %s" % (name, ", ".join(sorted(found)))) - - def test_proc_never_uses_a_shell(self): - # An argument list, never a command string: nothing a .conf supplies - # may be interpreted by a shell. - path = dict(context.modules())[SPAWNER] - for node in ast.walk(parse(path)): - if isinstance(node, ast.keyword) and node.arg == "shell": - value = getattr(node.value, "value", True) - self.assertFalse(value, "proc.py passes shell=True") - - -class DatabaseTests(unittest.TestCase): - - def test_no_module_imports_a_database_driver(self): - for name, path in context.modules(): - found = imported_names(parse(path)) & DATABASE_MODULES - self.assertEqual(found, set(), - "%s imports %s" % (name, ", ".join(sorted(found)))) - - -class XcatTests(unittest.TestCase): - - def test_no_module_names_an_xcat_command_in_executable_code(self): - for name, path in context.modules(): - for text in strings_in_code(parse(path)): - match = XCAT_NAMES.search(text) - self.assertIsNone( - match, - "%s has %r in executable code; what this suite expects " - "must come from --set, never from xCAT itself" - % (name, match.group(0) if match else "")) - - def test_no_module_names_an_xcat_path(self): - for name, path in context.modules(): - for text in strings_in_code(parse(path)): - self.assertIsNone(XCAT_PATHS.search(text), - "%s reaches into xCAT's own files: %r" - % (name, text)) - - def test_no_step_type_can_run_a_command(self): - # The reason a .conf is safe to read from anywhere: the step types are - # a closed list of protocols, and not one of them takes a program to - # run. A scenario cannot execute anything, so it cannot ask xCAT. - from provtest_lib import machine - - protocols = frozenset(["dns", "tftp", "http", "xcatreq", "monitor", - "flowrequest", "findme"]) - offline = frozenset(["extract", "sleep", "noop"]) - self.assertEqual(machine.known_step_types(), protocols | offline) - for name in machine.known_step_types(): - for key in machine.allowed_keys(name): - self.assertNotIn(key, ("command_line", "exec", "shell", - "program", "script")) - - def test_the_shipped_scenarios_use_only_declared_types(self): - from provtest_lib import config, machine - - scenarios, _ = config.load(context.conf_files(), {}, raw=True) - for scenario in scenarios: - for step in scenario.steps: - self.assertIn(step.type, machine.known_step_types(), - "%s step %s" % (scenario.name, step.name)) - - -class ImportTests(unittest.TestCase): - - def test_nothing_outside_the_standard_library_is_imported(self): - allowed = set(["provtest_lib", "argparse", "ast", "base64", "binascii", - "configparser", "errno", "gzip", "hashlib", "io", - "ipaddress", "json", "os", "re", "select", "shutil", - "socket", "ssl", "struct", "subprocess", "sys", - "tempfile", "threading", "time", "xml"]) - for name, path in context.modules(): - for imported in imported_names(parse(path)): - if imported in ("", "."): - continue - self.assertIn(imported, allowed, - "%s imports %s, which is not in the standard " - "library set this tool is allowed" % (name, imported)) - - def test_listing_a_file_never_reaches_the_runner(self): - # `validate` and `list` run in a checkout with no root and no network, - # so the module that opens sockets is imported inside the run command - # and nowhere else. - path = dict(context.modules())["cli"] - top_level = set() - for node in parse(path).body: - if isinstance(node, (ast.Import, ast.ImportFrom)): - for alias in node.names: - top_level.add(alias.name) - self.assertNotIn("runner", top_level) - - -if __name__ == "__main__": - unittest.main() diff --git a/xCAT-test/provtest/tests/test_config.py b/xCAT-test/provtest/tests/test_config.py deleted file mode 100644 index bba4b0fb7..000000000 --- a/xCAT-test/provtest/tests/test_config.py +++ /dev/null @@ -1,206 +0,0 @@ -"""Reading .conf files: sections, defaults, variables, and what is rejected. - -Every scenario in this suite is data, so a mistake in a file has to fail loudly -at load time. Silence would mean a step that never ran counted as a pass. -""" - -import os -import shutil -import tempfile -import unittest - -import context # noqa: F401 (sys.path) - -from provtest_lib import config -from provtest_lib.errors import ConfigError - - -BASIC = """ -[vars] -domain = provtest.cluster - -[defaults] -server = 10.99.1.1 -timeout = 8 - -[scenario node-reverse] -description = The node's address resolves to its name - -[step ptr] -type = dns -name = 11.1.99.10.in-addr.arpa -rrtype = PTR -assert = - status == NOERROR - data contains provtestcn.%(domain)s -""" - - -class Harness(unittest.TestCase): - - def setUp(self): - self.dir = tempfile.mkdtemp(prefix="provtest-config-") - self.addCleanup(shutil.rmtree, self.dir, True) - - def write(self, text, name="case.conf"): - path = os.path.join(self.dir, name) - with open(path, "w") as handle: - handle.write(text) - return path - - def load(self, text, overrides=None, raw=False, name="case.conf"): - return config.load([self.write(text, name)], overrides or {}, raw) - - -class StructureTests(Harness): - - def test_a_file_loads_into_scenarios_and_steps(self): - scenarios, variables = self.load(BASIC) - self.assertEqual(len(scenarios), 1) - scenario = scenarios[0] - self.assertEqual(scenario.name, "node-reverse") - self.assertEqual([step.name for step in scenario.steps], ["ptr"]) - self.assertEqual(variables["domain"], "provtest.cluster") - - def test_defaults_reach_a_step_that_does_not_set_them(self): - scenario = self.load(BASIC)[0][0] - self.assertEqual(scenario.steps[0].param("server"), "10.99.1.1") - self.assertEqual(scenario.steps[0].timeout, 8.0) - - def test_a_step_overrides_a_default(self): - text = BASIC.replace("rrtype = PTR", "rrtype = PTR\nserver = 10.99.1.9") - scenario = self.load(text)[0][0] - self.assertEqual(scenario.steps[0].param("server"), "10.99.1.9") - - def test_a_default_a_step_type_cannot_use_is_dropped_quietly(self): - # One [defaults] serves dns, tftp and http steps in the same file, so - # `path =` there must not make the dns step illegal. - text = BASIC.replace("timeout = 8", "timeout = 8\npath = /install/x") - scenario = self.load(text)[0][0] - self.assertIsNone(scenario.steps[0].param("path")) - - def test_steps_keep_their_file_order(self): - text = BASIC + "\n[step second]\ntype = noop\n\n[step third]\ntype = noop\n" - scenario = self.load(text)[0][0] - self.assertEqual([step.name for step in scenario.steps], - ["ptr", "second", "third"]) - - def test_assertions_are_parsed_into_the_step(self): - scenario = self.load(BASIC)[0][0] - targets = [item.target for item in scenario.steps[0].assertions] - self.assertEqual(targets, ["status", "data"]) - - -class VariableTests(Harness): - - def test_a_variable_is_interpolated_from_vars(self): - scenario = self.load(BASIC)[0][0] - value = scenario.steps[0].assertions[1].value - self.assertEqual(value, "provtestcn.provtest.cluster") - - def test_an_override_beats_vars(self): - scenario = self.load(BASIC, {"domain": "other.cluster"})[0][0] - self.assertEqual(scenario.steps[0].assertions[1].value, - "provtestcn.other.cluster") - - def test_an_undefined_variable_names_the_set_that_would_fix_it(self): - text = BASIC.replace("%(domain)s", "%(nosuchvar)s") - try: - self.load(text) - except ConfigError as exc: - self.assertIn("--set nosuchvar=", str(exc)) - else: - self.fail("an undefined variable was accepted") - - def test_raw_leaves_variables_alone(self): - # `validate` runs in a checkout where no value has been supplied yet. - scenario = self.load(BASIC, raw=True)[0][0] - self.assertIn("%(domain)s", scenario.steps[0].assertions[1].value) - - def test_a_variable_never_becomes_a_step_key(self): - # --set path=/install must not silently become every step's path=. - scenario = self.load(BASIC, {"path": "/install"})[0][0] - self.assertIsNone(scenario.steps[0].param("path")) - - def test_a_dollar_reference_survives_untouched(self): - text = BASIC + "\n[step after]\ntype = extract\nfrom = $ptr.data\npattern = (.*)\n" - scenario = self.load(text)[0][0] - self.assertEqual(scenario.steps[1].param("from"), "$ptr.data") - - -class RejectionTests(Harness): - - def test_an_unknown_step_key_is_rejected(self): - text = BASIC.replace("rrtype = PTR", "rrtype = PTR\nnosuchkey = 1") - self.assertRaises(ConfigError, self.load, text) - - def test_an_unknown_step_type_is_rejected(self): - self.assertRaises(ConfigError, self.load, - BASIC.replace("type = dns", "type = smtp")) - - def test_a_step_with_no_type_is_rejected(self): - self.assertRaises(ConfigError, self.load, - BASIC.replace("type = dns\n", "")) - - def test_a_step_before_any_scenario_is_rejected(self): - self.assertRaises(ConfigError, self.load, - "[step orphan]\ntype = noop\n") - - def test_a_scenario_with_no_steps_is_rejected(self): - self.assertRaises(ConfigError, self.load, - "[scenario empty]\ndescription = nothing\n") - - def test_an_unknown_section_is_rejected(self): - self.assertRaises(ConfigError, self.load, BASIC + "\n[nonsense]\nx = 1\n") - - def test_type_may_not_be_a_default(self): - # A default type would turn a misspelt type= into another protocol. - self.assertRaises(ConfigError, self.load, - BASIC.replace("[defaults]", "[defaults]\ntype = dns")) - - def test_a_repeated_step_name_in_one_file_is_rejected(self): - # configparser's own rule, and the reason step names are unique per - # file rather than per scenario. - self.assertRaises(ConfigError, self.load, - BASIC + "\n[scenario other]\n\n[step ptr]\ntype = noop\n") - - def test_a_scenario_defined_in_two_files_is_rejected(self): - first = self.write(BASIC, "first.conf") - second = self.write(BASIC, "second.conf") - self.assertRaises(ConfigError, config.load, [first, second], {}, True) - - def test_a_missing_file_says_so(self): - self.assertRaises(ConfigError, config.load, - [os.path.join(self.dir, "absent.conf")], {}, True) - - def test_a_non_numeric_timeout_is_rejected(self): - self.assertRaises(ConfigError, self.load, - BASIC.replace("timeout = 8", "timeout = soon")) - - -class ShippedFileTests(unittest.TestCase): - """The files this suite ships must load and validate as they stand.""" - - def test_every_shipped_file_parses(self): - from provtest_lib import machine - - paths = context.conf_files() - self.assertTrue(paths, "no .conf files are shipped") - scenarios, _ = config.load(paths, {}, raw=True) - problems = [] - for scenario in scenarios: - problems.extend(machine.validate_scenario(scenario)) - self.assertEqual(problems, []) - - def test_every_shipped_scenario_has_a_description_and_an_assertion(self): - scenarios, _ = config.load(context.conf_files(), {}, raw=True) - for scenario in scenarios: - self.assertTrue(scenario.description, - "%s has no description" % (scenario.name,)) - asserted = any(step.assertions for step in scenario.steps) - self.assertTrue(asserted, - "%s asserts nothing at all" % (scenario.name,)) - - -if __name__ == "__main__": - unittest.main() diff --git a/xCAT-test/provtest/tests/test_machine.py b/xCAT-test/provtest/tests/test_machine.py deleted file mode 100644 index ae873d1f5..000000000 --- a/xCAT-test/provtest/tests/test_machine.py +++ /dev/null @@ -1,173 +0,0 @@ -"""The offline half: what a step of each type may say, and what it returns. - -`validate` runs this in a checkout with no network and no root, which is where -a misspelt reply field or a reference to a step that has not run yet should be -caught -- long before a management node is involved. -""" - -import unittest - -import context # noqa: F401 (sys.path) - -from provtest_lib import machine -from provtest_lib.assertions import parse_block -from provtest_lib.errors import ConfigError -from provtest_lib.model import Scenario, Step - - -def step(name, type, assertions="", **params): - return Step(name=name, type=type, expect=params.pop("expect", None), - params=params, assertions=parse_block(assertions), - source="test") - - -def scenario(*steps): - return Scenario(name="case", description="d", steps=steps, source="test") - - -def problems(*steps): - return machine.validate_scenario(scenario(*steps)) - - -class TypeTests(unittest.TestCase): - - def test_every_shipped_type_is_declared_everywhere(self): - # A type with no key table or no reply table would validate nothing. - for name in machine.known_step_types(): - self.assertIn(name, machine.STEP_KEYS) - self.assertIn(name, machine.REPLY_FIELDS) - self.assertIn(name, machine.REQUIRED_KEYS) - - def test_an_unknown_type_is_rejected(self): - self.assertRaises(ConfigError, machine.check_type, "smtp") - - def test_bind_is_available_to_every_type(self): - # xcatd names a client by the reverse lookup of the address a request - # arrived from, so no step type may be unable to choose one. - for name in machine.known_step_types(): - self.assertIn("bind", machine.allowed_keys(name)) - - -class RequiredKeyTests(unittest.TestCase): - - def test_a_dns_step_needs_a_name(self): - found = problems(step("q", "dns", server="10.99.1.1")) - self.assertTrue(any("needs name=" in text for text in found), found) - - def test_a_tftp_step_needs_a_path(self): - found = problems(step("f", "tftp", server="10.99.1.1")) - self.assertTrue(any("needs path=" in text for text in found), found) - - def test_an_http_step_needs_a_url_or_a_server_and_path(self): - self.assertTrue(problems(step("g", "http"))) - self.assertFalse(problems(step("g", "http", url="http://h/x"))) - self.assertFalse(problems(step("g", "http", server="h", path="/x"))) - - def test_an_xcatreq_step_needs_a_command(self): - found = problems(step("r", "xcatreq", server="10.99.1.1")) - self.assertTrue(any("needs command=" in text for text in found), found) - - def test_an_extract_step_needs_a_source_and_a_pattern(self): - self.assertTrue(problems(step("e", "extract", pattern="(x)"))) - self.assertTrue(problems(step("e", "extract", **{"from": "$a.text"}))) - - def test_an_extract_group_must_be_a_number(self): - found = problems(step("e", "extract", pattern="(x)", group="first", - **{"from": "$a.text"})) - self.assertTrue(any("group= must be a number" in t for t in found), found) - - -class ExpectationTests(unittest.TestCase): - - def test_the_three_expectations_are_accepted(self): - for value in ("ok", "fail", "any"): - self.assertFalse(problems( - step("s", "tftp", server="h", path="p", expect=value))) - - def test_an_unknown_expectation_is_rejected(self): - found = problems(step("s", "tftp", server="h", path="p", expect="maybe")) - self.assertTrue(any("unknown expect=" in text for text in found), found) - - -class AssertionTargetTests(unittest.TestCase): - - def test_a_field_the_type_produces_is_accepted(self): - self.assertFalse(problems( - step("f", "tftp", "size > 0\nsha256 present", server="h", path="p"))) - - def test_a_field_of_another_type_is_rejected(self): - # `status` belongs to dns and http; a tftp reply has no such field, and - # an assertion naming it would silently never hold. - found = problems(step("f", "tftp", "status == 200", server="h", path="p")) - self.assertTrue(any("has no field" in text for text in found), found) - - def test_a_mapping_target_is_accepted_by_key(self): - self.assertFalse(problems( - step("g", "http", "header.content-type == text/plain", - url="http://h/x"))) - - def test_a_mapping_target_of_the_wrong_type_is_rejected(self): - found = problems(step("f", "tftp", "header.x == y", server="h", path="p")) - self.assertTrue(any("carries no header" in text for text in found), found) - - def test_an_indexed_list_target_is_accepted(self): - self.assertFalse(problems( - step("m", "monitor", "lines.0 == done", server="h", send="x"))) - - -class ReferenceTests(unittest.TestCase): - - def test_a_reference_backwards_is_accepted(self): - self.assertFalse(problems( - step("config", "tftp", "size > 0", server="h", path="p"), - step("kernel", "extract", "matched == yes", - pattern="(x)", **{"from": "$config.text"}))) - - def test_a_reference_forwards_is_rejected(self): - found = problems( - step("kernel", "extract", "matched == yes", - pattern="(x)", **{"from": "$config.text"}), - step("config", "tftp", "size > 0", server="h", path="p")) - self.assertTrue(any("has not run yet" in text for text in found), found) - - def test_a_reference_to_no_known_field_is_rejected(self): - found = problems( - step("config", "tftp", "size > 0", server="h", path="p"), - step("kernel", "tftp", "size > 0", server="h", - path="$config.nosuchfield")) - self.assertTrue(any("no reply field" in text for text in found), found) - - def test_a_reference_in_an_assertion_value_is_checked_too(self): - found = problems( - step("a", "http", "status == 200", url="http://h/x"), - step("b", "http", "sha256 == $a.nosuchfield", url="http://h/y")) - self.assertTrue(any("no reply field" in text for text in found), found) - - def test_a_step_defined_twice_is_reported(self): - found = problems(step("f", "noop"), step("f", "noop")) - self.assertTrue(any("defined twice" in text for text in found), found) - - -class RequiredVariableTests(unittest.TestCase): - - def test_variables_are_collected_from_keys_and_assertions(self): - found = machine.required_variables([scenario( - step("f", "tftp", "text contains %(node)s", - server="%(server)s", path="%(knownfile)s"))]) - self.assertEqual(found, set(["server", "knownfile", "node"])) - - def test_every_shipped_file_declares_what_it_needs(self): - # `validate` prints this list, and the fixture passes exactly it; a - # variable nobody supplies is a run that stops before its first socket. - from provtest_lib import config - - scenarios, _ = config.load(context.conf_files(), {}, raw=True) - needed = machine.required_variables(scenarios) - self.assertIn("server", needed) - self.assertIn("client", needed) - for name in needed: - self.assertRegex(name, r"^[a-z][a-z0-9_]*$") - - -if __name__ == "__main__": - unittest.main() diff --git a/xCAT-test/provtest/tests/test_report.py b/xCAT-test/provtest/tests/test_report.py deleted file mode 100644 index 943dc9000..000000000 --- a/xCAT-test/provtest/tests/test_report.py +++ /dev/null @@ -1,185 +0,0 @@ -"""Output, and the address arithmetic the scenarios rely on. - -CI reads TAP and nothing else, so the shape of a failing line is as much a part -of this tool's contract as any assertion. A failure has to be understandable -from the log alone: by the time anyone reads a CI result, the network it ran on -no longer exists. -""" - -import io -import unittest - -import context # noqa: F401 (sys.path) - -from provtest_lib import netutil, report -from provtest_lib.errors import EXIT_FAILED, EXIT_OK -from provtest_lib.model import Reply - - -def record(ok=True, **kwargs): - kwargs.setdefault("scenario", "node-reverse") - kwargs.setdefault("step", "ptr") - kwargs.setdefault("description", "the address resolves to its name") - return report.Record(ok=ok, **kwargs) - - -def run(reporter, records): - reporter.start() - for item in records: - reporter.add(item) - reporter.finish() - return reporter.stream.getvalue() - - -class TapTests(unittest.TestCase): - - def tap(self, records): - return run(report.TapReporter(io.StringIO()), records) - - def test_a_pass_is_one_ok_line(self): - text = self.tap([record()]) - self.assertIn("TAP version 13\n", text) - self.assertIn("ok 1 - node-reverse/ptr: the address resolves to its name", - text) - self.assertIn("1..1\n", text) - - def test_the_plan_counts_every_record(self): - text = self.tap([record(), record(), record()]) - self.assertIn("1..3\n", text) - self.assertIn("# passed 3, failed 0, skipped 0\n", text) - - def test_a_failure_says_not_ok_and_explains_itself(self): - text = self.tap([record( - ok=False, expected="provtestcn.provtest.cluster", - actual="(no answer records)", - sent="dig @10.99.1.1 -b 10.99.1.11 PTR 11.1.99.10.in-addr.arpa", - reply=Reply(kind="dns", fields={"status": "NXDOMAIN"}, ok=False))]) - self.assertIn("not ok 1 - ", text) - self.assertIn("---", text) - self.assertIn("...", text) - # The command as it was sent: the first thing an operator does after a - # failure is run it again by hand. - self.assertIn("dig @10.99.1.1 -b 10.99.1.11 PTR", text) - self.assertIn("provtestcn.provtest.cluster", text) - self.assertIn("no answer records", text) - - def test_a_pass_carries_no_diagnostic(self): - self.assertNotIn("---", self.tap([record()])) - - def test_a_skip_is_ok_with_a_reason(self): - text = self.tap([record(ok=False, skip_reason="tftp is not installed")]) - self.assertIn("ok 1 - ", text) - self.assertIn("# SKIP tftp is not installed", text) - self.assertNotIn("not ok", text) - - def test_a_skip_counts_as_neither_a_pass_nor_a_failure(self): - reporter = report.TapReporter(io.StringIO()) - run(reporter, [record(), record(ok=False, skip_reason="no client")]) - self.assertEqual((reporter.passed, reporter.failed, reporter.skipped), - (1, 0, 1)) - - -class ExitCodeTests(unittest.TestCase): - - def test_all_passing_is_zero(self): - reporter = report.TapReporter(io.StringIO()) - run(reporter, [record(), record()]) - self.assertEqual(reporter.exit_code(), EXIT_OK) - - def test_one_failure_is_one(self): - reporter = report.TapReporter(io.StringIO()) - run(reporter, [record(), record(ok=False)]) - self.assertEqual(reporter.exit_code(), EXIT_FAILED) - - def test_a_skip_alone_is_not_a_failure(self): - reporter = report.TapReporter(io.StringIO()) - run(reporter, [record(ok=False, skip_reason="no client")]) - self.assertEqual(reporter.exit_code(), EXIT_OK) - - -class OtherFormatTests(unittest.TestCase): - - def test_every_declared_format_can_be_built(self): - for name in report.REPORTERS: - self.assertIsNotNone(report.make(name, io.StringIO())) - - def test_an_unknown_format_is_refused(self): - self.assertRaises(ValueError, report.make, "xml", io.StringIO()) - - def test_pretty_groups_by_scenario_and_summarises(self): - text = run(report.PrettyReporter(io.StringIO()), - [record(), record(ok=False, step="a", description="d")]) - self.assertIn("PASS ptr", text) - self.assertIn("FAIL a", text) - self.assertIn("total=2 passed=1 failed=1 skipped=0", text) - - def test_json_is_one_parsable_object(self): - import json - - text = run(report.JsonReporter(io.StringIO()), - [record(), record(ok=False)]) - payload = json.loads(text) - self.assertEqual(payload["total"], 2) - self.assertEqual(payload["failed"], 1) - self.assertEqual(len(payload["records"]), 2) - - -class AddressTests(unittest.TestCase): - """The encodings a loader asks for its files by.""" - - def test_hex_ip_is_what_a_loader_asks_for(self): - # 10.99.1.11 is grub.cfg-0A63010B. A digit wrong here produces no error - # anywhere, just a node sitting at the loader. - self.assertEqual(netutil.hex_ip("10.99.1.11"), "0A63010B") - self.assertEqual(netutil.hex_ip("10.99.1.11", upper=False), "0a63010b") - - def test_hex_ip_of_a_network_address(self): - self.assertEqual(netutil.hex_ip("10.99.1.0"), "0A630100") - - def test_a_per_network_config_is_named_by_the_prefix_only(self): - # 10.99.1.0/24 is 0A6301, not 0A630100: the file is named by as many - # digits as the netmask covers. Asking for all eight gets a machine - # with no definition nothing at all, which is the whole failure this - # stage exists to catch. - self.assertEqual(netutil.hex_net("10.99.1.0", 24), "0A6301") - self.assertEqual(netutil.hex_net("10.99.0.0", 16), "0A63") - self.assertEqual(netutil.hex_net("10.99.1.0", 26), "0A63010") - self.assertEqual(netutil.hex_net("10.99.1.0", 24, upper=False), "0a6301") - - def test_a_prefix_that_is_not_one_is_refused(self): - self.assertRaises(ValueError, netutil.hex_net, "10.99.1.0", 33) - self.assertRaises(ValueError, netutil.hex_net, "10.99.1.0", "wide") - - def test_hex_ip_rejects_what_is_not_an_address(self): - self.assertRaises(ValueError, netutil.hex_ip, "provtestcn") - - def test_the_reverse_name_is_the_one_dig_is_asked_for(self): - self.assertEqual(netutil.reverse_name("10.99.1.11"), - "11.1.99.10.in-addr.arpa") - - def test_a_mac_normalises_to_the_colon_form(self): - for text in ("52:54:00:DC:11:01", "52-54-00-dc-11-01", - "52.54.00.dc.11.01", "52:54:0:dc:11:1"): - self.assertEqual(netutil.normalise_mac(text), "52:54:00:dc:11:01") - - def test_a_mac_becomes_the_dashed_form_a_config_is_named_by(self): - self.assertEqual(netutil.dashed_mac("52:54:00:DC:11:01"), - "52-54-00-dc-11-01") - - def test_a_mac_of_the_wrong_length_is_refused(self): - self.assertRaises(ValueError, netutil.normalise_mac, "52:54:00:dc:11") - self.assertRaises(ValueError, netutil.normalise_mac, "not a mac") - - def test_membership_helpers(self): - self.assertTrue(netutil.ip_in("10.99.1.11", "10.99.1.0/24")) - self.assertFalse(netutil.ip_in("10.99.2.11", "10.99.1.0/24")) - self.assertTrue(netutil.ip_in_range("10.99.1.11", "10.99.1.1-10.99.1.20")) - self.assertFalse(netutil.ip_in_range("10.99.1.21", "10.99.1.1-10.99.1.20")) - - def test_nonsense_is_not_a_member_of_anything(self): - self.assertFalse(netutil.ip_in("provtestcn", "10.99.1.0/24")) - self.assertFalse(netutil.ip_in("10.99.1.11", "not a network")) - - -if __name__ == "__main__": - unittest.main() diff --git a/xCAT-test/provtest/tests/test_subst.py b/xCAT-test/provtest/tests/test_subst.py deleted file mode 100644 index 59fbbde84..000000000 --- a/xCAT-test/provtest/tests/test_subst.py +++ /dev/null @@ -1,102 +0,0 @@ -"""`$step.field`: the references that make a scenario a chain. - -A cross-stage scenario fetches a config, reads a name out of it, and fetches -that name. The reference is what carries the second half, so it is resolved at -run time and not by configparser -- and the two syntaxes have to share a file -without escaping each other. -""" - -import unittest - -import context # noqa: F401 (sys.path) - -from provtest_lib import subst -from provtest_lib.errors import ConfigError -from provtest_lib.model import Reply - - -def reply(**fields): - return Reply(kind="test", fields=fields) - - -class ScanningTests(unittest.TestCase): - - def test_references_are_found(self): - found = subst.references("http://%(server)s:$configport.value/x") - self.assertEqual(found, [("configport", "value")]) - - def test_a_trailing_dot_is_not_part_of_the_field(self): - self.assertEqual(subst.references("$config.text."), [("config", "text")]) - - def test_an_indexed_field_is_one_reference(self): - self.assertEqual(subst.references("$mon.lines.0"), [("mon", "lines.0")]) - - def test_variables_are_found_separately(self): - self.assertEqual(subst.variables("%(server)s and %(client)s"), - set(["server", "client"])) - - def test_the_two_syntaxes_do_not_see_each_other(self): - text = "%(server)s/$config.text" - self.assertEqual(subst.variables(text), set(["server"])) - self.assertEqual(subst.references(text), [("config", "text")]) - - def test_nothing_to_find_is_not_an_error(self): - self.assertEqual(subst.references(None), []) - self.assertEqual(subst.variables(None), set()) - - -class ResolutionTests(unittest.TestCase): - - def setUp(self): - self.ctx = subst.Context() - self.ctx.bind("config", reply(text="linux /xcat/vmlinuz\n", - sha256="abc123")) - - def test_a_reference_is_replaced_by_the_field(self): - self.assertEqual(subst.resolve("$config.sha256", self.ctx), "abc123") - - def test_a_reference_is_replaced_inside_a_larger_string(self): - self.assertEqual(subst.resolve("sha=$config.sha256!", self.ctx), - "sha=abc123!") - - def test_an_unbound_step_says_it_has_not_run(self): - try: - subst.resolve("$later.value", self.ctx) - except ConfigError as exc: - self.assertIn("no step of that name has run yet", str(exc)) - else: - self.fail("a forward reference resolved") - - def test_a_field_the_reply_does_not_carry_is_an_error(self): - self.assertRaises(ConfigError, subst.resolve, "$config.kernel", self.ctx) - - def test_rebinding_a_name_keeps_one_entry(self): - self.ctx.bind("config", reply(sha256="second")) - self.assertEqual(self.ctx.order.count("config"), 1) - self.assertEqual(subst.resolve("$config.sha256", self.ctx), "second") - - -class FormattingTests(unittest.TestCase): - """How a resolved value is rendered: the way a .conf author would type it.""" - - def test_a_boolean_is_yes_or_no(self): - self.assertEqual(subst.format_value(True), "yes") - self.assertEqual(subst.format_value(False), "no") - - def test_bytes_are_decoded(self): - self.assertEqual(subst.format_value(b"ready"), "ready") - - def test_a_list_is_joined(self): - self.assertEqual(subst.format_value(["a", "b"]), "a, b") - - def test_a_mapping_is_sorted_and_joined(self): - self.assertEqual(subst.format_value({"b": 2, "a": 1}), "a=1, b=2") - - def test_a_reference_to_a_list_gives_its_first_element(self): - ctx = subst.Context() - ctx.bind("dns", reply(data=["10.99.1.11", "10.99.1.12"])) - self.assertEqual(subst.resolve("$dns.data", ctx), "10.99.1.11") - - -if __name__ == "__main__": - unittest.main() diff --git a/xCAT-test/xCAT-test.spec b/xCAT-test/xCAT-test.spec index fe7cd183c..1511e4433 100644 --- a/xCAT-test/xCAT-test.spec +++ b/xCAT-test/xCAT-test.spec @@ -21,11 +21,11 @@ BuildRequires: perl-Pod-Html BuildArch: noarch %endif -# dhcptest needs scapy, which lives in EPEL on EL rather than in the base -# repositories. A hard Requires: would make this package uninstallable on a -# management node without EPEL, so it is only recommended: dhcptest exits 3 -# with an actionable message when scapy is absent, and nothing else in -# xCAT-test needs it. Recommends: is not understood by rpm before EL 8. +# blackboxtest's DHCP steps need scapy, which lives in EPEL on EL. A hard +# Requires: would make this package uninstallable on a management node without +# EPEL, so it is only recommended: blackboxtest exits 3 with a message naming +# the package when a DHCP step finds no scapy. rpm before EL 8 has no +# Recommends:. %if 0%{?suse_version} || 0%{?rhel} >= 8 Recommends: python3-scapy %endif @@ -73,13 +73,8 @@ chmod 644 $RPM_BUILD_ROOT/%{prefix}/share/doc/man1/* cp -r autotest $RPM_BUILD_ROOT/%{prefix}/share/xcat/tools cp -r unit $RPM_BUILD_ROOT/%{prefix}/share/xcat/tools/autotest cp -r integration $RPM_BUILD_ROOT/%{prefix}/share/xcat/tools/autotest -cp -r dhcptest $RPM_BUILD_ROOT/%{prefix}/share/xcat/tools/autotest -chmod 755 $RPM_BUILD_ROOT/%{prefix}/share/xcat/tools/autotest/dhcptest/src/dhcptest -# provtest needs nothing that is not already on a management node: dig, tftp -# and curl come with the DNS, TFTP and HTTP the node itself talks to, and the -# xcatd client is python3 and the standard library. -cp -r provtest $RPM_BUILD_ROOT/%{prefix}/share/xcat/tools/autotest -chmod 755 $RPM_BUILD_ROOT/%{prefix}/share/xcat/tools/autotest/provtest/src/provtest +cp -r blackboxtest $RPM_BUILD_ROOT/%{prefix}/share/xcat/tools/autotest +chmod 755 $RPM_BUILD_ROOT/%{prefix}/share/xcat/tools/autotest/blackboxtest/src/blackboxtest %clean