diff --git a/xCAT-test/unit/configeth_netplan.t b/xCAT-test/unit/configeth_netplan.t new file mode 100644 index 000000000..02b2a7714 --- /dev/null +++ b/xCAT-test/unit/configeth_netplan.t @@ -0,0 +1,334 @@ +#!/usr/bin/env perl +use strict; +use warnings; + +use File::Spec; +use File::Temp qw(tempdir); +use FindBin; +use Test::More; + +# Regression (issue #7454): on Ubuntu 18.04+ the network is rendered by netplan. ifupdown is +# not installed and /etc/network/interfaces.d/* is ignored entirely, so configeth's Debian +# branch configured nothing at all. It must write /etc/netplan/*.yaml and `netplan apply`. +# +# These tests drive configipv4/configipv6/delete_nic_config_files -- the branch selection that +# is the actual fix -- rather than the private write_netplan_* writers underneath them. Driving +# only the writers passes even when the netplan branch is deleted from configipv4 outright. +# +# netplan parses every file under /etc/netplan as ONE document, so anything it rejects takes the +# whole node's network with it rather than one interface. Where netplan is installed (it is on +# the ubuntu-24.04 CI runner) the last test feeds the generated tree to the real parser. + +my $repo_root = File::Spec->rel2abs( + File::Spec->catdir( $FindBin::Bin, '..', '..' ) +); +my $configeth = File::Spec->catfile( $repo_root, 'xCAT', 'postscripts', 'configeth' ); +plan skip_all => "configeth not found" unless -f $configeth; + +my $src = do { local $/; open my $fh, '<', $configeth or die $!; <$fh> }; + +# Everything from the netplan detection through delete_nic_config_files: the writers, the +# configipv4/configipv6 dispatch, and the removal path. BAIL_OUT rather than skip, so that a +# rename which stops this matching fails loudly instead of silently covering nothing. +my ($unit) = $src =~ /^(netplan_active=0\n.*?\nfunction delete_nic_config_files\(\)\{.*?\n\})\n/ms; +BAIL_OUT('could not extract the netplan unit from configeth') unless defined $unit; + +my $dir = tempdir( CLEANUP => 1 ); + +# The real sentinel configeth uses for "this attribute is unset" (configeth sets +# str_default_token="default"); a made-up token here would let a hard-coded literal pass. +my ($token) = $src =~ /^str_default_token="([^"]+)"/m; +BAIL_OUT('could not read str_default_token from configeth') unless defined $token; +is( $token, 'default', 'the harness drives the sentinel configeth actually uses' ); + +my $run_no = 0; + +# Run a snippet against the extracted unit. netplan/ifup/systemctl and wait_for_ifstate are +# shadowed by shell functions, which bash resolves ahead of $PATH, so nothing here touches the +# host's network however the suite is run -- and `netplan apply` is recorded, not executed. +sub run_configeth { + my ($script) = @_; + $run_no++; + my $root = File::Spec->catdir( $dir, "run$run_no" ); + mkdir $root; + mkdir File::Spec->catdir( $root, 'etc' ); + mkdir File::Spec->catdir( $root, 'etc', 'netplan' ); + + my $harness = File::Spec->catfile( $root, 'harness.sh' ); + open my $fh, '>', $harness or die $!; + print $fh <<"PRE"; +#!/bin/bash +NODE=cn1 +str_default_token='$token' +str_os_type=debian +str_cfg_dir='$root/' +export NETPLAN_DIR='$root/etc/netplan' +log_info(){ :; } +log_warn(){ echo "WARN: \$*" >> '$root/warnings'; } +log_error(){ echo "ERR: \$*" >> '$root/warnings'; } +netplan(){ echo "netplan \$*" >> '$root/applied'; } +wait_for_ifstate(){ return 0; } +ip(){ :; } +v4mask2prefix(){ + local m=\$1 p=0 o + for o in \${m//./ }; do + case \$o in + 255) p=\$((p+8));; 254) p=\$((p+7));; 252) p=\$((p+6));; 248) p=\$((p+5));; + 240) p=\$((p+4));; 224) p=\$((p+3));; 192) p=\$((p+2));; 128) p=\$((p+1));; + esac + done + echo \$p +} +parse_nic_extra_params(){ + unset array_extra_param_names array_extra_param_values + local k=0 t + for t in \$1; do + array_extra_param_names[\$k]=\${t%%=*} + array_extra_param_values[\$k]=\${t#*=} + k=\$((k+1)) + done +} +declare -a array_extra_param_names +declare -a array_extra_param_values +PRE + print $fh "$unit\n"; + # the detection above ran against the real host; these tests are about the netplan branch + print $fh "netplan_active=1\n"; + print $fh "$script\n"; + close $fh; + system( '/bin/bash', $harness ) == 0 or die "harness failed"; + return $root; +} + +sub yaml_of { + my ( $root, $nic ) = @_; + my $f = File::Spec->catfile( $root, 'etc', 'netplan', "90-xcat-$nic.yaml" ); + return '' unless -f $f; + local $/; + open my $fh, '<', $f or die $!; + my $c = <$fh>; + # the "# xcat-state:" lines are this writer's own bookkeeping, not netplan config + $c =~ s/^# xcat-state:.*\n//mg; + return $c; +} + +sub slurp { my ($p) = @_; return '' unless -f $p; local $/; open my $f, '<', $p or die $!; return <$f>; } + +# --- configipv4 drives the netplan branch ------------------------------------ +# Two addresses, an mtu, a real nicextraparams pair, and an unset-mtu sentinel. +my $r = run_configeth( <<'SH' ); +configipv4 eth0 10.0.0.5 10.0.0.0 255.255.255.0 0 "MTU=1500 ONBOOT=no" 1500 +configipv4 eth0 10.0.1.5 10.0.1.0 255.255.255.0 1 default default +SH + +my $eth0 = yaml_of( $r, 'eth0' ); +ok( length $eth0, 'configipv4 writes a netplan drop-in on a netplan node' ); +like( $eth0, qr/^ ethernets:$/m, 'a plain NIC is declared under ethernets:' ); +unlike( $eth0, qr/^ vlans:$/m, 'a plain NIC is not declared as a vlan' ); +like( $eth0, qr/addresses:\n\s*- 10\.0\.0\.5\/24\n\s*- 10\.0\.1\.5\/24/, + 'multiple addresses keep the order they were added' ); +like( $eth0, qr/^ mtu: 1500$/m, 'an MTU nicextraparam maps onto the netplan mtu key' ); + +# netplan merges same-id netdefs across files key by key, so an earlier cloud-init dhcp4:true +# survives unless this stanza turns it off explicitly. +like( $eth0, qr/^ dhcp4: false$/m, 'DHCP is switched off for a statically addressed NIC' ); +like( $eth0, qr/^ dhcp6: false$/m, 'DHCP6 is switched off for a statically addressed NIC' ); + +# ONBOOT is an ifcfg key with no netplan meaning; passing it through fails the whole file. +unlike( $eth0, qr/ONBOOT/i, 'a nicextraparams key netplan does not know is not emitted' ); +like( slurp("$r/warnings"), qr/ONBOOT/, 'and dropping it is reported' ); + +# the sentinel arm: an unset mtu must not reach the file as the literal "default" +unlike( $eth0, qr/mtu:\s*default/, 'an unset mtu is skipped rather than written as the sentinel' ); + +# --- an address removed from the nics table must actually go ------------------- +$r = run_configeth( <<'SH' ); +configipv4 eth0 10.0.0.5 10.0.0.0 255.255.255.0 0 default default +configipv4 eth0 10.0.0.99 10.0.0.0 255.255.255.0 0 default default +SH +$eth0 = yaml_of( $r, 'eth0' ); +like( $eth0, qr/- 10\.0\.0\.99\/24/, 're-running with a new address writes the new address' ); +unlike( $eth0, qr/- 10\.0\.0\.5\/24/, 'and the address it replaces is dropped, not accumulated' ); + +# --- a VLAN interface --------------------------------------------------------- +# The parent carries no address of its own, so nothing else declares it. netplan resolves link: +# at parse time, so the parent has to be in this file or the whole configuration is rejected. +$r = run_configeth( <<'SH' ); +configipv4 eth1.100 10.100.0.5 10.100.0.0 255.255.255.0 0 default default +SH +my $vlan = yaml_of( $r, 'eth1.100' ); +like( $vlan, qr/^ vlans:$/m, 'a . NIC is declared under vlans:' ); +like( $vlan, qr/^ id: 100$/m, 'the VLAN carries its id' ); +like( $vlan, qr/^ link: eth1$/m, 'the VLAN is linked to its parent interface' ); +like( $vlan, qr/^ ethernets:\n eth1: \{\}/m, + 'the VLAN parent is declared in the same file so link: resolves' ); + +# --- a dotted name that is NOT a vlan ---------------------------------------- +$r = run_configeth( <<'SH' ); +configipv4 eno1.custom 10.9.0.5 10.9.0.0 255.255.255.0 0 default default +SH +like( yaml_of( $r, 'eno1.custom' ), qr/^ ethernets:$/m, + 'a dotted name with a non-numeric suffix is not treated as a VLAN' ); + +# --- configipv6 drives the netplan branch too --------------------------------- +$r = run_configeth( <<'SH' ); +configipv6 eth2 2001:db8::5 2001:db8:: 64 0 0 2001:db8::1 "MTU=9000 ONBOOT=no" +SH +my $v6 = yaml_of( $r, 'eth2' ); +like( $v6, qr/- 2001:db8::5\/64/, 'configipv6 writes the address' ); + +# "to: default" is only understood from netplan 0.103; 18.04 never ships past 0.99. +like( $v6, qr/- to: ::\/0\n\s*via: 2001:db8::1/, + 'the v6 default route is written as an explicit CIDR, not the "default" alias' ); +unlike( $v6, qr/to:\s*default/, 'the "default" route alias is not emitted' ); +like( $v6, qr/^ mtu: 9000$/m, 'configipv6 carries nicextraparams into the stanza' ); + +# configipv6 takes no mtu argument; $str_nic_mtu would be whatever configipv4 last set. +$r = run_configeth( <<'SH' ); +configipv4 eth3 10.0.3.5 10.0.3.0 255.255.255.0 0 default 1500 +configipv6 eth4 2001:db8:4::5 2001:db8:4:: 64 0 0 default default +SH +unlike( yaml_of( $r, 'eth4' ), qr/mtu:/, + 'a v6-only NIC does not inherit the MTU of a NIC configured before it' ); + +# --- route de-duplication ----------------------------------------------------- +$r = run_configeth( <<'SH' ); +configipv6 eth5 2001:db8:5::5 2001:db8:5:: 64 0 0 2001:db8:5::1 default +configipv6 eth5 2001:db8:5::6 2001:db8:5:: 64 1 0 2001:db8:5::1 default +write_netplan_route eth5 2001:db8:9::/64 2001:db8:5::1 +SH +my $eth5 = yaml_of( $r, 'eth5' ); +my @defaults = ( $eth5 =~ /- to: ::\/0/g ); +is( scalar(@defaults), 1, 'an identical route added twice appears once' ); +like( $eth5, qr/- to: 2001:db8:9::\/64\n\s*via: 2001:db8:5::1/, + 'a second route sharing the same gateway is still written' ); + +# --- removal ------------------------------------------------------------------ +$r = run_configeth( <<'SH' ); +configipv4 eth6 10.0.6.5 10.0.6.0 255.255.255.0 0 default default +echo "eth6" > "${str_cfg_dir}xcat_history_important" +delete_nic_config_files eth6 +SH +is( yaml_of( $r, 'eth6' ), '', 'delete_nic_config_files removes the netplan drop-in' ); +unlike( slurp("$r/xcat_history_important"), qr/eth6/, + 'and clears the NIC from xcat_history_important, as the other branches do' ); + +# --- a nicextraparams name containing a regex metacharacter -------------------- +$r = run_configeth( <<'SH' ); +configipv4 eth7 10.0.7.5 10.0.7.0 255.255.255.0 0 "a/b=c MTU=1400" default +SH +like( yaml_of( $r, 'eth7' ), qr/^ mtu: 1400$/m, + 'a param name with a "/" does not break the recorded state around it' ); + +# --- the apply is scoped to the NIC being configured where the backend allows it ------ +# `netplan apply` takes no interface argument and re-applies every netdef on the node, which +# during the install postscripts stage would bounce the install NIC the postscript is running +# over. Where systemd-networkd is the renderer and networkctl has the verbs, only this device +# should be reconfigured. +$r = run_configeth( <<'SH' ); +networkctl(){ echo "networkctl $*" >> "$str_cfg_dir/applied"; [ "$1" = "--help" ] && echo " reconfigure DEVICES... Reconfigure interfaces"; return 0; } +systemctl(){ return 0; } +netplan_apply eth0 +SH +my $applied = slurp("$r/applied"); +like( $applied, qr/^networkctl reconfigure eth0$/m, + 'the apply is scoped to the NIC being configured' ); +like( $applied, qr/^netplan generate$/m, + 'the backend configuration is generated before reconfiguring the device' ); +unlike( $applied, qr/^netplan apply$/m, + 'and the node-wide apply is not used when a single device can be reconfigured' ); + +# Ubuntu 18.04 ships systemd 237, which has no `networkctl reconfigure`; the NetworkManager +# renderer has no networkctl path at all. Both must still apply, node-wide. +$r = run_configeth( <<'SH' ); +networkctl(){ echo "networkctl $*" >> "$str_cfg_dir/applied"; return 0; } +systemctl(){ return 0; } +netplan_apply eth0 +SH +$applied = slurp("$r/applied"); +like( $applied, qr/^netplan apply$/m, + 'a backend that cannot reconfigure one device falls back to the node-wide apply' ); +unlike( $applied, qr/reconfigure/, + 'and does not attempt a verb it does not have' ); + +# --- and netplan itself must accept what we wrote ----------------------------- +# --root-dir is a filesystem root: netplan reads /etc/netplan/*.yaml, which is why the +# harness writes the drop-ins there rather than flat into the scratch directory. +SKIP: { + my $netplan = `command -v netplan 2>/dev/null`; + chomp $netplan; + skip 'netplan not installed', 1 unless $netplan && -x $netplan; + + my $root = File::Spec->catdir( $dir, 'generate' ); + mkdir $root; + mkdir File::Spec->catdir( $root, 'etc' ); + my $np = File::Spec->catdir( $root, 'etc', 'netplan' ); + mkdir $np; + + # every shape this writer emits, in one tree, the way a real node accumulates them -- + # including a netdef an earlier-sorting file already declares as DHCP + run_configeth_into( $np, <<'SH' ); +configipv4 eth0 10.0.0.5 10.0.0.0 255.255.255.0 0 "MTU=1500 ONBOOT=no" 1500 +configipv4 eth0 10.0.1.5 10.0.1.0 255.255.255.0 1 default default +configipv4 eth1.100 10.100.0.5 10.100.0.0 255.255.255.0 0 default default +configipv6 eth2 2001:db8::5 2001:db8:: 64 0 0 2001:db8::1 default +SH + open my $ci, '>', File::Spec->catfile( $np, '50-cloud-init.yaml' ) or die $!; + print $ci "network:\n version: 2\n ethernets:\n eth0:\n dhcp4: true\n"; + close $ci; + chmod 0600, glob("$np/*.yaml"); + + my $out = `netplan generate --root-dir '$root' 2>&1`; + is( $? >> 8, 0, "netplan generate accepts the generated configuration" ) + or diag($out); +} + +# same harness, writing into a caller-chosen netplan directory +sub run_configeth_into { + my ( $np, $script ) = @_; + my $root = File::Spec->catdir( $dir, "gen_stage" ); + mkdir $root; + my $harness = File::Spec->catfile( $root, 'harness.sh' ); + open my $fh, '>', $harness or die $!; + print $fh <<"PRE"; +#!/bin/bash +NODE=cn1 +str_default_token='$token' +str_os_type=debian +str_cfg_dir='$root/' +export NETPLAN_DIR='$np' +log_info(){ :; } +log_warn(){ :; } +log_error(){ :; } +netplan(){ :; } +wait_for_ifstate(){ return 0; } +ip(){ :; } +v4mask2prefix(){ + local m=\$1 p=0 o + for o in \${m//./ }; do + case \$o in + 255) p=\$((p+8));; 254) p=\$((p+7));; 252) p=\$((p+6));; 248) p=\$((p+5));; + 240) p=\$((p+4));; 224) p=\$((p+3));; 192) p=\$((p+2));; 128) p=\$((p+1));; + esac + done + echo \$p +} +parse_nic_extra_params(){ + unset array_extra_param_names array_extra_param_values + local k=0 t + for t in \$1; do + array_extra_param_names[\$k]=\${t%%=*} + array_extra_param_values[\$k]=\${t#*=} + k=\$((k+1)) + done +} +declare -a array_extra_param_names +declare -a array_extra_param_values +PRE + print $fh "$unit\nnetplan_active=1\n$script\n"; + close $fh; + system( '/bin/bash', $harness ) == 0 or die "harness failed"; + return; +} + +done_testing(); diff --git a/xCAT-test/unit/configeth_netplan_reboot_gate.t b/xCAT-test/unit/configeth_netplan_reboot_gate.t new file mode 100644 index 000000000..c2ca3f7af --- /dev/null +++ b/xCAT-test/unit/configeth_netplan_reboot_gate.t @@ -0,0 +1,123 @@ +#!/usr/bin/env perl +use strict; +use warnings; + +use File::Spec; +use File::Temp qw(tempdir); +use FindBin; +use Test::More; + +# Regression: on a netplan-rendered node configeth took the NIC down on every reconfigure, +# while the `netplan apply` that brings it back is gated on reboot_nic_bool. In the diskful +# provision postscripts stage reboot_nic_bool is 0, so the interface went down and nothing +# brought it back until the node rebooted -- over the very NIC the postscripts are talking on. +# +# The other two arms already pair the two halves: the ifupdown arm answers `ifdown` with an +# unconditional `ifup`, and the redhat arm gates BOTH the down and the up on reboot_nic_bool. +# Only the netplan arm took one half of the pair. +# +# This drives the two blocks configeth actually executes rather than matching its text: the +# down-selection inside the modify branch, and the restart block underneath it. `ip` and +# `netplan` are shadowed by shell functions, which bash resolves ahead of $PATH, so the host's +# network is never touched -- every call is recorded to a file instead. + +my $repo_root = File::Spec->rel2abs( + File::Spec->catdir( $FindBin::Bin, '..', '..' ) +); +my $configeth = File::Spec->catfile( $repo_root, 'xCAT', 'postscripts', 'configeth' ); +plan skip_all => "configeth not found" unless -f $configeth; + +my $src = do { local $/; open my $fh, '<', $configeth or die $!; <$fh> }; + +# BAIL_OUT rather than skip: a rename that stops these matching must fail loudly instead of +# silently covering nothing. +my ($down_block) = $src =~ /\n( if \[ "\$str_nic_status" = "up" \];then\n.*?\n fi\n)/ms; +BAIL_OUT('could not extract the nic-down block from configeth') + unless defined $down_block; + +my ($restart_block) = $src =~ /\n( #restart the nic\n if \[ \$bool_restart_flag -eq 1 \];then\n.*?\n fi\n)/ms; +BAIL_OUT('could not extract the restart block from configeth') + unless defined $restart_block; + +my $dir = tempdir( CLEANUP => 1 ); +my $run_no = 0; + +# Run both blocks back to back for one (reboot_nic_bool, arm) combination and return what the +# script asked the system to do, in order. +sub drive { + my (%opt) = @_; + $run_no++; + my $root = File::Spec->catdir( $dir, "run$run_no" ); + mkdir $root; + my $calls = File::Spec->catfile( $root, 'calls' ); + + my $harness = File::Spec->catfile( $root, 'harness.sh' ); + open my $fh, '>', $harness or die $!; + print $fh <<"PRE"; +#!/bin/bash +NODE=cn1 +str_nic_name=eth0 +str_nic_status=up +str_os_type='$opt{os_type}' +netplan_active='$opt{netplan_active}' +networkmanager_active=0 +reboot_nic_bool=$opt{reboot_nic_bool} +bool_modify_flag=1 +bool_restart_flag=1 +error_code=0 +array_ip_old_temp=() +log_info(){ :; } +log_warn(){ :; } +log_error(){ echo "log_error \$*" >> '$calls'; } +ip(){ echo "ip \$*" >> '$calls'; } +ifdown(){ echo "ifdown \$*" >> '$calls'; } +ifup(){ echo "ifup \$*" >> '$calls'; } +nmcli(){ echo "nmcli \$*" >> '$calls'; } +netplan(){ echo "netplan \$*" >> '$calls'; } +networkctl(){ echo "networkctl \$*" >> '$calls'; } +netplan_apply(){ echo "netplan_apply \$*" >> '$calls'; return 0; } +wait_for_ifstate(){ echo 0; return 0; } +PRE + print $fh $down_block, "\n", $restart_block, "\n"; + close $fh; + + system( '/bin/bash', $harness ); + return '' unless -f $calls; + my $out = do { local $/; open my $c, '<', $calls or die $!; <$c> }; + return $out; +} + +# The bug, stated as behaviour: during the install postscripts stage the NIC must not be left +# down. Either the link is not touched, or something brings it back. +my $install = drive( os_type => 'debian', netplan_active => 1, reboot_nic_bool => 0 ); +my $took_down = $install =~ /^ip link set dev eth0 down/m; +my $brought_up = $install =~ /^netplan_apply/m; +ok( !$took_down || $brought_up, + 'netplan: with reboot_nic_bool=0 the NIC is not left down' ); +unlike( $install, qr/^ip link set dev eth0 down/m, + 'netplan: with reboot_nic_bool=0 the link is not taken down at all' ); + +# The redhat arm is the reference: it has always gated both halves. +my $rh_install = drive( os_type => 'rhel', netplan_active => 0, reboot_nic_bool => 0 ); +unlike( $rh_install, qr/^ip link set dev eth0 down/m, + 'redhat: with reboot_nic_bool=0 the link is not taken down (reference behaviour)' ); + +# ifupdown pairs its own halves unconditionally, so it may take the link down. +my $deb_install = drive( os_type => 'debian', netplan_active => 0, reboot_nic_bool => 0 ); +like( $deb_install, qr/^ifdown --force eth0/m, + 'ifupdown: takes the link down' ); +like( $deb_install, qr/^ifup -a -i /m, + 'ifupdown: and brings it back unconditionally, so it is never left down' ); + +# reboot_nic_bool=1 is the normal reconfigure: both halves must run. +my $reboot = drive( os_type => 'debian', netplan_active => 1, reboot_nic_bool => 1 ); +like( $reboot, qr/^ip link set dev eth0 down/m, + 'netplan: with reboot_nic_bool=1 the link is taken down' ); +like( $reboot, qr/^netplan_apply eth0/m, + 'netplan: with reboot_nic_bool=1 the apply brings it back' ); + +my ($down_at) = $reboot =~ /\A(.*?)^netplan_apply/ms; +ok( defined $down_at && $down_at =~ /ip link set dev eth0 down/, + 'netplan: the down happens before the apply, not after it' ); + +done_testing(); diff --git a/xCAT/postscripts/configeth b/xCAT/postscripts/configeth index bfcc76e6d..079a732a6 100755 --- a/xCAT/postscripts/configeth +++ b/xCAT/postscripts/configeth @@ -40,6 +40,318 @@ fi str_conf_file="" str_conf_file_xcatbak="" tmp_con_name="" + +# --------------------------------------------------------------------------- +# netplan support (Ubuntu 18.04+, issue #7454) +# +# On a netplan-rendered Debian/Ubuntu node ifupdown is not installed and +# /etc/network/interfaces.d/* is ignored entirely, so writing there configures nothing. +# Write /etc/netplan/*.yaml and `netplan apply` instead. +# +# The drop-in this owns is regenerated in full on every change rather than edited in place, +# because inserting into the middle of a YAML list with sed reverses multi-address order and +# cannot tell one route's fields from another's. The inputs are kept in "# xcat-state:" comment +# lines at the top of the same file: netplan ignores comments, the file stays the single source +# of truth, and re-rendering is idempotent. +# +# netplan parses every file under /etc/netplan as one document, so a single rejected key takes +# the whole node's network with it -- not just the interface being configured. That is why the +# writers below validate rather than pass values through. +# --------------------------------------------------------------------------- + +netplan_active=0 +if command -v netplan >/dev/null 2>&1 && [ -d /etc/netplan ]; then + netplan_active=1 + # netplan.io is Priority:important and a dependency of cloud-init, so its presence says + # nothing about what actually renders this node. Only ifupdown both installed AND running + # means /etc/network/interfaces is still the live configuration, in which case writing YAML + # would orphan it. (checkservicestatus is no help here: servicemap has no entry for + # "networking", so it returns 127 whatever the service is doing.) + if command -v ifup >/dev/null 2>&1 && systemctl is-active networking >/dev/null 2>&1; then + netplan_active=0 + fi +fi + +# netplan_file -- the per-NIC drop-in this script owns. NETPLAN_DIR overrides the +# directory so the writers can be unit tested without touching the real configuration. +netplan_file(){ + echo "${NETPLAN_DIR:-/etc/netplan}/90-xcat-${1}.yaml" +} + +_netplan_touch(){ # -- create the drop-in already unreadable to other users + [ -f "$1" ] || ( umask 077; : > "$1" ) +} + +_netplan_state(){ # -- emit the recorded values of one kind, in insertion order + local f + f="$(netplan_file "$1")" + [ -f "$f" ] || return 0 + sed -n "s/^# xcat-state: $2 //p" "$f" +} + +_netplan_record(){ # -- record once, preserving order + local nic="$1" kind="$2" value="$3" f + f="$(netplan_file "$nic")" + if [ -f "$f" ] && grep -qxF "# xcat-state: $kind $value" "$f"; then + return 0 + fi + printf '# xcat-state: %s %s\n' "$kind" "$value" >> "$f" +} + +# _netplan_forget -- drop the recorded lines whose value starts with . +# awk index() is a literal string match: a nicextraparams key is admin-supplied and a '/' or a +# regex metacharacter in a sed address would either error out or delete unrelated state. +_netplan_forget(){ + local f="$1" pfx="$2" tmp + [ -f "$f" ] || return 0 + tmp="${f}.forget.$$" + if awk -v p="# xcat-state: $pfx" 'index($0, p) != 1' "$f" > "$tmp"; then + mv -f "$tmp" "$f" + else + rm -f "$tmp" + fi +} + +# _netplan_param_key -- the netplan key a nicextraparams name maps to, or non-zero if it +# has none. nicextraparams are ifcfg/ifupdown names -- the documented examples are +# "MTU=1456 ONBOOT=no" -- and netplan rejects the entire configuration for one unknown key, so +# anything without a netplan meaning has to be dropped rather than passed through. +_netplan_param_key(){ + local k + k=$(echo "$1" | tr 'A-Z' 'a-z') + case "$k" in + mtu|optional|critical|wakeonlan|dhcp4|dhcp6|accept-ra|ipv6-privacy) echo "$k"; return 0 ;; + dhcp-identifier|activation-mode|ipv6-mtu|macaddress) echo "$k"; return 0 ;; + esac + return 1 +} + +# _netplan_yaml_scalar -- a value safe to paste into the generated YAML. Integers and +# booleans keep their type (netplan wants mtu as an int, optional as a bool); anything else is +# quoted so that a ':' or '#' in a value cannot invent a key or comment out the rest of a line. +_netplan_yaml_scalar(){ + case "$1" in + true|false) echo "$1"; return 0 ;; + esac + case "$1" in + ''|*[!0-9]*) ;; + *) echo "$1"; return 0 ;; + esac + printf '"%s"\n' "$(echo "$1" | sed -e 's/\\/\\\\/g' -e 's/"/\\"/g')" +} + +# _netplan_route_dest -- the explicit default-route destination for the gateway's family. +# "to: default" is only understood from netplan 0.103 onwards; Ubuntu 18.04 never ships past +# 0.99 and a stock 20.04 ships 0.99 too, where it fails the whole file. The CIDR works on all. +_netplan_route_dest(){ + case "$1" in + *:*) echo "::/0" ;; + *) echo "0.0.0.0/0" ;; + esac +} + +# _netplan_render -- rebuild the whole drop-in from its recorded state. +# A NIC named . is a VLAN and MUST be declared under vlans: with id and link, or +# netplan will not recreate it after a reboot; anything else is a plain ethernet. +_netplan_render(){ + local nic="$1" f tmp section parent vid addr route to via mtu param name value key + local has_dhcp4=0 has_dhcp6=0 + f="$(netplan_file "$nic")" + tmp="${f}.tmp.$$" + + section="ethernets" + parent="" + vid="" + case "$nic" in + *.*) + parent="${nic%.*}" + vid="${nic##*.}" + case "$vid" in + ''|*[!0-9]*) parent=""; vid="" ;; # not . + *) section="vlans" ;; + esac + ;; + esac + + # a nicextraparams key may set dhcp4/dhcp6 itself; do not then emit the default as well, + # which would be a duplicate mapping key in the same stanza + while read -r param; do + key=$(_netplan_param_key "${param%% *}") || continue + [ "$key" = "dhcp4" ] && has_dhcp4=1 + [ "$key" = "dhcp6" ] && has_dhcp6=1 + done <..yaml always sorts before 90-xcat-.yaml + # ('1' < 'y'). The parent may also carry no address of its own, in which case xCAT + # never writes a file for it at all. Declaring it here as an empty netdef resolves + # the reference without overriding any key the parent's own netdef sets. + echo " ethernets:" + echo " ${parent}: {}" + fi + echo " ${section}:" + echo " ${nic}:" + if [ "$section" = "vlans" ]; then + echo " id: ${vid}" + echo " link: ${parent}" + fi + # netplan merges netdefs of the same name across files key by key rather than replacing + # them, so without these an earlier-sorting dhcp4:true -- cloud-init's + # 50-cloud-init.yaml, or xCAT's own netboot drop-in -- survives and the node runs a DHCP + # lease alongside the static address xCAT just assigned. + [ $has_dhcp4 -eq 0 ] && echo " dhcp4: false" + [ $has_dhcp6 -eq 0 ] && echo " dhcp6: false" + + if [ -n "$(_netplan_state "$nic" addr)" ]; then + echo " addresses:" + _netplan_state "$nic" addr | while read -r addr; do + [ -n "$addr" ] && echo " - ${addr}" + done + fi + + mtu="$(_netplan_state "$nic" mtu | tail -1)" + [ -n "$mtu" ] && echo " mtu: ${mtu}" + + if [ -n "$(_netplan_state "$nic" route)" ]; then + echo " routes:" + _netplan_state "$nic" route | while read -r route; do + to="${route%% *}" + via="${route#* }" + [ -n "$to" ] || continue + [ "$to" = "default" ] && to="$(_netplan_route_dest "$via")" + printf ' - to: %s\n via: %s\n' "$to" "$via" + done + fi + + # nicextraparams, already normalised and filtered to keys netplan understands by + # write_netplan_param below + _netplan_state "$nic" param | while read -r param; do + name="${param%% *}" + value="${param#* }" + [ -n "$name" ] && echo " ${name}: $(_netplan_yaml_scalar "$value")" + done + } > "$tmp" ) + + mv -f "$tmp" "$f" + chmod 600 "$f" 2>/dev/null +} + +# netplan_reset_nic -- forget everything recorded for this NIC. Called for the first +# address of a NIC, mirroring the truncating '>' of the ifupdown branch: without it an address +# that was removed from the nics table is never dropped and the node keeps it forever. +netplan_reset_nic(){ + rm -f "$(netplan_file "$1")" 2>/dev/null + return 0 +} + +# _netplan_can_reconfigure -- can one device be re-applied without disturbing the others? +# Only when systemd-networkd is the renderer and networkctl carries the verbs (systemd 244+; +# Ubuntu 18.04 ships 237 and does not). Probed rather than version-gated, so this follows the +# node's actual backend -- under the NetworkManager renderer it is correctly false. +_netplan_can_reconfigure(){ + command -v networkctl >/dev/null 2>&1 || return 1 + networkctl --help 2>&1 | grep -qw reconfigure || return 1 + systemctl is-active systemd-networkd >/dev/null 2>&1 || return 1 + return 0 +} + +# netplan_apply [nic] -- make the drop-in live, and verify the link actually came up. +# +# `netplan apply` takes no interface argument: it re-applies every netdef on the node, so +# configuring one secondary NIC would also bounce the install NIC the postscripts are still +# running over. Where the backend can scope it, generate the backend configuration and +# reconfigure only . Anything that cannot be scoped -- a device netplan has yet to create, +# the NetworkManager renderer, a systemd without the verbs -- falls back to the node-wide apply. +# +# `netplan apply` returns 0 even when it changed nothing, so its exit status alone cannot stand +# in for the interface state the ifup path used to check. +netplan_apply(){ + local nic="$1" + if [ -n "$nic" ] && _netplan_can_reconfigure; then + netplan generate + if [ $? -ne 0 ]; then + log_error "netplan generate failed." + return 1 + fi + networkctl reload >/dev/null 2>&1 + networkctl reconfigure "$nic" >/dev/null 2>&1 + if [ $? -eq 0 ]; then + wait_for_ifstate "$nic" UP 20 5 >/dev/null + if [ $? -ne 0 ]; then + log_error "bring $nic up failed." + return 1 + fi + return 0 + fi + log_info "configeth on $NODE: networkctl could not reconfigure $nic alone, applying all." + fi + netplan apply + if [ $? -ne 0 ]; then + log_error "netplan apply failed." + return 1 + fi + [ -n "$nic" ] || return 0 + wait_for_ifstate "$nic" UP 20 5 >/dev/null + if [ $? -ne 0 ]; then + log_error "bring $nic up failed." + return 1 + fi + return 0 +} + +# write_netplan_addr [mtu] -- add an address (IPv4 or IPv6). Idempotent per +# address, and preserves the order in which addresses were added. +write_netplan_addr(){ + local nic="$1" cidr="$2" mtu="$3" f + f="$(netplan_file "$nic")" + _netplan_touch "$f" + _netplan_record "$nic" addr "$cidr" + if [ -n "$mtu" ] && [ "$mtu" != "$str_default_token" ]; then + # last mtu wins; drop any previously recorded one so the state stays single-valued + _netplan_forget "$f" "mtu " + _netplan_record "$nic" mtu "$mtu" + fi + _netplan_render "$nic" +} + +# write_netplan_route -- add a route. Idempotent on the (to, via) pair. +write_netplan_route(){ + local nic="$1" to="$2" via="$3" f + f="$(netplan_file "$nic")" + [ -f "$f" ] || return 0 + _netplan_record "$nic" route "$to $via" + _netplan_render "$nic" +} + +# write_netplan_param -- record a nicextraparams key for this interface. +write_netplan_param(){ + local nic="$1" name="$2" value="$3" f key + key=$(_netplan_param_key "$name") + if [ $? -ne 0 ]; then + log_warn "configeth on $NODE: nicextraparams key '$name' has no netplan equivalent, ignored for $nic." + return 0 + fi + f="$(netplan_file "$nic")" + _netplan_touch "$f" + if [ "$key" = "mtu" ]; then + # the same knob write_netplan_addr records; keeping one source avoids a duplicate key + _netplan_forget "$f" "mtu " + _netplan_record "$nic" mtu "$value" + else + _netplan_forget "$f" "param $key " + _netplan_record "$nic" param "$key $value" + fi + _netplan_render "$nic" +} function configipv4(){ str_if_name=$1 str_v4ip=$2 @@ -111,7 +423,23 @@ function configipv4(){ if [[ ${str_if_name} == [a-zA-Z0-9]*.[0-9]* ]]; then echo "VLAN=yes" >> $str_conf_file fi - #debian ubuntu + #ubuntu/debian rendered by netplan (18.04+): ifupdown is absent and interfaces.d is + #ignored, so write /etc/netplan/*.yaml instead (issue #7454) + elif [ "$str_os_type" = "debian" ] && [ "$netplan_active" = "1" ];then + #the first address of a NIC starts a fresh drop-in, the way the ifupdown branch below + #truncates with '>' -- otherwise an address dropped from the nics table is never removed + if [ $num_v4num -eq 0 ];then + netplan_reset_nic "${str_if_name}" + fi + str_prefix=$(v4mask2prefix $str_v4mask) + write_netplan_addr "${str_if_name}" "${str_v4ip}/${str_prefix}" "${str_nic_mtu}" + i=0 + while [ $i -lt ${#array_extra_param_names[@]} ] + do + write_netplan_param "${str_if_name}" "${array_extra_param_names[$i]}" "${array_extra_param_values[$i]}" + i=$((i+1)) + done + #debian ubuntu (legacy ifupdown) elif [ "$str_os_type" = "debian" ];then str_conf_file="/etc/network/interfaces.d/${str_if_name}" if [ $num_v4num -eq 0 ];then @@ -290,6 +618,26 @@ configipv6(){ fi i=$((i+1)) done + #ubuntu/debian rendered by netplan (18.04+) -- see configipv4 (issue #7454) + elif [ "$str_os_type" = "debian" ] && [ "$netplan_active" = "1" ];then + if [ $num_v4num -eq 0 -a $num_v6num -eq 0 ];then + netplan_reset_nic "${str_if_name}" + fi + #configipv6 takes no mtu argument: $str_nic_mtu here would be whatever configipv4 last + #assigned for this NIC, and the ifupdown v6 branch below writes no mtu at all + write_netplan_addr "${str_if_name}" "${str_v6ip}/${str_v6prefix}" "" + if [ "$str_v6gateway" != "$str_default_token" ] && [ -n "$str_v6gateway" ] \ + && echo "$str_v6gateway" | grep -qv 'xcatmaster'; then + write_netplan_route "${str_if_name}" "default" "${str_v6gateway}" + fi + #the ifupdown v6 branch below writes these into the stanza; dropping them here would + #silently discard configuration on a v6-only NIC + i=0 + while [ $i -lt ${#array_extra_param_names[@]} ] + do + write_netplan_param "${str_if_name}" "${array_extra_param_names[$i]}" "${array_extra_param_values[$i]}" + i=$((i+1)) + done elif [ "$str_os_type" = "debian" ];then #debian or ubuntu str_conf_file="/etc/network/interfaces.d/${str_if_name}" @@ -393,7 +741,10 @@ function delete_nic_config_files(){ str_temp_name=$1 #delete the configuration files #delete the configuration history - if [ "$str_os_type" = "debian" ];then + if [ "$str_os_type" = "debian" ] && [ "$netplan_active" = "1" ];then + rm -f "$(netplan_file "$str_temp_name")" 2>/dev/null + sed -i "/${str_temp_name}/d" "${str_cfg_dir}xcat_history_important" 2>/dev/null + elif [ "$str_os_type" = "debian" ];then rm -f /etc/network/interfaces.d/$str_temp_name 2>/dev/null sed -i "/${str_temp_name}/d" /etc/network/xcat_history_important elif [ "$str_os_type" = "sles" ];then @@ -542,7 +893,10 @@ if [ "$1" = "-r" ];then #shut down the nic if it is on ip link show $str_nic_name | grep -i ',up' if [ $? -eq 0 ];then - if [ "$str_os_type" = "debian" ];then + if [ "$str_os_type" = "debian" ] && [ "$netplan_active" = "1" ];then + #ifdown belongs to ifupdown and does not exist on a netplan-rendered node + ip link set dev $str_nic_name down + elif [ "$str_os_type" = "debian" ];then ifdown --force $str_nic_name else ip link set dev $str_nic_name down @@ -551,6 +905,14 @@ if [ "$1" = "-r" ];then #delete the configuration files delete_nic_config_files $str_nic_name + #the drop-in just removed WAS the live configuration; without applying, the address + #stays on the link until the next reboot + if [ "$str_os_type" = "debian" ] && [ "$netplan_active" = "1" ];then + netplan_apply + if [ $? -ne 0 ]; then + error_code=1 + fi + fi fi exit $error_code elif [ "$1" = "-s" ];then @@ -679,7 +1041,25 @@ elif [ "$1" = "-s" ];then # cofniguring the interface - if [ -f "/etc/debian_version" ];then + #ubuntu/debian rendered by netplan (18.04+): interfaces.d is ignored, so the install NIC + #has to be written as a netplan drop-in too -- this is the path confignetwork takes for + #every provisioned node, and leaving it on interfaces.d configured nothing (issue #7454) + if [ -f "/etc/debian_version" ] && [ "$netplan_active" = "1" ];then + netplan_reset_nic "${str_inst_nic}" + str_inst_prefix=$(v4mask2prefix ${str_inst_mask}) + write_netplan_addr "${str_inst_nic}" "${str_inst_ip}/${str_inst_prefix}" "${str_inst_mtu}" + if [ -n "$str_inst_gateway" ];then + write_netplan_route "${str_inst_nic}" "default" "${str_inst_gateway}" + fi + i=0 + while [ $i -lt ${#array_extra_param_names[@]} ] + do + write_netplan_param "${str_inst_nic}" "${array_extra_param_names[$i]}" "${array_extra_param_values[$i]}" + i=$((i+1)) + done + hostname $NODE + echo $NODE > /etc/hostname + elif [ -f "/etc/debian_version" ];then str_conf_file="/etc/network/interfaces.d/${str_inst_nic}" echo "auto ${str_inst_nic}" > $str_conf_file echo "iface ${str_inst_nic} inet static" >> $str_conf_file @@ -869,12 +1249,19 @@ elif [ "$1" = "-s" ];then if [ "$UPDATENODE" = "1" ] || [ "$NODESETSTATE" = "netboot" ] || [ "$NODESETSTATE" = "statelite" ] || grep "REBOOT=TRUE" /opt/xcat/xcatinfo >/dev/null 2>&1; then if_state=0 - if [ "$str_os_type" = "debian" ];then + if [ "$str_os_type" = "debian" ] && [ "$netplan_active" = "1" ];then + : # netplan apply below reconfigures the interface; no ifdown needed + elif [ "$str_os_type" = "debian" ];then ifdown --force $str_inst_nic else ip link set dev $str_inst_nic down fi - if [ $networkmanager_active -eq 1 ]; then + #tested before NetworkManager: on an NM-rendered netplan node this is the correct arm, + #and as an elif after it, it was dead code that fell through to an nmcli call with an + #unset connection name + if [ "$str_os_type" = "debian" ] && [ "$netplan_active" = "1" ]; then + netplan_apply "$str_inst_nic" || error_code=1 + elif [ $networkmanager_active -eq 1 ]; then nmcli con modify $con_name ipv4.dns "${NAMESERVERS}" nmcli con reload nmcli con up $con_name @@ -1210,7 +1597,14 @@ else if [ $bool_modify_flag -eq 1 ];then if [ $bool_restart_flag -eq 1 ];then if [ "$str_nic_status" = "up" ];then - if [ "$str_os_type" = "debian" ];then + if [ "$str_os_type" = "debian" ] && [ "$netplan_active" = "1" ];then + #the netplan_apply that brings the link back is gated on reboot_nic_bool, + #so taking it down here ungated leaves the NIC down until the next reboot + #in the provision postscripts stage -- over the NIC we are talking on. + if [ $reboot_nic_bool -eq 1 ]; then + ip link set dev $str_nic_name down > /dev/null 2>/dev/null + fi + elif [ "$str_os_type" = "debian" ];then ifdown --force $str_nic_name > /dev/null else if [ $reboot_nic_bool -eq 1 ]; then @@ -1285,7 +1679,19 @@ else #restart the nic if [ $bool_restart_flag -eq 1 ];then - if [ "$str_os_type" = "debian" ];then + if [ "$str_os_type" = "debian" ] && [ "$netplan_active" = "1" ];then + #`netplan apply` has no per-interface form: it re-applies every netdef on the node. + #In the diskful provision postscripts stage that would bounce the install NIC the + #postscripts are still talking over, which is what reboot_nic_bool guards against + #for the redhat arm below. The drop-in is written either way and takes effect at + #boot, so skipping the apply here costs nothing. + if [ $reboot_nic_bool -eq 1 ]; then + netplan_apply "$str_nic_name" + if [ $? -ne 0 ]; then + error_code=1 + fi + fi + elif [ "$str_os_type" = "debian" ];then ifup -a -i /etc/network/interfaces.d/$str_nic_name if [ $? -ne 0 ]; then log_error "ifup -a -i /etc/network/interfaces.d/$str_nic_name failed."