diff --git a/docs/source/guides/admin-guides/manage_clusters/common/deployment/additionalpkg/additional_pkg_overview.rst b/docs/source/guides/admin-guides/manage_clusters/common/deployment/additionalpkg/additional_pkg_overview.rst index f867e901a..27ce4d7a3 100644 --- a/docs/source/guides/admin-guides/manage_clusters/common/deployment/additionalpkg/additional_pkg_overview.rst +++ b/docs/source/guides/admin-guides/manage_clusters/common/deployment/additionalpkg/additional_pkg_overview.rst @@ -6,6 +6,8 @@ The name of the packages that will be installed on the node are stored in the pa * The package list file contains the names of the packages that comes from the os distro. They are stored in .pkglist file. * The other package list file contains the names of the packages that do NOT come from the os distro. They are stored in .otherpkgs.pkglist file. +On Ubuntu releases that install with Subiquity, the packages in the .pkglist file are installed during the autoinstall from the configured apt mirror and the pkgdir mirrors, without recommended packages as ``ospkgs`` installs them, and ``ospkgs`` applies the whole list again after the first boot. A version pin, a target release or an architecture qualifier in the list, a list that carries a ``#ENV:`` setting, and the list of an osimage with ``environvar`` are installed by ``ospkgs`` only. The apt sources the installer uses for the pkgdir mirrors and the otherpkgs repository do not remain on the node: ``ospkgs`` and ``otherpkgs`` write their own after the first boot, as before. ``ospkgs`` writes http mirrors only, so an https mirror or a local directory in pkgdir serves the autoinstall and is not an apt source after the first boot. + The path to the package lists will be read from the osimage definition. Which osimage a node is using is specified by the provmethod attribute. To display this value for a node: :: lsdef node1 -i provmethod diff --git a/xCAT-server/lib/perl/xCAT/Postage.pm b/xCAT-server/lib/perl/xCAT/Postage.pm index e9d40ba8c..0eccbb26c 100644 --- a/xCAT-server/lib/perl/xCAT/Postage.pm +++ b/xCAT-server/lib/perl/xCAT/Postage.pm @@ -1889,6 +1889,68 @@ sub get_pkglist_tex #---------------------------------------------------------------------------- +=head3 get_pkglist_records + + The records of one or more pkglist files, one per line kept whole and + includes followed. get_pkglist_tex joins records with commas, so it + cannot separate a record that itself contains a comma. + Arguments: comma-separated pkglist file names + Returns: list of records +=cut + +#----------------------------------------------------------------------------- +sub get_pkglist_records +{ + my $allfiles_pkglist = shift; + if ($allfiles_pkglist =~ "xCAT::") { + $allfiles_pkglist = shift; + } + my @records; + foreach my $pkglist (split(/,/, $allfiles_pkglist // '')) + { + next if $pkglist eq ''; + push(@records, pkglist_file_records($pkglist, dirname($pkglist), 0)); + } + return @records; +} + +# pkglist_file_records: the records of one pkglist file, read as get_pkglist_tex reads them, with an +# #INCLUDE: record replaced by the records of the named file. +# A nested include resolves against the directory of the listed pkglist, as get_pkglist_tex resolves it. +sub pkglist_file_records +{ + my ($file, $idir, $depth) = @_; + my @records; + open(my $fh, '<', $file) or return ("#INCLUDEBAD:cannot open pkglist file $file#"); + while (my $line = <$fh>) + { + chomp($line); + $line =~ s/\s+$//; + $line =~ s/^\s*//; + next if $line eq ''; + next + if ($line =~ /^#/ + && $line !~ /^#INCLUDE:[^#^\n]+#/ + && $line !~ /^#NEW_INSTALL_LIST#/ + && $line !~ /^#ENV:[^#^\n]+#/); + if ($line =~ /^#INCLUDE:([^#^\n]+)#(.*)$/ && $depth < 20) + { + my ($name, $note) = ($1, $2); + my $include = xCAT::Utils->varsubinline($name, \%ENV); + $include = "$idir/$include" unless $include =~ m{^/}; + my @included = pkglist_file_records($include, $idir, $depth + 1); + $included[-1] .= $note if @included && $note ne ''; + push(@records, @included); + next; + } + push(@records, $line); + } + close($fh); + return @records; +} + +#---------------------------------------------------------------------------- + =head3 includefile handles #INCLUDE# in otherpkg.pkglist file diff --git a/xCAT-server/lib/perl/xCAT/Template.pm b/xCAT-server/lib/perl/xCAT/Template.pm index de954065f..79b26101f 100644 --- a/xCAT-server/lib/perl/xCAT/Template.pm +++ b/xCAT-server/lib/perl/xCAT/Template.pm @@ -173,6 +173,11 @@ sub subvars { $inc =~ s/#INCLUDE_DEFAULT_RMPKGLIST_S#/#INCLUDE_RMPKGLIST:$pkglistfile#/g; } + # osimage environvar reaches apt-get through ospkgs alone, so such an image installs its list there, + # and its pkgdir mirrors, which may need those variables too, stay out of the installer's sources + my $environvar_set = ( $namedargs{environvar} // '' ) =~ /\S/; + my $installer_pkgdirs = $environvar_set ? undef : $namedargs{pkgdirs}; + my @autoinstall; if (("ubuntu" eq $platform) || ("debian" eq $platform)) { # since debian/ubuntu uses a preseed file instead of a kickstart file, pkglist @@ -186,6 +191,7 @@ sub subvars { if ($allpkglist =~ /#INCLUDEBAD:(.*)#/) { return "$1"; } + @autoinstall = ubuntu_autoinstall_packages( xCAT::Postage->get_pkglist_records($pkglistfile) ) unless $environvar_set; $allpkglist =~ s/,/ /g; $inc =~ s/#INCLUDE_DEFAULT_PKGLIST_PRESEED#/$allpkglist/g; @@ -364,7 +370,7 @@ sub subvars { $inc =~ s/#INSTALL_SOURCES_IN_PRE#/$source_in_pre/g; if (("ubuntu" eq $platform) || ("debian" eq $platform)) { $inc =~ s/#INCLUDE_OSIMAGE_PKGDIR#/$pkgdirs[-1]/; - $inc =~ s/#UBUNTU_SUBIQUITY_APT_CONFIG#/ubuntu_subiquity_apt_config($media_dir, $namedargs{osarch})/eg; + $inc =~ s/#UBUNTU_SUBIQUITY_APT_CONFIG#/ubuntu_subiquity_apt_config($media_dir, $namedargs{osarch}, $installer_pkgdirs)/eg; } $inc =~ s/#WRITEREPO#/$writerepo/g; } @@ -377,7 +383,9 @@ sub subvars { $inc =~ s/#INCLUDE_NOP:([^#^\n]+)#/includefile($1,1,0)/eg; $inc =~ s/#XCATVAR:([^#]+)#/envvar($1)/eg; $inc =~ s/#ENV:([^#]+)#/envvar($1)/eg; - $inc =~ s/#UBUNTU_SUBIQUITY_APT_CONFIG#/ubuntu_subiquity_apt_config($media_dir, $namedargs{osarch})/eg; + $inc =~ s/#UBUNTU_SUBIQUITY_APT_CONFIG#/ubuntu_subiquity_apt_config($media_dir, $namedargs{osarch}, $installer_pkgdirs)/eg; + # in the include pass, so a template that includes the stock Subiquity one gets its list as well + $inc =~ s/^((?:[ \t]*- [^\n]*\n)*)([ \t]*)- #INCLUDE_DEFAULT_PKGLIST_AUTOINSTALL#[ \t]*\n/$1 . ubuntu_autoinstall_items($2, $1, \@autoinstall)/meg; $inc =~ s/#SUBIQUITYINSTALLNIC#/subiquity_install_nic()/eg; $inc =~ s/#SUBIQUITYINSTALLMAC#/subiquity_install_mac()/eg; $inc =~ s/#MACHINEPASSWORD#/machinepassword()/eg; @@ -1764,6 +1772,59 @@ sub subiquity_install_mac { return $macaddress; } +# ubuntu_autoinstall_packages: the packages of the pkglist records (whole lines, as +# get_pkglist_records returns them) that a Subiquity autoinstall can install through its packages +# list. A record holds one or more space-separated packages, as the preseed path reads it, each a +# plain name or a task. A version pin or a target release stays with ospkgs, because the installer +# runs apt-get without --allow-downgrades and a pin can require one, and so does a name with an +# architecture qualifier, because a foreign architecture is enabled by a postscript that runs later. A preseed directive, told by its question type, a record that begins with a removal +# or a group, which ospkgs removes or installs whole, a removal written with a trailing hyphen as +# apt-get reads it, or a marker has +# no autoinstall form, a comment ends the packages of a record, and a record with a token that is +# none of these is left out whole. A list +# that carries a #ENV: setting, which only ospkgs can pass to apt-get, or an unreadable include is +# left to ospkgs whole; ospkgs still applies the whole list after the install. +my %PRESEED_TYPE = map { $_ => 1 } qw(string boolean select multiselect note password text seen title error); + +sub ubuntu_autoinstall_packages +{ + my @records = grep { defined } @_; + return () if grep { /#(?:ENV:|INCLUDEBAD:)/ } @records; + my (@packages, %seen); + RECORD: foreach my $record (@records) { + my @tokens = grep { length } split( /\s+/, $record ); + next unless @tokens; + next if @tokens >= 3 && $PRESEED_TYPE{ $tokens[2] }; + next if $tokens[0] =~ /^[-@]/; # ospkgs removes or installs the whole record + my @found; + foreach my $token (@tokens) { + last if $token =~ /^#/; + next if $token =~ /^[-@]/ || $token =~ /-$/; + next RECORD unless $token =~ m{^[a-z0-9][a-z0-9+.-]*(?::[a-z0-9-]+)?(?:[=/][^\s/=]+|\^)?$}; + next if $token =~ m{[:=/]}; + push @found, $token; + } + push @packages, grep { !$seen{$_}++ } @found; + } + return @packages; +} + +# ubuntu_autoinstall_items: the list items for the pkglist packages at the token's indentation, +# leaving out packages the items above the token already name, each quoted so a name such as null +# or true stays a string. The time daemons exclude each +# other, so when the template names one, the pkglist's stay with ospkgs, as they did before. +my %UBUNTU_TIME_DAEMON = map { $_ => 1 } qw(chrony ntp ntpsec ntpdate ntpsec-ntpdate openntpd systemd-timesyncd); + +sub ubuntu_autoinstall_items +{ + my ($indent, $listed, $packages) = @_; + my %named = map { $_ => 1 } ( $listed =~ /^[ \t]*- (\S+)[ \t]*$/mg ); + my $fixed_time_daemon = grep { $UBUNTU_TIME_DAEMON{$_} } keys %named; + my @items = grep { !$named{$_} } @$packages; + @items = grep { !$UBUNTU_TIME_DAEMON{ (split /[:=\/^]/, $_)[0] } } @items if $fixed_time_daemon; + return join( '', map { "$indent- \"$_\"\n" } @items ); +} + sub ubuntu_subiquity_apt_mirror { my ($osarch) = @_; @@ -1786,13 +1847,28 @@ sub ubuntu_subiquity_apt_mirror return ($ent && defined($ent->{value}) && length($ent->{value})) ? $ent->{value} : $default; } +# The key curtin names in the Deb822 source it writes for the primary apt mirror on 24.04 and later. +my $UBUNTU_ARCHIVE_KEYRING = '/usr/share/keyrings/ubuntu-archive-keyring.gpg'; + sub ubuntu_subiquity_apt_config { - my ($media_dir, $osarch) = @_; + my ($media_dir, $osarch, $pkgdirs) = @_; my $use_deb822 = ubuntu_subiquity_uses_deb822_sources($media_dir); - my @otherpkg_sources = ubuntu_subiquity_otherpkg_sources(); + my $online_mirror = ubuntu_subiquity_apt_mirror($osarch); + my $mirror_key = $use_deb822 ? $UBUNTU_ARCHIVE_KEYRING : ''; + my @otherpkg_sources = map { ubuntu_subiquity_otherpkg_source_spec( $_, $mirror_key, $online_mirror ) } ubuntu_subiquity_otherpkg_sources(); + my @pkgdir_sources = ubuntu_subiquity_pkgdir_source_specs( $pkgdirs, $mirror_key, $online_mirror ); + + # apt rejects two sources for one repository whose options differ, so a pkgdir entry that repeats an + # otherpkgs repository adds its components to that source instead + my %otherpkg_by_key = map { ( my $uri = $_->{uri} ) =~ s{/+$}{}; ( "$uri $_->{suites}" => $_ ) } @otherpkg_sources; + @pkgdir_sources = grep { + ( my $uri = $_->{uri} ) =~ s{/+$}{}; + my $other = $otherpkg_by_key{"$uri $_->{suites}"}; + ubuntu_subiquity_add_components( $other, $_->{components} ) if $other; + !$other; + } @pkgdir_sources; - my $online_mirror = ubuntu_subiquity_apt_mirror($osarch); if ($online_mirror) { # Online install: use the configured archive as the primary apt mirror so # Subiquity/curtin can fetch whatever the minimal media lacks. No @@ -1801,6 +1877,7 @@ sub ubuntu_subiquity_apt_config ' apt:', ' preserve_sources_list: false', ' geoip: false', + q( conf: 'APT::Install-Recommends "false";'), ' mirror-selection:', ' primary:', " - uri: $online_mirror", @@ -1818,12 +1895,16 @@ sub ubuntu_subiquity_apt_config push @lines, ' xcat-ubuntu-updates.list:'; push @lines, qq( source: "deb $online_mirror \$RELEASE-updates main restricted universe multiverse"); } - if (@otherpkg_sources) { + if (@otherpkg_sources || @pkgdir_sources) { push @lines, ' sources:' unless $need_sources_block; my $index = 0; foreach my $source (@otherpkg_sources) { - push @lines, " xcat-otherpkgs-$index.list:"; - push @lines, qq( source: "deb [trusted=yes] $source ./"); + push @lines, ubuntu_subiquity_source_lines( "xcat-otherpkgs-$index", $source, $use_deb822 ); + $index++; + } + $index = 0; + foreach my $source (@pkgdir_sources) { + push @lines, ubuntu_subiquity_source_lines( "xcat-pkgdir-$index", $source, $use_deb822 ); $index++; } } @@ -1835,6 +1916,7 @@ sub ubuntu_subiquity_apt_config ' preserve_sources_list: false', ' fallback: offline-install', ' geoip: false', + q( conf: 'APT::Install-Recommends "false";'), ' disable_suites:', ' - updates', ' - backports', @@ -1864,22 +1946,36 @@ sub ubuntu_subiquity_apt_config foreach my $source (@otherpkg_sources) { push @lines, ''; push @lines, ' Types: deb'; - push @lines, " URIs: $source"; - push @lines, ' Suites: ./'; - push @lines, ' Components:'; + push @lines, " URIs: $source->{uri}"; + push @lines, " Suites: $source->{suites}"; + push @lines, ' Components:' . ( length $source->{components} ? " $source->{components}" : '' ); push @lines, ' Trusted: yes'; } + foreach my $source (@pkgdir_sources) { + push @lines, ''; + push @lines, ' Types: deb'; + push @lines, " URIs: $source->{uri}"; + push @lines, " Suites: $source->{suites}"; + push @lines, ' Components:' . ( length $source->{components} ? " $source->{components}" : '' ); + push @lines, ' Trusted: yes' if $source->{trusted}; + } } else { push @lines, ' mirror-selection:'; push @lines, ' primary:'; push @lines, ' - uri: file:/cdrom'; - if (@otherpkg_sources) { + if (@otherpkg_sources || @pkgdir_sources) { push @lines, ' sources:'; my $index = 0; foreach my $source (@otherpkg_sources) { push @lines, " xcat-otherpkgs-$index.list:"; - push @lines, qq( source: "deb [trusted=yes] $source ./"); + push @lines, qq( source: "$source->{line}"); + $index++; + } + $index = 0; + foreach my $source (@pkgdir_sources) { + push @lines, " xcat-pkgdir-$index.list:"; + push @lines, qq( source: "$source->{line}"); $index++; } } @@ -1888,6 +1984,56 @@ sub ubuntu_subiquity_apt_config return join( "\n", @lines ); } +# ubuntu_subiquity_pkgdir_source_specs: the apt sources of the entries after the install media in +# an osimage pkgdir value, which mkinstall hands over as pkgdirs and ospkgs receives as OSPKGDIR. +# An entry written as "URL suite components" is an apt source line, as ospkgs writes it, and a +# suite that is an exact path needs no component. A local directory that is a flat repository is +# served by the management node and trusted, as an otherpkgdir is. Anything else is no apt source +# for ospkgs either and is left out. An entry that names an Ubuntu archive mirror the installer +# already has a source for, the configured one or a default one, carries that source's signing +# key, the archive keyring on the Deb822 releases and none before them: apt rejects a second source +# for the same suite whose signing key differs. +sub ubuntu_subiquity_pkgdir_source_specs +{ + my ( $pkgdirval, $mirror_key, @mirrors ) = @_; + $mirror_key //= ''; + my %signed_uri = ubuntu_subiquity_signed_mirror_uris(@mirrors); + my @specs; + foreach my $entry ( split( /,/, $pkgdirval // '' ) ) { + $entry =~ s/^\s+|\s+$//g; + next if $entry eq ''; + if ( $entry =~ m{^https?://} ) { + my ( $uri, $suite, @components ) = split( /\s+/, $entry ); + next unless defined $suite && ( @components || $suite =~ m{/$} ); + ( my $bare = $uri ) =~ s{/+$}{}; + my %spec = ( uri => $uri, suites => $suite, components => join( ' ', @components ), trusted => 0, signed_by => $signed_uri{$bare} ? $mirror_key : '' ); + $spec{line} = ubuntu_subiquity_source_line( \%spec ); + push @specs, \%spec; + } + elsif ( $entry !~ m{^[a-z]+://} && ubuntu_subiquity_local_apt_repo($entry) ) { + my $uri = ubuntu_subiquity_pkgdir_uri($entry); + my %spec = ( uri => $uri, suites => './', components => '', trusted => 1, signed_by => '' ); + $spec{line} = ubuntu_subiquity_source_line( \%spec ); + push @specs, \%spec; + } + } + + # a directory and its own URL are one repository: one source, with the trust and components of both + my ( %kept, @unique ); + foreach my $spec (@specs) { + ( my $uri = $spec->{uri} ) =~ s{/+$}{}; + if ( my $first = $kept{"$uri $spec->{suites}"} ) { + $first->{trusted} ||= $spec->{trusted}; + $first->{signed_by} ||= $spec->{signed_by}; + ubuntu_subiquity_add_components( $first, $spec->{components} ); + next; + } + push @unique, $kept{"$uri $spec->{suites}"} = $spec; + } + return @unique; +} + + sub ubuntu_subiquity_otherpkg_sources { my $nodetype_tab = xCAT::Table->new('nodetype'); @@ -1915,6 +2061,73 @@ sub ubuntu_subiquity_otherpkg_sources return @sources; } +# ubuntu_subiquity_add_components: the components of a repeated repository join the source kept for it. +sub ubuntu_subiquity_add_components +{ + my ( $spec, $components ) = @_; + my %have = map { $_ => 1 } split( ' ', $spec->{components} ); + $spec->{components} = join( ' ', split( ' ', $spec->{components} ), grep { !$have{$_}++ } split( ' ', $components // '' ) ); + $spec->{line} = ubuntu_subiquity_source_line($spec); + return; +} + +# ubuntu_subiquity_signed_mirror_uris: the apt mirror the installer already has a source for, the +# configured one or the architecture default, without a trailing slash. +sub ubuntu_subiquity_signed_mirror_uris +{ + my (@mirrors) = @_; + return map { ( my $uri = $_ ) =~ s{/+$}{}; ( $uri => 1 ) } grep { defined && length } @mirrors; +} + +# ubuntu_subiquity_source_line: the one-line form of a source, with the option its Deb822 form carries. +sub ubuntu_subiquity_source_line +{ + my ($spec) = @_; + my @option = $spec->{trusted} ? ('[trusted=yes]') : $spec->{signed_by} ? ("[signed-by=$spec->{signed_by}]") : (); + return join( ' ', 'deb', @option, $spec->{uri}, $spec->{suites}, grep { length } $spec->{components} ); +} + +# ubuntu_subiquity_otherpkg_source_spec: the apt source of one otherpkgdir entry the installer gets. +# A bare URL or a local repository is a flat trusted repository, as otherpkgs treats it. An entry +# written as URL, suite and components is that source, trusted as well, unless the URL is an Ubuntu +# archive mirror the installer already has a source for: that one gets the same signing key and no +# trust, since apt rejects a second source for one suite whose options differ. +sub ubuntu_subiquity_otherpkg_source_spec +{ + my ( $entry, $mirror_key, @mirrors ) = @_; + my ( $uri, $suite, @components ) = split( /\s+/, $entry ); + my %spec = ( uri => $uri, suites => './', components => '', trusted => 1, signed_by => '' ); + if ( defined $suite && length $suite ) { + my %signed = ubuntu_subiquity_signed_mirror_uris(@mirrors); + ( my $bare = $uri ) =~ s{/+$}{}; + @spec{qw(suites components)} = ( $suite, join( ' ', @components ) ); + @spec{qw(trusted signed_by)} = ( 0, $mirror_key // '' ) if $signed{$bare}; + } + $spec{line} = ubuntu_subiquity_source_line( \%spec ); + return \%spec; +} + +# ubuntu_subiquity_source_lines: one entry of the autoinstall sources mapping, a one-line source +# before Deb822 and a Deb822 stanza from 24.04 on: curtin converts a one-line source to Deb822 +# there and keeps only its type, URI, suite and components, so a trusted repository would come out +# unsigned and be rejected. +sub ubuntu_subiquity_source_lines +{ + my ( $name, $source, $use_deb822 ) = @_; + return ( " $name.list:", qq( source: "$source->{line}") ) unless $use_deb822; + my @lines = ( + " $name.sources:", + ' source: |', + ' Types: deb', + " URIs: $source->{uri}", + " Suites: $source->{suites}", + ' Components:' . ( length $source->{components} ? " $source->{components}" : '' ), + ); + push @lines, " Signed-By: $source->{signed_by}" if $source->{signed_by}; + push @lines, ' Trusted: yes' if $source->{trusted}; + return @lines; +} + sub ubuntu_subiquity_uses_deb822_sources { my ($media_dir) = @_; diff --git a/xCAT-server/lib/xcat/plugins/debian.pm b/xCAT-server/lib/xcat/plugins/debian.pm index 2a4c2fb39..e085efc28 100644 --- a/xCAT-server/lib/xcat/plugins/debian.pm +++ b/xCAT-server/lib/xcat/plugins/debian.pm @@ -895,6 +895,7 @@ sub mkinstall { my $partitionfile; my $pkgdir; my $pkgdirval; + my $environvar; my @mirrors; my $pkglistfile; my $imagename; # set it if running of 'nodeset osimage=xxx' @@ -917,12 +918,13 @@ sub mkinstall { if (!$osimagetab) { $osimagetab = xCAT::Table->new('osimage', -create => 1); } - (my $ref) = $osimagetab->getAttribs({ imagename => $imagename }, 'osvers', 'osarch', 'profile', 'provmethod'); + (my $ref) = $osimagetab->getAttribs({ imagename => $imagename }, 'osvers', 'osarch', 'profile', 'provmethod', 'environvar'); if ($ref) { $img_hash{$imagename}->{osver} = $ref->{'osvers'}; $img_hash{$imagename}->{osarch} = $ref->{'osarch'}; $img_hash{$imagename}->{profile} = $ref->{'profile'}; $img_hash{$imagename}->{provmethod} = $ref->{'provmethod'}; + $img_hash{$imagename}->{environvar} = $ref->{'environvar'}; if (!$linuximagetab) { $linuximagetab = xCAT::Table->new('linuximage', -create => 1); } @@ -995,6 +997,7 @@ sub mkinstall { $tmplfile = $ph->{template}; $pkgdirval = $ph->{pkgdir}; + $environvar = $ph->{environvar}; my @pkgdirlist = split(/,/, $pkgdirval); foreach (@pkgdirlist) { if ($_ =~ /^http|ssh/) { @@ -1123,7 +1126,9 @@ sub mkinstall { $platform, $partitionfile, \%tmpl_hash, - osarch => $arch + osarch => $arch, + pkgdirs => $pkgdirval, + environvar => $environvar ); } diff --git a/xCAT-server/share/xcat/install/ubuntu/compute.subiquity.tmpl b/xCAT-server/share/xcat/install/ubuntu/compute.subiquity.tmpl index 5918a61e3..328ccd258 100644 --- a/xCAT-server/share/xcat/install/ubuntu/compute.subiquity.tmpl +++ b/xCAT-server/share/xcat/install/ubuntu/compute.subiquity.tmpl @@ -43,6 +43,7 @@ autoinstall: - bind9-dnsutils - chrony - gpg + - #INCLUDE_DEFAULT_PKGLIST_AUTOINSTALL# early-commands: - | exec >/tmp/pre-install.log 2>&1 @@ -111,6 +112,11 @@ autoinstall: cp ./#HOSTNAME#.post /target/root/post.script; curtin in-target --target /target /root/post.script; } >>/target/var/log/xcat/xcat.log 2>&1' + # The installer's sources for the otherpkgs repository and the pkgdir mirrors, and the apt + # configuration that kept recommended packages out, served the install; ospkgs and otherpkgs + # write their own after the first boot. A separate item, so the status of the post script above + # still decides whether the install goes on. + - rm -f /target/etc/apt/sources.list.d/xcat-otherpkgs-*.list /target/etc/apt/sources.list.d/xcat-otherpkgs-*.sources /target/etc/apt/sources.list.d/xcat-pkgdir-*.list /target/etc/apt/sources.list.d/xcat-pkgdir-*.sources /target/etc/apt/apt.conf.d/94curtin-config # Flip the node to local-disk boot, or it PXE-loops back into the installer on reboot. # xcatd's install monitor greets with "ready", then answers "next" with "done" and runs # "nodeset next". Require both tokens: another service on that port is not a flipped diff --git a/xCAT-server/share/xcat/install/ubuntu/compute.ubuntu20.04.pkglist b/xCAT-server/share/xcat/install/ubuntu/compute.ubuntu20.04.pkglist new file mode 100644 index 000000000..aaec41835 --- /dev/null +++ b/xCAT-server/share/xcat/install/ubuntu/compute.ubuntu20.04.pkglist @@ -0,0 +1,5 @@ +openssh-server +chrony +gawk +nfs-common +snmpd diff --git a/xCAT-server/share/xcat/install/ubuntu/compute.ubuntu22.04.pkglist b/xCAT-server/share/xcat/install/ubuntu/compute.ubuntu22.04.pkglist new file mode 100644 index 000000000..aaec41835 --- /dev/null +++ b/xCAT-server/share/xcat/install/ubuntu/compute.ubuntu22.04.pkglist @@ -0,0 +1,5 @@ +openssh-server +chrony +gawk +nfs-common +snmpd diff --git a/xCAT-server/share/xcat/install/ubuntu/compute.ubuntu24.04.pkglist b/xCAT-server/share/xcat/install/ubuntu/compute.ubuntu24.04.pkglist new file mode 100644 index 000000000..aaec41835 --- /dev/null +++ b/xCAT-server/share/xcat/install/ubuntu/compute.ubuntu24.04.pkglist @@ -0,0 +1,5 @@ +openssh-server +chrony +gawk +nfs-common +snmpd diff --git a/xCAT-test/unit/ubuntu_shared_pkglists.t b/xCAT-test/unit/ubuntu_shared_pkglists.t index 0fa526fdc..3046c3985 100644 --- a/xCAT-test/unit/ubuntu_shared_pkglists.t +++ b/xCAT-test/unit/ubuntu_shared_pkglists.t @@ -39,13 +39,20 @@ sub resolved { } # The shared lists keep ntp for the releases that still carry it. aarch64 has no list of its own. -foreach my $case ( [ $install, 'compute' ], [ $install, 'service' ], [ $install, 'kvm' ], [ $netboot, 'compute' ] ) { +# The install compute profile has a 24.04 list of its own, with chrony: the Subiquity template +# installs chrony and ntp cannot join it in one apt transaction (ubuntu_subiquity_pkglists.t). +foreach my $case ( [ $install, 'service' ], [ $install, 'kvm' ], [ $netboot, 'compute' ] ) { my ( $dir, $profile ) = @$case; my ( $file, $p ) = resolved( $dir, $profile, 'ubuntu24.04.4', 'aarch64' ); is( $file, "$profile.pkglist", "$profile on 24.04 without a list of its own resolves to the shared list" ); ok( $p->{ntp}, "... which keeps ntp" ); ok( !$p->{$_}, "... and no longer names $_" ) for qw(libodbc1 qemu-kvm libvirt-bin); } +{ + my ( $file, $p ) = resolved( $install, 'compute', 'ubuntu24.04.4', 'aarch64' ); + is( $file, 'compute.ubuntu24.04.pkglist', 'the install compute profile on 24.04 resolves its own list' ); + ok( $p->{chrony} && !$p->{ntp}, '... which names chrony and not ntp' ); +} ok( packages_in("$install/service.pkglist")->{unixodbc}, 'the shared service list names unixodbc' ); ok( packages_in("$install/service.pkglist")->{'libdbd-pg-perl'}, '... and the PostgreSQL driver beside the MySQL one' ); diff --git a/xCAT-test/unit/ubuntu_subiquity_apt_sources.t b/xCAT-test/unit/ubuntu_subiquity_apt_sources.t index 0c55d0e8c..d1e53b9d2 100644 --- a/xCAT-test/unit/ubuntu_subiquity_apt_sources.t +++ b/xCAT-test/unit/ubuntu_subiquity_apt_sources.t @@ -21,11 +21,32 @@ sub apt_config_for { no warnings 'redefine'; local *xCAT::Template::ubuntu_subiquity_apt_mirror = sub { $opt{mirror} }; local *xCAT::Template::ubuntu_subiquity_uses_deb822_sources = sub { $opt{deb822} }; - local *xCAT::Template::ubuntu_subiquity_otherpkg_sources = sub { () }; + local *xCAT::Template::ubuntu_subiquity_otherpkg_sources = sub { @{ $opt{others} || [] } }; local *xCAT::Template::ubuntu_subiquity_uses_generated_cdrom_source = sub { 0 }; return xCAT::Template::ubuntu_subiquity_apt_config('/some/media/dir'); } +# --- the otherpkgs repository: a one-line source before Deb822, a Deb822 stanza from 24.04 on, --- +# --- since curtin drops the options of a one-line source when it converts it there --- +my $others = [ 'http://192.0.2.10/install/post/otherpkgs/ubuntu24.04/x86_64' ]; +my $classic_others = apt_config_for( mirror => $MIRROR, deb822 => 0, others => $others ); +like( $classic_others, qr{^ xcat-otherpkgs-0\.list:\n source: "deb \[trusted=yes\] http://192\.0\.2\.10/install/post/otherpkgs/ubuntu24\.04/x86_64 \./"$}m, + 'classic: the otherpkgs repository is a one-line trusted source' ); +my $deb822_others = apt_config_for( mirror => $MIRROR, deb822 => 1, others => $others ); +like( $deb822_others, + qr{^ xcat-otherpkgs-0\.sources:\n source: \|\n Types: deb\n URIs: http://192\.0\.2\.10/install/post/otherpkgs/ubuntu24\.04/x86_64\n Suites: \./\n Components:\n Trusted: yes(?:\n|\z)}m, + 'Deb822: the otherpkgs repository is a Deb822 stanza carrying Trusted: yes' ); +unlike( $deb822_others, qr/xcat-otherpkgs-0\.list|trusted=yes/, 'Deb822: and no one-line form remains' ); + +# an otherpkgdir written as URL, suite and components is that source, trusted, not a flat repository at a URL with spaces +my $mirror_others = [ 'http://mirror.example/ubuntu noble main universe' ]; +like( apt_config_for( mirror => $MIRROR, deb822 => 0, others => $mirror_others ), + qr{^ xcat-otherpkgs-0\.list:\n source: "deb \[trusted=yes\] http://mirror\.example/ubuntu noble main universe"$}m, + 'classic: an otherpkgdir mirror entry keeps its suite and components' ); +like( apt_config_for( mirror => $MIRROR, deb822 => 1, others => $mirror_others ), + qr{^ xcat-otherpkgs-0\.sources:\n source: \|\n Types: deb\n URIs: http://mirror\.example/ubuntu\n Suites: noble\n Components: main universe\n Trusted: yes(?:\n|\z)}m, + 'Deb822: and becomes a stanza with them as fields, so apt reads one URI' ); + # --- online, classic sources (20.04 / 22.04): the archive must be added via sources: --- my $classic = apt_config_for( mirror => $MIRROR, deb822 => 0 ); diff --git a/xCAT-test/unit/ubuntu_subiquity_pkgdir_sources.t b/xCAT-test/unit/ubuntu_subiquity_pkgdir_sources.t new file mode 100644 index 000000000..2dc89dfb9 --- /dev/null +++ b/xCAT-test/unit/ubuntu_subiquity_pkgdir_sources.t @@ -0,0 +1,138 @@ +#!/usr/bin/env perl +use strict; +use warnings; + +use File::Spec; +use File::Temp; +use FindBin; +use Test::More; + +# ospkgs installs the pkglist from every pkgdir entry, the media first and the mirrors after it, +# and the Subiquity autoinstall now installs the pkglist too, so the installer must be given the +# same mirrors. The specs are derived from the pkgdir value alone, and the apt configuration is +# rendered for real with only the database readers stubbed. + +my $repo = File::Spec->rel2abs( File::Spec->catdir( $FindBin::Bin, '..', '..' ) ); +my @incs = ( "$repo/perl-xCAT", "$repo/xCAT-server/lib/perl" ); +my $devnull = File::Spec->devnull(); +my $probe = join( ' ', $^X, ( map { "-I$_" } @incs ), '-e', "'require xCAT::Template; 1'", ">$devnull", "2>&1" ); +plan skip_all => 'xCAT::Template cannot be loaded here' if system($probe) != 0; +require lib; +lib->import(@incs); +require xCAT::Template; + +{ + no warnings qw(redefine once); + *xCAT::Template::ubuntu_subiquity_pkgdir_uri = sub { return "http://192.0.2.10$_[0]" }; +} +my $flat = File::Temp->newdir(); +open( my $pfh, '>', "$flat/Packages" ) or die $!; close $pfh; +open( my $rfh, '>', "$flat/Release" ) or die $!; close $rfh; +my @specs = xCAT::Template::ubuntu_subiquity_pkgdir_source_specs( + "/install/ubuntu24.04.4/x86_64, http://mirror.example/ubuntu noble main universe ,http://repo.example/extra,ssh://host/path,http://mirror.example/ubuntu jammy,http://flat.example/pool ./,$flat" +); +is( scalar(@specs), 3, 'the media path, the bare URL, the ssh entry and a suite without components are no apt sources; the rest are' ); +is_deeply( $specs[0], { uri => 'http://mirror.example/ubuntu', suites => 'noble', components => 'main universe', trusted => 0, signed_by => '', + line => 'deb http://mirror.example/ubuntu noble main universe' }, + 'an entry written as URL suite components is used as written, and is not marked trusted' ); +is_deeply( $specs[1], { uri => 'http://flat.example/pool', suites => './', components => '', trusted => 0, signed_by => '', + line => 'deb http://flat.example/pool ./' }, + 'an exact-path suite needs no component' ); +is_deeply( $specs[2], { uri => "http://192.0.2.10$flat", suites => './', components => '', trusted => 1, signed_by => '', + line => "deb [trusted=yes] http://192.0.2.10$flat ./" }, + 'a local flat repository is served by the management node and trusted, as an otherpkgdir is' ); +is_deeply( [ xCAT::Template::ubuntu_subiquity_pkgdir_source_specs(undef) ], [], 'no pkgdir gives no sources' ); +my @alias = xCAT::Template::ubuntu_subiquity_pkgdir_source_specs("$flat,http://192.0.2.10$flat/ ./"); +is( scalar(@alias), 1, 'a local repository and its own URL in pkgdir are one source' ); +is( $alias[0]{trusted}, 1, '... trusted, as the directory entry is' ); +my $keyring = '/usr/share/keyrings/ubuntu-archive-keyring.gpg'; +is_deeply( + [ xCAT::Template::ubuntu_subiquity_pkgdir_source_specs( 'http://archive.example/ubuntu/ noble-proposed main,http://mirror.example/ubuntu noble main', $keyring, 'http://archive.example/ubuntu' ) ], + [ { uri => 'http://archive.example/ubuntu/', suites => 'noble-proposed', components => 'main', trusted => 0, signed_by => $keyring, + line => "deb [signed-by=$keyring] http://archive.example/ubuntu/ noble-proposed main" }, + { uri => 'http://mirror.example/ubuntu', suites => 'noble', components => 'main', trusted => 0, signed_by => '', line => 'deb http://mirror.example/ubuntu noble main' } ], + 'an entry that names the apt mirror, trailing slash or not, carries the key given for that mirror; another mirror gets none' ); +is_deeply( + [ xCAT::Template::ubuntu_subiquity_pkgdir_source_specs( 'http://archive.example/ubuntu jammy main', '', 'http://archive.example/ubuntu' ) ], + [ { uri => 'http://archive.example/ubuntu', suites => 'jammy', components => 'main', trusted => 0, signed_by => '', line => 'deb http://archive.example/ubuntu jammy main' } ], + 'without a key for the mirror the entry is used as written' ); + +our ( $apt_mirror, @otherpkg_sources ) = ( '', () ); +{ + no warnings qw(redefine once); + *xCAT::Template::ubuntu_subiquity_apt_mirror = sub { return $main::apt_mirror }; + *xCAT::Template::ubuntu_subiquity_otherpkg_sources = sub { return @main::otherpkg_sources }; +} +# the value mkinstall hands over: the media first, then two mirrors +my $mirrors = '/install/ubuntu24.04.4/x86_64,http://mirror.example/ubuntu noble main,http://repo.example/extra'; + +sub apt_config_for { + my ( $media_dir, %args ) = @_; + local $apt_mirror = $args{mirror} // ''; + local @otherpkg_sources = @{ $args{others} || [] }; + return xCAT::Template::ubuntu_subiquity_apt_config( $media_dir, undef, $args{pkgdirs} ); +} + +my $online = apt_config_for( 'ubuntu24.04', mirror => 'http://archive.example/ubuntu', pkgdirs => $mirrors ); +like( $online, qr/^ sources:$/m, 'online: the pkgdir mirrors open the sources mapping' ); +like( $online, qr{^ xcat-pkgdir-0\.sources:\n source: \|\n Types: deb\n URIs: http://mirror\.example/ubuntu\n Suites: noble\n Components: main(?:\n|\z)}m, + '... as a Deb822 stanza on a Deb822 release, with the suite and components as written' ); +unlike( $online, qr{xcat-pkgdir-1|repo\.example}, '... and the bare URL is not offered as a repository' ); + +my $online_both = apt_config_for( 'ubuntu22.04', mirror => 'http://archive.example/ubuntu', others => ['http://mn/otherpkgs'], pkgdirs => $mirrors ); +is( scalar( () = $online_both =~ /^ sources:$/mg ), 1, 'online with otherpkgs and pkgdir mirrors: one sources mapping' ); +like( $online_both, qr/xcat-otherpkgs-0\.list:.*xcat-pkgdir-0\.list:/s, '... otherpkgs first, then the mirror' ); + +my $offline_deb822 = apt_config_for( 'ubuntu24.04', pkgdirs => $mirrors ); +like( $offline_deb822, qr{^ URIs: http://mirror\.example/ubuntu\n Suites: noble\n Components: main(?:\n|\z)}m, 'offline Deb822: a stanza per mirror' ); +unlike( $offline_deb822, qr{repo\.example|Trusted: yes\n(?:.*\n)* URIs: http://mirror}, '... nothing trusted and no bare URL' ); + +my $offline_legacy = apt_config_for( 'ubuntu22.04', pkgdirs => $mirrors ); +like( $offline_legacy, qr{^ sources:\n xcat-pkgdir-0\.list:\n source: "deb http://mirror\.example/ubuntu noble main"$}m, 'offline legacy: .list files under sources' ); + +my $same = apt_config_for( 'ubuntu24.04', mirror => 'http://mirror.example/ubuntu', pkgdirs => 'http://mirror.example/ubuntu noble-proposed main' ); +like( $same, qr{^ xcat-pkgdir-0\.sources:\n source: \|\n Types: deb\n URIs: http://mirror\.example/ubuntu\n Suites: noble-proposed\n Components: main\n Signed-By: \Q$keyring\E(?:\n|\z)}m, + 'another suite of the apt mirror is added with the signing key the installer gives that mirror' ); + +# before Deb822 the archive sources are rendered here without a key, so a repeated mirror entry must carry none either +foreach my $release ( [ 'ubuntu20.04', 'focal' ], [ 'ubuntu22.04', 'jammy' ] ) { + my ( $media, $suite ) = @$release; + my $legacy_same = apt_config_for( $media, mirror => 'http://mirror.example/ubuntu', pkgdirs => "http://mirror.example/ubuntu $suite main" ); + like( $legacy_same, qr{^ xcat-pkgdir-0\.list:\n source: "deb http://mirror\.example/ubuntu \Q$suite\E main"$}m, + "$media: an entry that repeats the apt mirror carries no signing key, like the archive sources rendered next to it" ); + like( $legacy_same, qr{^ xcat-ubuntu-archive\.list:\n source: "deb http://mirror\.example/ubuntu \$RELEASE main restricted universe multiverse"$}m, + "$media: which stay as they were" ); +} + +# a repository named by otherpkgdir and by pkgdir is offered once, trusted, on both source forms +foreach my $media ( 'ubuntu24.04', 'ubuntu22.04' ) { + my $overlap = apt_config_for( $media, mirror => 'http://archive.example/ubuntu', others => ['http://repo.example/ubuntu'], + pkgdirs => '/install/ubuntu24.04.4/x86_64,http://repo.example/ubuntu/ ./' ); + like( $overlap, qr/xcat-otherpkgs-0\./, "$media: the repository the otherpkgdir names is offered as the trusted otherpkgs source" ); + unlike( $overlap, qr/xcat-pkgdir/, "$media: and not again as a pkgdir source with other options" ); +} + +# an otherpkgdir entry that names the apt mirror itself gets the options of the source the installer already has for it +my $archive_others = ['http://mirror.example/ubuntu noble universe']; +like( apt_config_for( 'ubuntu22.04', mirror => 'http://mirror.example/ubuntu', others => $archive_others ), + qr{^ xcat-otherpkgs-0\.list:\n source: "deb http://mirror\.example/ubuntu noble universe"$}m, + 'classic: an otherpkgdir entry for the apt mirror carries no option, like the archive sources next to it' ); +like( apt_config_for( 'ubuntu24.04', mirror => 'http://mirror.example/ubuntu', others => $archive_others ), + qr{^ xcat-otherpkgs-0\.sources:\n source: \|\n Types: deb\n URIs: http://mirror\.example/ubuntu\n Suites: noble\n Components: universe\n Signed-By: \Q$keyring\E(?:\n|\z)}m, + 'Deb822: and carries the archive keyring and no Trusted, as the primary source does' ); + +# the same repository and suite with other components: the pkgdir components join the otherpkgs source +my $union = apt_config_for( 'ubuntu24.04', mirror => 'http://archive.example/ubuntu', others => ['http://repo.example/team stable tools'], + pkgdirs => '/install/ubuntu24.04.4/x86_64,http://repo.example/team/ stable compute tools' ); +like( $union, qr{^ xcat-otherpkgs-0\.sources:\n source: \|\n Types: deb\n URIs: http://repo\.example/team\n Suites: stable\n Components: tools compute\n Trusted: yes(?:\n|\z)}m, + 'a pkgdir entry that repeats an otherpkgs repository adds its components to that source' ); +unlike( $union, qr/xcat-pkgdir/, '... and is not a second source' ); +my $union_legacy = apt_config_for( 'ubuntu22.04', mirror => 'http://archive.example/ubuntu', others => ['http://repo.example/team stable tools'], + pkgdirs => '/install/ubuntu24.04.4/x86_64,http://repo.example/team stable compute' ); +like( $union_legacy, qr{^ xcat-otherpkgs-0\.list:\n source: "deb \[trusted=yes\] http://repo\.example/team stable tools compute"$}m, + 'in the one-line form as well' ); + +my $none = apt_config_for( 'ubuntu24.04', mirror => 'http://archive.example/ubuntu' ); +unlike( $none, qr/sources:/, 'without otherpkgs or mirrors no sources mapping is rendered on a Deb822 release' ); + +done_testing(); diff --git a/xCAT-test/unit/ubuntu_subiquity_pkglist.t b/xCAT-test/unit/ubuntu_subiquity_pkglist.t new file mode 100644 index 000000000..05d15c361 --- /dev/null +++ b/xCAT-test/unit/ubuntu_subiquity_pkglist.t @@ -0,0 +1,200 @@ +#!/usr/bin/env perl +use strict; +use warnings; +no warnings 'once'; + +use FindBin; +use lib "$FindBin::Bin/../lib"; +use File::Spec; +use File::Temp; +use Test::More; + +use XCAT::Test::File qw(repo_path); + +sub read_text { my ($path) = @_; open( my $fh, '<', $path ) or die "$path: $!"; local $/; my $text = <$fh>; close($fh); return $text; } +sub write_text { my ( $path, $text ) = @_; open( my $fh, '>', $path ) or die "$path: $!"; print {$fh} $text; close($fh); return; } + +# A Subiquity autoinstall installs the packages of its user-data packages list, and until now the +# template named a fixed set, so the osimage pkglist reached the node only through ospkgs after +# the first boot. The template can now carry #INCLUDE_DEFAULT_PKGLIST_AUTOINSTALL# on a list line, +# and Template.pm renders one list item per pkglist package in its place. + +my $module = repo_path('xCAT-server/lib/perl/xCAT/Template.pm'); +plan skip_all => 'Template.pm not found' unless -r $module; + +my @incs = ( repo_path('perl-xCAT'), repo_path('xCAT-server/lib/perl') ); + +# Every xCAT module prepends $XCATROOT/lib/perl as it compiles, so on a host with xCAT installed +# the modules Template.pm loads afterwards would come from /opt/xcat. Point it at the checkout. +my $xcatroot = File::Temp->newdir(); +mkdir "$xcatroot/lib" or die "$xcatroot/lib: $!"; +symlink( repo_path('xCAT-server/lib/perl'), "$xcatroot/lib/perl" ) or die "symlink: $!"; +$ENV{XCATROOT} = "$xcatroot"; + +my $devnull = File::Spec->devnull(); +my $probe = join( ' ', $^X, ( map { "-I$_" } @incs ), '-e', "'require xCAT::Template; 1'", ">$devnull", "2>&1" ); +plan skip_all => 'xCAT::Template cannot be loaded here' if system($probe) != 0; + +require lib; +lib->import(@incs); +require xCAT::Template; +require xCAT::Postage; # the pkglist reader Template.pm calls, loaded by the plugin in production + +# ---- the record filter: only what apt can be asked for in an autoinstall packages list -------- +my @packages = xCAT::Template::ubuntu_autoinstall_packages( + 'openssh-server', ' gawk', 'ntp', '-snmpd', '@core', '#NEW_INSTALL_LIST#', 'd-i pkgsel/include string foo', + 'd-i tasksel/first multiselect standard,not-a-real-package', 'nfs-common=1:2.6.4-3ubuntu5', 'gawk', 'Bad_Name', '', + 'libc6', 'libc6:i386', 'curl/noble', 'dns-server^', 'a/b/c', 'vim rsync -busybox-static gpg', '@Group With Space', 'wget Bad_Name', 'bc # a calculator', + 'wget-', 'libc6:i386-', 'tree+', '-snmpd apache2', '@core bc' +); +is_deeply( \@packages, [qw(openssh-server gawk ntp libc6 dns-server^ vim rsync gpg bc tree+)], + 'names and tasks are kept once each, a space-separated line gives each package, a comment ends it; pins, target releases, architecture qualifiers, removals in either hyphen form, a record that begins with a removal or a group, markers, directives and unknown syntax are not' ); +is_deeply( [ xCAT::Template::ubuntu_autoinstall_packages( 'msodbcsql18', '#ENV:ACCEPT_EULA=Y#', 'gawk' ) ], [], + 'a list with an apt environment setting stays with ospkgs whole' ); +is_deeply( [ xCAT::Template::ubuntu_autoinstall_packages( 'gawk', 'msodbcsql18 #ENV:ACCEPT_EULA=Y#' ) ], [], + 'so does a list with the setting after a package on the same line, where get_envlist reads it too' ); +is_deeply( [ xCAT::Template::ubuntu_autoinstall_packages( 'gawk', '#INCLUDEBAD:cannot open pkglist file /absent.pkglist#' ) ], [], + 'a list with an unreadable include stays with ospkgs whole' ); +is_deeply( [ xCAT::Template::ubuntu_autoinstall_packages() ], [], 'no records give no packages' ); + +# ---- the record reader: lines kept whole, includes followed, the comma text unchanged --------- +{ + my $d = File::Temp->newdir(); + write_text( "$d/common.pkglist", "# shared\nnfs-common\n\@Group With Space\n" ); + write_text( "$d/compute.pkglist", "openssh-server\n # a comment\nd-i tasksel/first multiselect standard,not-a-real-package\n#INCLUDE:$d/common.pkglist#\n#NEW_INSTALL_LIST#\nchrony\n" ); + my @records = xCAT::Postage->get_pkglist_records("$d/compute.pkglist"); + is_deeply( \@records, + [ 'openssh-server', 'd-i tasksel/first multiselect standard,not-a-real-package', 'nfs-common', '@Group With Space', '#NEW_INSTALL_LIST#', 'chrony' ], + 'records are whole lines, comments dropped, the include expanded in place' ); + is( xCAT::Postage->get_pkglist_tex("$d/compute.pkglist"), + 'openssh-server,d-i tasksel/first multiselect standard,not-a-real-package,nfs-common,@Group With Space,#NEW_INSTALL_LIST#,chrony', + 'the comma text ospkgs receives is unchanged, and cannot tell the directive comma apart' ); + my @missing = xCAT::Postage->get_pkglist_records("$d/absent.pkglist"); + like( $missing[0], qr/^#INCLUDEBAD:/, 'an unreadable file yields the INCLUDEBAD marker record' ); + + # top.pkglist includes sub/common.pkglist, which includes leaf.pkglist: the leaf next to top.pkglist is the one meant + mkdir "$d/sub" or die "$d/sub: $!"; + write_text( "$d/top.pkglist", "#INCLUDE:sub/common.pkglist#\n" ); + write_text( "$d/sub/common.pkglist", "#INCLUDE:leaf.pkglist#\n" ); + write_text( "$d/leaf.pkglist", "nfs-common\n" ); + write_text( "$d/sub/leaf.pkglist", "snmpd\n" ); + is_deeply( [ xCAT::Postage->get_pkglist_records("$d/top.pkglist") ], ['nfs-common'], + 'a nested include resolves against the directory of the listed pkglist' ); + is_deeply( [ xCAT::Postage->get_pkglist_records("$d/top.pkglist") ], [ split /,/, xCAT::Postage->get_pkglist_tex("$d/top.pkglist") ], + 'and reads the same files get_pkglist_tex reads' ); + + write_text( "$d/note.pkglist", "#INCLUDE:leaf.pkglist# # the shared leaf\nbc # a calculator\n" ); + my @noted = xCAT::Postage->get_pkglist_records("$d/note.pkglist"); + is_deeply( \@noted, [ split /,/, xCAT::Postage->get_pkglist_tex("$d/note.pkglist") ], + 'an include followed by a note is expanded, the note staying on the last record as get_pkglist_tex leaves it' ); + is_deeply( [ xCAT::Template::ubuntu_autoinstall_packages(@noted) ], [qw(nfs-common bc)], 'and the notes add no packages' ); +} +is( xCAT::Template::ubuntu_autoinstall_items( " ", " - wget\n - gpg\n", [qw(gawk gpg chrony)] ), + " - \"gawk\"\n - \"chrony\"\n", 'items already listed above the token are not repeated, and every item is a quoted string' ); +is( xCAT::Template::ubuntu_autoinstall_items( " ", "", [qw(null true 12)] ), " - \"null\"\n - \"true\"\n - \"12\"\n", + 'names YAML would read as null, boolean or number stay strings' ); +is( xCAT::Template::ubuntu_autoinstall_items( " ", " - wget\n - chrony\n", [qw(gawk ntp ntpdate snmpd)] ), + " - \"gawk\"\n - \"snmpd\"\n", 'a time daemon the template installs keeps the pkglist time daemons with ospkgs' ); +is( xCAT::Template::ubuntu_autoinstall_items( " ", " - wget\n", [qw(gawk ntp)] ), + " - \"gawk\"\n - \"ntp\"\n", 'without a fixed time daemon the pkglist one is installed' ); + +# ---- the rendering: the token line becomes one item per package, at its own indentation ------- +my %site; +no warnings 'redefine', 'once'; +local *xCAT::TableUtils::get_site_attribute = sub { + my ( undef, $key ) = @_; + return defined $site{$key} ? ( $site{$key} ) : (); +}; +local *xCAT::NetworkUtils::getipaddr = sub { return '192.0.2.10'; }; +local *xCAT::Template::getPersistentKcmdline = sub { return ''; }; +use warnings; + +my $dir = File::Temp->newdir(); +my $included = File::Spec->catfile( "$dir", 'common.pkglist' ); +my $pkglist = File::Spec->catfile( "$dir", 'compute.pkglist' ); +write_text( $included, "# shared\nnfs-common\nsnmpd\n" ); +write_text( $pkglist, "openssh-server\n# a comment\nchrony rsync # time and files\nwget=1.21.2-2ubuntu1\n-ntp\nwget-\n\@standard\nd-i tasksel/first multiselect standard,not-a-real-package\n#INCLUDE:$included#\n" ); + +my $in = File::Spec->catfile( "$dir", 'in.tmpl' ); +write_text( $in, + " packages:\n" + . " - openssh-server\n" + . " - wget\n" + . " - #INCLUDE_DEFAULT_PKGLIST_AUTOINSTALL#\n" + . " late-commands:\n" + . " - echo done\n" ); + +my $render = sub { + my ( $list, %extra ) = @_; + %site = ( installdir => '/install' ); + my $out = File::Spec->catfile( "$dir", 'out.' . ( defined $list ? 'list' : 'none' ) ); + xCAT::Template->subvars( $in, $out, 'testnode', $list, '/install/ubuntu24.04/x86_64', 'ubuntu', undef, + { xcatmaster => '192.0.2.10' }, osarch => 'x86_64', %extra ); + return read_text($out); +}; + +my $rendered = $render->($pkglist); +is( $rendered, + " packages:\n" + . " - openssh-server\n" + . " - wget\n" + . " - \"chrony\"\n" + . " - \"rsync\"\n" + . " - \"nfs-common\"\n" + . " - \"snmpd\"\n" + . " late-commands:\n" + . " - echo done\n", + 'the pkglist packages, includes followed, become list items at the token indentation, without repeating the items above' ); +unlike( $rendered, qr/not-a-real-package/, 'a package name inside a preseed directive with commas is not a package' ); +unlike( $rendered, qr/wget=/, 'a version pin is not an item: it stays with ospkgs' ); +unlike( $rendered, qr/wget-/, 'a trailing-hyphen removal is not an item either: the installer would remove the package the postscripts need' ); +unlike( $rendered, qr/"(?:time|and|files)"/, 'an inline comment adds no items' ); + +# a site template that includes the stock one: the token arrives with the include and must be expanded too +my $wrapper = File::Spec->catfile( "$dir", 'wrapper.tmpl' ); +write_text( $wrapper, "#INCLUDE:$in#\n" ); +my $render_via = sub { + my ($list) = @_; + %site = ( installdir => '/install' ); + my $out = File::Spec->catfile( "$dir", 'out.wrapper.' . ( defined $list ? 'list' : 'none' ) ); + xCAT::Template->subvars( $wrapper, $out, 'testnode', $list, '/install/ubuntu24.04/x86_64', 'ubuntu', undef, { xcatmaster => '192.0.2.10' }, osarch => 'x86_64' ); + return read_text($out); +}; +is( $render_via->($pkglist), $rendered, 'a template that includes the stock one renders the same package list' ); + +my $without = $render->(undef); +is( $without, + " packages:\n - openssh-server\n - wget\n late-commands:\n - echo done\n", + 'an osimage without a pkglist keeps the template packages and loses only the token line' ); +is( $render_via->(undef), $without, 'and through an including template the token line goes as well, leaving no empty item' ); + +my $env_list = File::Spec->catfile( "$dir", 'env.pkglist' ); +my $env_included = File::Spec->catfile( "$dir", 'env-common.pkglist' ); +write_text( $env_included, "msodbcsql18 #ENV:ACCEPT_EULA=Y#\n" ); +write_text( $env_list, "gawk\n#INCLUDE:$env_included#\n" ); +is( $render->($env_list), $without, 'a pkglist whose include carries an apt environment setting is left to ospkgs whole, and the token line goes' ); +is( $render->( $pkglist, environvar => 'ACCEPT_EULA=Y' ), $without, + 'an osimage with environvar installs its pkglist through ospkgs alone, where the variables reach apt-get, and the token line goes' ); + +# the pkgdir mirrors may need those variables as well, so they stay out of the installer's apt sources too +{ + no warnings 'redefine', 'once'; + local *xCAT::Template::ubuntu_subiquity_otherpkg_sources = sub { () }; + local *xCAT::Template::ubuntu_subiquity_apt_mirror = sub { 'http://archive.example/ubuntu' }; + my $apt_in = File::Spec->catfile( "$dir", 'apt.tmpl' ); + write_text( $apt_in, "#UBUNTU_SUBIQUITY_APT_CONFIG#\n" ); + my $apt_render = sub { + my (%extra) = @_; + %site = ( installdir => '/install' ); + my $out = File::Spec->catfile( "$dir", 'out.apt' ); + xCAT::Template->subvars( $apt_in, $out, 'testnode', $pkglist, '/install/ubuntu24.04/x86_64', 'ubuntu', undef, { xcatmaster => '192.0.2.10' }, + osarch => 'x86_64', pkgdirs => '/install/ubuntu24.04/x86_64,http://mirror.example/ubuntu noble main', %extra ); + return read_text($out); + }; + like( $apt_render->(), qr{URIs: http://mirror\.example/ubuntu}, 'the pkgdir mirror joins the installer sources' ); + like( $apt_render->(), qr{^ conf: 'APT::Install-Recommends "false";'$}m, 'the installer installs without recommended packages, as ospkgs does' ); + unlike( $apt_render->( environvar => 'http_proxy=http://proxy.example:3128' ), qr{mirror\.example|xcat-pkgdir}, + 'but not for an osimage with environvar, whose mirrors may need those variables' ); +} + +done_testing(); diff --git a/xCAT-test/unit/ubuntu_subiquity_pkglists.t b/xCAT-test/unit/ubuntu_subiquity_pkglists.t new file mode 100644 index 000000000..393dee965 --- /dev/null +++ b/xCAT-test/unit/ubuntu_subiquity_pkglists.t @@ -0,0 +1,62 @@ +#!/usr/bin/env perl +use strict; +use warnings; + +use File::Spec; +use FindBin; +use Test::More; + +use lib "$FindBin::Bin/../../perl-xCAT"; +use lib "$FindBin::Bin/../../xCAT-server/lib/perl"; +use xCAT::Postage; +use xCAT::SvrUtils; + +# The Subiquity autoinstall installs the template's fixed packages and the osimage pkglist in one +# apt transaction, and the template names chrony. The shared compute.pkglist names ntp for the +# releases before Subiquity, and on the Subiquity releases ntp pulls ntpsec, which conflicts with +# chrony, so those releases need their own default list. The lists are resolved the way +# mkinstall resolves them and their packages are read the way ospkgs reads them. + +my $repo = File::Spec->rel2abs( File::Spec->catdir( $FindBin::Bin, '..', '..' ) ); +my $install = "$repo/xCAT-server/share/xcat/install/ubuntu"; +my $template = "$install/compute.subiquity.tmpl"; +plan skip_all => 'the Ubuntu install directory is not here' unless -d $install && -f $template; + +sub packages_in { + my ($path) = @_; + return { map { $_ => 1 } grep { length } split /,/, xCAT::Postage::get_pkglist_tex($path) }; +} + +sub resolved { + my ( $os, $arch ) = @_; + return xCAT::SvrUtils->get_pkglist_file_name( $install, 'compute', $os, $arch, $os =~ /^(ubuntu\d+\.\d+)/ ? $1 : $os ); +} + +open( my $tfh, '<', $template ) or die "$template: $!"; +my $body = do { local $/; <$tfh> }; +close($tfh); +my ($block) = $body =~ /^ packages:\n((?: - .*\n)+)/m; +my %fixed = map { $_ => 1 } ( $block =~ /^ - ([^#\s]+)$/mg ); +ok( $fixed{chrony}, 'the Subiquity template names chrony among its fixed packages' ); + +my %time_daemon = map { $_ => 1 } qw(chrony ntp ntpsec); +foreach my $case ( [ 'ubuntu20.04.6', 'x86_64' ], [ 'ubuntu20.04.6', 'ppc64le' ], + [ 'ubuntu22.04.5', 'x86_64' ], [ 'ubuntu22.04.5', 'ppc64le' ], + [ 'ubuntu24.04.4', 'x86_64' ], [ 'ubuntu24.04.4', 'ppc64le' ], [ 'ubuntu24.04.4', 'riscv64' ], + [ 'ubuntu26.04.1', 'x86_64' ], [ 'ubuntu26.04.1', 'riscv64' ] ) { + my ( $os, $arch ) = @$case; + my $file = resolved( $os, $arch ); + ok( $file, "$os $arch resolves a compute pkglist" ) or next; + my $packages = packages_in($file); + ok( $packages->{chrony}, "$os $arch: the list names chrony, the daemon the template installs" ); + ok( !$packages->{ntp} && !$packages->{ntpdate}, "$os $arch: ... and not ntp, which would conflict with it" ); + my @daemons = grep { $time_daemon{$_} } keys %{ { %fixed, %$packages } }; + is( scalar(@daemons), 1, "$os $arch: the autoinstall transaction carries exactly one time daemon" ); +} + +# The releases before Subiquity keep the shared list and its ntp. +my $legacy = resolved( 'ubuntu16.04.7', 'x86_64' ); +is( $legacy, "$install/compute.pkglist", '16.04 still resolves the shared list' ); +ok( packages_in($legacy)->{ntp}, '... which keeps ntp for the releases where chrony was not the default' ); + +done_testing(); diff --git a/xCAT-test/unit/ubuntu_subiquity_template.t b/xCAT-test/unit/ubuntu_subiquity_template.t index 140dfaf39..ecff4688e 100644 --- a/xCAT-test/unit/ubuntu_subiquity_template.t +++ b/xCAT-test/unit/ubuntu_subiquity_template.t @@ -16,6 +16,9 @@ like($tmpl, qr/autoinstall:/, 'template has autoinstall: key'); like($tmpl, qr/version:\s*1/, 'template has version: 1'); like($tmpl, qr/^\s*identity:/m, 'template has an identity section so subiquity does not prompt'); +like($tmpl, qr/^ - #INCLUDE_DEFAULT_PKGLIST_AUTOINSTALL#\n/m, 'the packages list carries the osimage pkglist through the autoinstall token'); +like($tmpl, qr/^ - openssh-server\n/m, '... and keeps openssh-server, which xCAT needs on the node'); +like($tmpl, qr/^ - wget\n/m, '... and wget, which the early and late commands use'); like($tmpl, qr/kernel:/, 'template has kernel section'); like($tmpl, qr/package:\s*linux-generic/, 'template specifies linux-generic kernel'); like($tmpl, qr/#UBUNTU_SUBIQUITY_APT_CONFIG#/, 'template renders apt section from osimage context'); @@ -78,5 +81,39 @@ unlike($tmpl, qr/if \[ -x \/tmp\/pre\.sh \]/, 'pre.sh not checked with -x'); # Subiquity behavior can be handled without cloning this template per release. unlike($tmpl, qr/noble-|jammy-|focal-/, 'template avoids release-specific apt suite names'); like($tmpl, qr/#UBUNTU_SUBIQUITY_APT_CONFIG#/, 'template keeps dynamic apt renderer marker'); +like($tmpl, qr{2>&1'\n(?:\s*#[^\n]*\n)*\s*- rm -f (?:/target/etc/apt/sources\.list\.d/xcat-(?:otherpkgs|pkgdir)-\*\.(?:list|sources)\s+){4}/target/etc/apt/apt\.conf\.d/94curtin-config$}m, + 'the installer sources for the otherpkgs repository and the pkgdir mirrors, and the apt configuration curtin wrote, are removed from the target by a late-command of their own, after the post script block'); + +# the post script block: its status is the post script's, so a failed post script stops the install +{ + my ($block) = $tmpl =~ /\n - '(\{\n.*?\n \} >>\/target\/var\/log\/xcat\/xcat\.log 2>&1)'\n/s; + ok( defined $block, 'the post script block is found' ) or last; + $block =~ s/''/'/g; + require File::Temp; + my $root = File::Temp->newdir(); + mkdir "$root/bin" or die; + for my $tool ( 'curtin', 'wget' ) { + open( my $fh, '>', "$root/bin/$tool" ) or die; + print {$fh} $tool eq 'curtin' ? "#!/bin/sh\ncase \"\$*\" in *post.script*) exit 42;; esac\nexit 0\n" : "#!/bin/sh\nfor a; do case \"\$a\" in http*) touch \"\${a##*/}\";; esac; done\nexit 0\n"; + close $fh; chmod 0755, "$root/bin/$tool"; + } + ( my $script = $block ) =~ s{/target}{$root/target}g; + $script =~ s{/tmp/pre-install\.log}{$root/pre-install.log}g; + for my $token ( [ '#SUBIQUITYINSTALLNIC#', '' ], [ '#SUBIQUITYINSTALLMAC#', '52:54:00:00:00:01' ], [ '#HOSTNAME#', 'cn1' ], [ '#XCATVAR:XCATMASTER#', '192.0.2.10' ], + [ '#COLONHTTPPORT#', '' ], [ '#TABLEBLANKOKAY:bootparams:$NODE:kcmdline#', '' ] ) { + $script =~ s/\Q$token->[0]\E/$token->[1]/g; + } + require File::Path; + File::Path::make_path( map { "$root/target/$_" } qw(etc/default root var/log/xcat) ); + open( my $hosts, '>', "$root/target/etc/hosts" ) or die; print {$hosts} "127.0.0.1 localhost\n"; close $hosts; + open( my $pre, '>', "$root/pre-install.log" ) or die; close $pre; + my $cwd = File::Spec->rel2abs('.'); + chdir $root or die; + local $ENV{PATH} = "$root/bin:$ENV{PATH}"; + system( 'sh', '-c', $script ); + my $status = $? >> 8; + chdir $cwd or die; + is( $status, 42, 'a failing post script fails the late-command block, so Subiquity stops the install instead of switching the node to disk boot' ); +} done_testing();