From 69a00bf43f017e25a550e9b6d3e75ed8a9e896f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Sat, 22 Aug 2026 19:24:45 -0300 Subject: [PATCH] refactor(genesis): share release build helpers --- genesis-openembedded/build | 110 +++--------- genesis-openembedded/package | 100 +++-------- genesis-openembedded/verify-release | 36 +--- lib/XCAT/BuildUtils.pm | 160 ++++++++++++++++++ .../lib => lib}/XCAT/GenesisRelease.pm | 55 +----- mockbuild-all.pl | 151 ++++++----------- 6 files changed, 282 insertions(+), 330 deletions(-) create mode 100644 lib/XCAT/BuildUtils.pm rename {genesis-openembedded/lib => lib}/XCAT/GenesisRelease.pm (81%) diff --git a/genesis-openembedded/build b/genesis-openembedded/build index 0121ad0..41fa3ba 100755 --- a/genesis-openembedded/build +++ b/genesis-openembedded/build @@ -4,17 +4,22 @@ use strict; use warnings; use Cwd qw(abs_path); -use Digest::SHA (); use File::Basename qw(basename dirname); use File::Copy qw(copy); -use File::Find qw(find); use File::Path qw(make_path); -use File::Spec; use File::Temp qw(tempdir); use FindBin; use Getopt::Long qw(GetOptions); use POSIX qw(strftime); -use lib "$FindBin::Bin/lib"; +use lib "$FindBin::Bin/../lib"; +use XCAT::BuildUtils qw( + capture_command + digest_manifest + read_first_line + relative_files + run_command + write_binary +); use XCAT::GenesisRelease qw( architectures deb_package_name @@ -60,20 +65,24 @@ for my $path (qw(Version xCAT-genesis-builder/oe/build xCAT-genesis-builder/oe/e die "xcat-core source is missing $path\n" unless -f "$xcat_source/$path"; } die "xcat-core checkout is not clean\n" - if _capture('git', '-C', $xcat_source, 'status', '--porcelain') ne ''; + if capture_command('git', '-C', $xcat_source, 'status', '--porcelain') ne ''; -my $revision = _capture('git', '-C', $xcat_source, 'rev-parse', 'HEAD'); +my $revision = capture_command('git', '-C', $xcat_source, 'rev-parse', 'HEAD'); die "Invalid xcat-core revision: $revision\n" unless $revision =~ /^[0-9a-f]{40}$/; if ($xcat_ref ne '') { die "Invalid xcat-core ref: $xcat_ref\n" unless $xcat_ref =~ /^[A-Za-z0-9][A-Za-z0-9._\/-]*$/; - my $expected = _capture('git', '-C', $xcat_source, 'rev-parse', '--verify', "$xcat_ref^{commit}"); + my $expected = capture_command( + 'git', '-C', $xcat_source, 'rev-parse', '--verify', "$xcat_ref^{commit}", + ); die "xcat-core HEAD $revision does not match $xcat_ref ($expected)\n" unless $revision eq $expected; } -my $version = _read_line("$xcat_source/Version"); +my $version = read_first_line("$xcat_source/Version"); die "Invalid xCAT version: $version\n" unless $version =~ /^\d+(?:\.\d+){1,3}$/; -my $source_date_epoch = _capture('git', '-C', $xcat_source, 'show', '-s', '--format=%ct', 'HEAD'); +my $source_date_epoch = capture_command( + 'git', '-C', $xcat_source, 'show', '-s', '--format=%ct', 'HEAD', +); die "Invalid xcat-core commit time\n" unless $source_date_epoch =~ /^\d+$/; my $release = strftime('snap%Y%m%d%H%M', gmtime($source_date_epoch)); @@ -101,7 +110,7 @@ local $ENV{XCAT_GENESIS_WORK_DIR} = $oe_work; { local $ENV{TMPDIR} = $oe_tmp; make_path($oe_tmp); - _run("$xcat_source/xCAT-genesis-builder/oe/build", @requested_architectures); + run_command("$xcat_source/xCAT-genesis-builder/oe/build", @requested_architectures); } my $effective_deploy = "$oe_tmp/deploy"; die "Invalid OpenEmbedded deploy directory: $effective_deploy\n" @@ -110,13 +119,13 @@ die "Invalid OpenEmbedded deploy directory: $effective_deploy\n" for my $architecture (@requested_architectures) { my $export = "$work/exports/$architecture"; make_path(dirname($export)); - _run( + run_command( "$xcat_source/xCAT-genesis-builder/oe/export", $architecture, $effective_deploy, $export, ); my $packages = "$work/packages/$architecture"; - _run( + run_command( "$FindBin::Bin/package", '--architecture', $architecture, '--export-dir', $export, @@ -131,7 +140,7 @@ for my $architecture (@requested_architectures) { } my @formats = $format eq 'all' ? qw(deb rpm) : ($format); -_write_file( +write_binary( "$staging/release.manifest", "format=xcat-genesis-packages\n" . "version=1\n" @@ -142,11 +151,15 @@ _write_file( . "architectures=" . join(',', @requested_architectures) . "\n" . "formats=" . join(',', @formats) . "\n", ); -_write_checksums($staging); +my @release_files = grep { $_ ne 'SHA256SUMS' } relative_files($staging); +write_binary( + "$staging/SHA256SUMS", + digest_manifest($staging, 'sha256', @release_files), +); validate_release($staging); my @verify_args = ('--format', $format); push(@verify_args, '--complete') if $all; -_run("$FindBin::Bin/verify-release", @verify_args, $staging); +run_command("$FindBin::Bin/verify-release", @verify_args, $staging); chmod(0755, $staging) or die "Cannot make release directory readable: $!\n"; rename($staging, $output_dir) or die "Cannot publish $output_dir: $!\n"; @@ -183,73 +196,6 @@ sub _collect_one { copy($source, $destination) or die "Cannot collect $source: $!\n"; } -sub _write_checksums { - my ($directory) = @_; - my @files; - find( - { - no_chdir => 1, - wanted => sub { - return unless -f $_ && !-l $_; - my $relative = File::Spec->abs2rel($File::Find::name, $directory); - $relative =~ tr{\\}{/}; - push(@files, $relative) unless $relative eq 'SHA256SUMS'; - }, - }, - $directory, - ); - my $content = ''; - for my $relative (sort @files) { - open(my $fh, '<:raw', "$directory/$relative") - or die "Cannot read $directory/$relative: $!\n"; - my $digest = Digest::SHA->new(256)->addfile($fh)->hexdigest; - close($fh) or die "Cannot close $directory/$relative: $!\n"; - $content .= "$digest $relative\n"; - } - _write_file("$directory/SHA256SUMS", $content); -} - -sub _read_line { - my ($path) = @_; - open(my $fh, '<:raw', $path) or die "Cannot read $path: $!\n"; - my $line = <$fh>; - close($fh) or die "Cannot close $path: $!\n"; - die "Empty file: $path\n" unless defined($line); - chomp($line); - $line =~ s/\r\z//; - return $line; -} - -sub _capture { - my (@command) = @_; - open(my $fh, '-|', @command) or die "Cannot run $command[0]: $!\n"; - local $/; - my $output = <$fh> // ''; - close($fh) or die "Command failed: $command[0]\n"; - $output =~ s/\s+\z//; - return $output; -} - -sub _run { - my (@command) = @_; - print '+ ', join(' ', map { _display_quote($_) } @command), "\n"; - system(@command) == 0 or die "Command failed: $command[0]\n"; -} - -sub _display_quote { - my ($value) = @_; - return $value if $value =~ /^[A-Za-z0-9_.,+\/:=@~-]+$/; - $value =~ s/'/'\\''/g; - return "'$value'"; -} - -sub _write_file { - my ($path, $content) = @_; - open(my $fh, '>:raw', $path) or die "Cannot write $path: $!\n"; - print {$fh} $content or die "Cannot write $path: $!\n"; - close($fh) or die "Cannot close $path: $!\n"; -} - sub usage { return <<'USAGE'; Usage: build [--xcat-source DIR] [--xcat-ref REF] [--output-dir DIR] diff --git a/genesis-openembedded/package b/genesis-openembedded/package index 977cee8..683ed25 100755 --- a/genesis-openembedded/package +++ b/genesis-openembedded/package @@ -4,15 +4,21 @@ use strict; use warnings; use Cwd qw(abs_path); -use Digest::MD5 (); use File::Basename qw(basename dirname); use File::Copy qw(copy); use File::Path qw(make_path remove_tree); -use File::Spec; use File::Temp qw(tempdir); use FindBin; use Getopt::Long qw(GetOptions); -use lib "$FindBin::Bin/lib"; +use lib "$FindBin::Bin/../lib"; +use XCAT::BuildUtils qw( + capture_command + digest_manifest + relative_files + require_command + run_command + write_binary +); use XCAT::GenesisRelease qw( deb_package_name rpm_package_name @@ -84,16 +90,16 @@ for my $entry (_flat_files($export_dir)) { copy("$export_dir/$entry", "$source_root/image/$entry") or die "Cannot copy export file $entry: $!\n"; } -_write_file("$source_root/xcat-core-revision", "$revision\n"); +write_binary("$source_root/xcat-core-revision", "$revision\n"); local $ENV{SOURCE_DATE_EPOCH} = $source_date_epoch; if ($format eq 'all' || $format eq 'rpm') { - _require_command('rpmbuild'); + require_command('rpmbuild'); _build_rpm($work, $staging, $source_root, $source_name, $rpm_name); } if ($format eq 'all' || $format eq 'deb') { - _require_command('dpkg-deb'); + require_command('dpkg-deb'); _build_deb($work, $staging, $source_root, $deb_name); } @@ -111,7 +117,7 @@ sub _build_rpm { make_path("$staging/rpm", "$staging/srpm"); my $archive = "$rpm_top/SOURCES/$source_name.tar.gz"; - _run( + run_command( 'tar', '--sort=name', '--owner=0', '--group=0', '--numeric-owner', "--mtime=\@$source_date_epoch", '--use-compress-program=gzip -n', '-cf', $archive, '-C', dirname($source_root), basename($source_root), @@ -119,7 +125,7 @@ sub _build_rpm { my $spec = "$FindBin::Bin/rpm/xCAT-genesis-openembedded.spec"; copy($spec, "$rpm_top/SPECS/xCAT-genesis-openembedded.spec") or die "Cannot stage RPM spec: $!\n"; - _run( + run_command( 'rpmbuild', '-ba', '--define', "_topdir $rpm_top", '--define', "genesis_arch $architecture", @@ -162,7 +168,7 @@ sub _build_deb { my $installed_kib = _tree_bytes($root); $installed_kib = int(($installed_kib + 1023) / 1024); - _write_file( + write_binary( "$root/DEBIAN/control", "Package: $package_name\n" . "Version: $version-$release\n" @@ -177,36 +183,17 @@ sub _build_deb { _normalize_mtime($root, $source_date_epoch); my $deb = "$staging/deb/${package_name}_${version}-${release}_all.deb"; - _run('dpkg-deb', '--root-owner-group', '-Zgzip', '-z9', '--build', $root, $deb); + run_command('dpkg-deb', '--root-owner-group', '-Zgzip', '-z9', '--build', $root, $deb); die "DEB build did not produce $deb\n" unless -f $deb; } sub _write_deb_md5sums { my ($root) = @_; - my @files; - require File::Find; - File::Find::find( - { - no_chdir => 1, - wanted => sub { - return unless -f $_ && !-l $_; - return if $File::Find::name =~ m{\Q$root/DEBIAN/\E}; - my $relative = File::Spec->abs2rel($File::Find::name, $root); - $relative =~ tr{\\}{/}; - push(@files, $relative); - }, - }, - $root, + my @files = grep { $_ !~ m{\ADEBIAN/} } relative_files($root); + write_binary( + "$root/DEBIAN/md5sums", + digest_manifest($root, 'md5', @files), ); - my $content = ''; - for my $relative (sort @files) { - open(my $fh, '<:raw', "$root/$relative") - or die "Cannot read $root/$relative: $!\n"; - my $digest = Digest::MD5->new->addfile($fh)->hexdigest; - close($fh) or die "Cannot close $root/$relative: $!\n"; - $content .= "$digest $relative\n"; - } - _write_file("$root/DEBIAN/md5sums", $content); } sub _normalize_mtime { @@ -226,12 +213,9 @@ sub _normalize_mtime { sub _flat_files { my ($directory) = @_; - opendir(my $dh, $directory) or die "Cannot read $directory: $!\n"; - my @entries = sort grep { $_ ne '.' && $_ ne '..' } readdir($dh); - closedir($dh) or die "Cannot close $directory: $!\n"; + my @entries = relative_files($directory); for my $entry (@entries) { - my $path = "$directory/$entry"; - die "Expected a regular file: $path\n" unless -f $path && !-l $path; + die "Expected a flat directory: $directory\n" if $entry =~ m{/}; } return @entries; } @@ -239,51 +223,17 @@ sub _flat_files { sub _tree_bytes { my ($directory) = @_; my $bytes = 0; - require File::Find; - File::Find::find( - { no_chdir => 1, wanted => sub { $bytes += -s $_ if -f $_ && !-l $_ } }, - $directory, - ); + $bytes += -s "$directory/$_" for relative_files($directory); return $bytes; } -sub _require_command { - my ($command) = @_; - for my $directory (File::Spec->path()) { - return if -x "$directory/$command"; - } - die "Required command not found: $command\n"; -} - sub _require_gnu_tar { - _require_command('tar'); - open(my $fh, '-|', 'tar', '--version') or die "Cannot run tar: $!\n"; - my $version_line = <$fh> // ''; - close($fh) or die "Cannot query tar version\n"; + require_command('tar'); + my $version_line = capture_command('tar', '--version'); die "GNU tar is required to build Genesis source packages\n" unless $version_line =~ /^tar \(GNU tar\)/; } -sub _run { - my (@command) = @_; - print '+ ', join(' ', map { _display_quote($_) } @command), "\n"; - system(@command) == 0 or die "Command failed: $command[0]\n"; -} - -sub _display_quote { - my ($value) = @_; - return $value if $value =~ /^[A-Za-z0-9_.,+\/:=@~-]+$/; - $value =~ s/'/'\\''/g; - return "'$value'"; -} - -sub _write_file { - my ($path, $content) = @_; - open(my $fh, '>:raw', $path) or die "Cannot write $path: $!\n"; - print {$fh} $content or die "Cannot write $path: $!\n"; - close($fh) or die "Cannot close $path: $!\n"; -} - sub usage { return <<'USAGE'; Usage: package --architecture ARCH --export-dir DIR --output-dir DIR diff --git a/genesis-openembedded/verify-release b/genesis-openembedded/verify-release index 623a748..da7563a 100755 --- a/genesis-openembedded/verify-release +++ b/genesis-openembedded/verify-release @@ -4,10 +4,10 @@ use strict; use warnings; use Cwd qw(abs_path); -use File::Spec; use FindBin; use Getopt::Long qw(GetOptions); -use lib "$FindBin::Bin/lib"; +use lib "$FindBin::Bin/../lib"; +use XCAT::BuildUtils qw(capture_command require_command); use XCAT::GenesisRelease qw( deb_package_name rpm_package_name @@ -49,7 +49,7 @@ print "Verified Genesis package release: $directory\n"; sub _verify_rpm { my ($directory, $manifest, $architecture) = @_; - _require_command('rpm'); + require_command('rpm'); my $name = rpm_package_name($architecture); my $source_name = "$name-$manifest->{xcat_version}-$manifest->{xcat_release}.src.rpm"; my @packages = ( @@ -59,7 +59,7 @@ sub _verify_rpm { ); for my $package (@packages) { my ($path, $expected_arch, $expected_source) = @{$package}; - my $metadata = _capture( + my $metadata = capture_command( 'rpm', '-qp', '--qf', "%{NAME}\t%{VERSION}\t%{RELEASE}\t%{ARCH}\t%{EPOCHNUM}\t%{SOURCERPM}" . "\t%{BUILDHOST}\t%{BUILDTIME}\n", @@ -72,10 +72,10 @@ sub _verify_rpm { ); die "Unexpected RPM identity for $path: $metadata\n" unless $metadata eq $expected; for my $relationship (qw(conflicts obsoletes)) { - my $value = _capture('rpm', '-qp', "--$relationship", $path); + my $value = capture_command('rpm', '-qp', "--$relationship", $path); die "Unexpected RPM $relationship for $path: $value\n" if $value ne ''; } - my $provides = _capture('rpm', '-qp', '--provides', $path); + my $provides = capture_command('rpm', '-qp', '--provides', $path); die "Legacy RPM relationship in $path: $provides\n" if $provides =~ /xCAT-genesis-base/; } @@ -83,12 +83,12 @@ sub _verify_rpm { sub _verify_deb { my ($directory, $manifest, $architecture) = @_; - _require_command('dpkg-deb'); + require_command('dpkg-deb'); my $name = deb_package_name($architecture); my $path = "$directory/deb/${name}_$manifest->{xcat_version}-$manifest->{xcat_release}_all.deb"; my $metadata = join( "\n", - map { _capture('dpkg-deb', '-f', $path, $_) } + map { capture_command('dpkg-deb', '-f', $path, $_) } qw(Package Version Architecture), ); my $expected = join( @@ -97,29 +97,11 @@ sub _verify_deb { ); die "Unexpected DEB identity for $path: $metadata\n" unless $metadata eq $expected; for my $field (qw(Breaks Conflicts Provides Replaces)) { - my $value = _capture('dpkg-deb', '-f', $path, $field); + my $value = capture_command('dpkg-deb', '-f', $path, $field); die "Unexpected DEB $field for $path: $value\n" if $value ne ''; } } -sub _require_command { - my ($command) = @_; - for my $directory (File::Spec->path()) { - return if -x "$directory/$command"; - } - die "Required command not found: $command\n"; -} - -sub _capture { - my (@command) = @_; - open(my $fh, '-|', @command) or die "Cannot run $command[0]: $!\n"; - local $/; - my $output = <$fh> // ''; - close($fh) or die "Command failed: $command[0]\n"; - $output =~ s/\s+\z//; - return $output; -} - sub usage { return "Usage: verify-release [--format all|rpm|deb] [--complete] RELEASE_DIRECTORY\n"; } diff --git a/lib/XCAT/BuildUtils.pm b/lib/XCAT/BuildUtils.pm new file mode 100644 index 0000000..c8fa7b0 --- /dev/null +++ b/lib/XCAT/BuildUtils.pm @@ -0,0 +1,160 @@ +package XCAT::BuildUtils; + +use strict; +use warnings; + +use Digest::MD5 (); +use Digest::SHA (); +use Exporter qw(import); +use File::Find qw(find); +use File::Slurper qw(read_binary write_binary); +use File::Spec; +use IPC::Cmd qw(can_run); + +our @EXPORT_OK = qw( + capture_command + command_exists + digest_file + digest_manifest + display_quote + hashes_equal + print_step + read_binary + read_first_line + read_lines + relative_files + require_command + run_command + shell_quote + write_binary +); + +sub command_exists { + my ($command) = @_; + return defined(can_run($command)); +} + +sub require_command { + my ($command) = @_; + return can_run($command) + // die "Required command not found: $command\n"; +} + +sub capture_command { + my (@command) = @_; + open(my $fh, '-|', @command) or die "Cannot run $command[0]: $!\n"; + local $/; + my $output = <$fh> // ''; + close($fh) or die "Command failed: $command[0]\n"; + $output =~ s/\s+\z//; + return $output; +} + +sub run_command { + my (@command) = @_; + print '+ ', join(' ', map { display_quote($_) } @command), "\n"; + my $status = system(@command); + return 1 if $status == 0; + + my $exit = $status == -1 + ? 255 + : ($status & 127) ? 128 + ($status & 127) : $status >> 8; + die "Command failed (rc=$exit): " + . join(' ', map { display_quote($_) } @command) . "\n"; +} + +sub display_quote { + my ($value) = @_; + return $value if $value =~ /^[A-Za-z0-9_.,+\/:=@~-]+$/; + return shell_quote($value); +} + +sub shell_quote { + my ($value) = @_; + $value = '' unless defined($value); + $value =~ s/'/'"'"'/g; + return "'$value'"; +} + +sub print_step { + my ($message) = @_; + print "\n== $message ==\n"; +} + +sub read_lines { + my ($path) = @_; + my $content = read_binary($path); + return () if $content eq ''; + + my @lines = split(/\n/, $content, -1); + pop(@lines) if @lines && $lines[-1] eq ''; + s/\r\z// for @lines; + return @lines; +} + +sub read_first_line { + my ($path) = @_; + my @lines = read_lines($path); + die "Empty file: $path\n" unless @lines; + return $lines[0]; +} + +sub relative_files { + my ($root) = @_; + die "Invalid directory: $root\n" unless -d $root && !-l $root; + + my $absolute = File::Spec->rel2abs($root); + my @files; + find( + { + no_chdir => 1, + wanted => sub { + my $path = $File::Find::name; + return if $path eq $absolute; + die "Symbolic links are not allowed: $path\n" if -l $path; + return if -d $path; + die "Non-regular entry: $path\n" unless -f $path; + my $relative = File::Spec->abs2rel($path, $absolute); + $relative =~ tr{\\}{/}; + push(@files, $relative); + }, + }, + $absolute, + ); + my @sorted = sort @files; + return @sorted; +} + +sub digest_file { + my ($path, $algorithm) = @_; + $algorithm //= 'sha256'; + + my $digest = $algorithm eq 'sha256' ? Digest::SHA->new(256) + : $algorithm eq 'md5' ? Digest::MD5->new + : die "Unsupported digest algorithm: $algorithm\n"; + open(my $fh, '<:raw', $path) or die "Cannot read $path: $!\n"; + my $value = $digest->addfile($fh)->hexdigest; + close($fh) or die "Cannot close $path: $!\n"; + return $value; +} + +sub digest_manifest { + my ($root, $algorithm, @files) = @_; + return join( + '', + map { digest_file("$root/$_", $algorithm) . " $_\n" } + sort @files, + ); +} + +sub hashes_equal { + my ($left, $right) = @_; + return 0 unless keys(%{$left}) == keys(%{$right}); + for my $name (keys %{$left}) { + return 0 unless exists($right->{$name}) + && $left->{$name} eq $right->{$name}; + } + return 1; +} + +1; diff --git a/genesis-openembedded/lib/XCAT/GenesisRelease.pm b/lib/XCAT/GenesisRelease.pm similarity index 81% rename from genesis-openembedded/lib/XCAT/GenesisRelease.pm rename to lib/XCAT/GenesisRelease.pm index f10b2ee..0bcc9ae 100644 --- a/genesis-openembedded/lib/XCAT/GenesisRelease.pm +++ b/lib/XCAT/GenesisRelease.pm @@ -3,10 +3,8 @@ package XCAT::GenesisRelease; use strict; use warnings; -use Digest::SHA (); use Exporter qw(import); -use File::Find qw(find); -use File::Spec; +use XCAT::BuildUtils qw(digest_file read_lines relative_files); our @EXPORT_OK = qw( architectures @@ -50,11 +48,8 @@ sub deb_package_name { sub _read_key_values { my ($path, $allowed) = @_; - open(my $fh, '<:raw', $path) or die "Cannot read $path: $!\n"; my %values; - while (my $line = <$fh>) { - chomp($line); - $line =~ s/\r\z//; + for my $line (read_lines($path)) { die "Invalid manifest entry in $path: $line\n" unless $line =~ /^([a-z][a-z0-9_]*)=([A-Za-z0-9][A-Za-z0-9.,_+~-]*)$/; my ($key, $value) = ($1, $2); @@ -62,43 +57,14 @@ sub _read_key_values { die "Duplicate manifest key in $path: $key\n" if exists($values{$key}); $values{$key} = $value; } - close($fh) or die "Cannot close $path: $!\n"; return \%values; } -sub _regular_files { - my ($root) = @_; - die "Invalid directory: $root\n" unless -d $root && !-l $root; - - my @files; - find( - { - no_chdir => 1, - wanted => sub { - my $path = $File::Find::name; - return if $path eq $root; - die "Symbolic links are not allowed: $path\n" if -l $path; - return if -d $path; - die "Non-regular release entry: $path\n" unless -f $path; - my $relative = File::Spec->abs2rel($path, $root); - $relative =~ tr{\\}{/}; - push(@files, $relative); - }, - }, - $root, - ); - my @sorted = sort @files; - return @sorted; -} - sub _read_checksums { my ($root) = @_; my $path = "$root/SHA256SUMS"; - open(my $fh, '<:raw', $path) or die "Cannot read $path: $!\n"; my %checksums; - while (my $line = <$fh>) { - chomp($line); - $line =~ s/\r\z//; + for my $line (read_lines($path)) { die "Invalid checksum entry in $path: $line\n" unless $line =~ /^([0-9a-f]{64}) ([A-Za-z0-9][A-Za-z0-9._\/+~-]*)$/; my ($digest, $name) = ($1, $2); @@ -108,21 +74,18 @@ sub _read_checksums { if exists($checksums{$name}); $checksums{$name} = $digest; } - close($fh) or die "Cannot close $path: $!\n"; return \%checksums; } sub _verify_checksums { my ($root) = @_; - my @files = grep { $_ ne 'SHA256SUMS' } _regular_files($root); + my @files = grep { $_ ne 'SHA256SUMS' } relative_files($root); my $checksums = _read_checksums($root); my %files = map { $_ => 1 } @files; for my $name (@files) { die "Missing checksum for $name\n" unless exists($checksums->{$name}); - open(my $fh, '<:raw', "$root/$name") or die "Cannot read $root/$name: $!\n"; - my $digest = Digest::SHA->new(256)->addfile($fh)->hexdigest; - close($fh) or die "Cannot close $root/$name: $!\n"; + my $digest = digest_file("$root/$name", 'sha256'); die "Checksum mismatch for $name\n" unless $digest eq $checksums->{$name}; } for my $name (keys %{$checksums}) { @@ -148,7 +111,7 @@ sub validate_export { ); $required{'fw_jump.elf'} = 1 if $architecture eq 'riscv64'; - my @files = _regular_files($directory); + my @files = relative_files($directory); my %files = map { $_ => 1 } @files; for my $name (sort keys %required) { die "Genesis export is missing $name\n" unless $files{$name}; @@ -240,7 +203,7 @@ sub validate_release { $expected{"deb/${deb}_$manifest->{xcat_version}-$manifest->{xcat_release}_all.deb"} = 1; } } - for my $file (grep { $_ ne 'SHA256SUMS' } _regular_files($directory)) { + for my $file (grep { $_ ne 'SHA256SUMS' } relative_files($directory)) { die "Unexpected Genesis release artifact: $file\n" unless $expected{$file}; delete($expected{$file}); } @@ -269,9 +232,7 @@ sub verify_release_file { die "Missing verified checksum for $relative\n" unless ref($checksums) eq 'HASH' && exists($checksums->{$relative}); die "Invalid collected release file: $path\n" unless -f $path && !-l $path; - open(my $fh, '<:raw', $path) or die "Cannot read $path: $!\n"; - my $digest = Digest::SHA->new(256)->addfile($fh)->hexdigest; - close($fh) or die "Cannot close $path: $!\n"; + my $digest = digest_file($path, 'sha256'); die "Collected release file checksum mismatch: $path\n" unless $digest eq $checksums->{$relative}; return 1; diff --git a/mockbuild-all.pl b/mockbuild-all.pl index 9de89af..8711969 100755 --- a/mockbuild-all.pl +++ b/mockbuild-all.pl @@ -13,7 +13,16 @@ use FindBin; use Getopt::Long qw(GetOptions); use Parallel::ForkManager; use POSIX qw(strftime); -use lib "$FindBin::Bin/genesis-openembedded/lib"; +use lib "$FindBin::Bin/lib"; +use XCAT::BuildUtils qw( + capture_command + hashes_equal + print_step + read_first_line + require_command + run_command + shell_quote +); use XCAT::GenesisRelease qw( validated_release_checksums verify_release_file @@ -96,7 +105,7 @@ $repo_root = abs_path($repo_root); my $SOURCE_DATE_EPOCH; $SOURCE_DATE_EPOCH = $build_timestamp if defined $build_timestamp; if (!$SOURCE_DATE_EPOCH && -f "$repo_root/Gitepoch") { - $SOURCE_DATE_EPOCH = slurp_chomp("$repo_root/Gitepoch"); + $SOURCE_DATE_EPOCH = read_first_line("$repo_root/Gitepoch"); } unless ($SOURCE_DATE_EPOCH && $SOURCE_DATE_EPOCH =~ /^\d+$/) { $SOURCE_DATE_EPOCH = `git -C \Q$repo_root\E log -1 --format=%ct HEAD 2>/dev/null`; @@ -128,7 +137,7 @@ acquire_output_lock($output_base, $force_unlock); $xcat_src = resolve_xcat_source($xcat_src, $repo_root); -my $arch = capture('uname -m'); +my $arch = capture_command('uname', '-m'); my %os = read_os_release('/etc/os-release'); my $os_id = $os{ID} // ''; my $version_id = $os{VERSION_ID} // ''; @@ -153,10 +162,10 @@ if ($genesis_release ne '') { my $verifier = "$script_dir/genesis-openembedded/verify-release"; die "Genesis release verifier not found: $verifier\n" unless -x $verifier; my $checksums_before = validated_release_checksums($genesis_release); - run_argv($^X, $verifier, '--complete', '--format', 'rpm', $genesis_release); + run_command($^X, $verifier, '--complete', '--format', 'rpm', $genesis_release); my $checksums_after = validated_release_checksums($genesis_release); die "Genesis release changed during verification\n" - unless checksum_sets_match($checksums_before, $checksums_after); + unless hashes_equal($checksums_before, $checksums_after); $genesis_release_checksums = $checksums_before; } @@ -190,7 +199,7 @@ $tgt_workers = scalar(@build_targets) if $tgt_workers > scalar(@build_targets); # mock build already gets a unique --uniqueext (separate chroot), so the only limit needed is # hardware: total concurrent builds across all targets stays <= $cap (default host nproc). The # per-target build-step concurrency is therefore the cap divided across the active targets. -my $cap = $max_parallel > 0 ? $max_parallel : (capture('nproc') || 4); +my $cap = $max_parallel > 0 ? $max_parallel : (capture_command('nproc') || 4); my $per_target_builds = defined($parallel_builds) ? $parallel_builds : int($cap / $tgt_workers); $per_target_builds = 1 if $per_target_builds < 1; print "parallel_targets: " . ($parallel_targets > 0 ? $parallel_targets : "auto($tgt_workers)") . "\n"; @@ -304,7 +313,7 @@ my @collect_roots; if ($scrub_all_chroots) { run_step( step => "Scrub all chroots for target $target", - cmd => "mock -r " . sh_quote($target) . " --scrub=all", + cmd => "mock -r " . shell_quote($target) . " --scrub=all", log => "$log_root/scrub-all-chroots.log", ); } @@ -321,13 +330,13 @@ if (!$skip_build) { my $step_log = "$log_root/$name"; my $step_uniqueext = build_mock_uniqueext($run_id, ++$build_step_seq, $name); my $cmd = join(' ', - 'perl', sh_quote($script), - '--mock-cfg', sh_quote($target), - '--mock-uniqueext', sh_quote($step_uniqueext), - '--result-dir', sh_quote($step_result), - '--log-dir', sh_quote($step_log), + 'perl', shell_quote($script), + '--mock-cfg', shell_quote($target), + '--mock-uniqueext', shell_quote($step_uniqueext), + '--result-dir', shell_quote($step_result), + '--log-dir', shell_quote($step_log), # host-local, run-scoped work dir so /tmp doesn't collide between runs - '--work-dir', sh_quote("/tmp/mockbuild-all-$run_id/$name"), + '--work-dir', shell_quote("/tmp/mockbuild-all-$run_id/$name"), '--build-timestamp', $SOURCE_DATE_EPOCH, ($skip_install ? '--skip-install' : ()), ); @@ -349,12 +358,12 @@ if (!$skip_build) { # forks one mock build per perl package (~7), which -- multiplied by parallel EL targets -- # oversubscribes the host. my $cmd = join(' ', - 'perl', sh_quote($perl_builder), - '--mock-cfg', sh_quote($target), - '--mock-uniqueext', sh_quote($perl_uniqueext), - '--result-dir', sh_quote($perl_result), - '--log-dir', sh_quote($perl_log), - '--work-dir', sh_quote("/tmp/mockbuild-all-$run_id/perl-list6"), + 'perl', shell_quote($perl_builder), + '--mock-cfg', shell_quote($target), + '--mock-uniqueext', shell_quote($perl_uniqueext), + '--result-dir', shell_quote($perl_result), + '--log-dir', shell_quote($perl_log), + '--work-dir', shell_quote("/tmp/mockbuild-all-$run_id/perl-list6"), (($max_build_workers && $max_build_workers >= 1) ? ('--jobs', $max_build_workers) : ()), '--build-timestamp', $SOURCE_DATE_EPOCH, ($skip_install ? '--skip-install' : ()), @@ -371,14 +380,14 @@ if (!$skip_build) { if (!$skip_xcat) { # Own HOME per target (buildrpms.pl uses $HOME/rpmbuild) so parallel targets don't race. my $xcat_home = "/tmp/mockbuild-all-$run_id/xcat-home"; - my $mktree = join(' ', map { sh_quote("$xcat_home/rpmbuild/$_") } qw(SOURCES SPECS BUILD BUILDROOT RPMS SRPMS)); - my $cmd = "mkdir -p $mktree && HOME=" . sh_quote($xcat_home) . ' ' . join(' ', - 'perl', sh_quote("$xcat_src/buildrpms.pl"), - '--target', sh_quote($target), + my $mktree = join(' ', map { shell_quote("$xcat_home/rpmbuild/$_") } qw(SOURCES SPECS BUILD BUILDROOT RPMS SRPMS)); + my $cmd = "mkdir -p $mktree && HOME=" . shell_quote($xcat_home) . ' ' . join(' ', + 'perl', shell_quote("$xcat_src/buildrpms.pl"), + '--target', shell_quote($target), '--nproc', int($nproc), '--force', '--verbose', - '--xcat_dep_path', sh_quote($repo_root), + '--xcat_dep_path', shell_quote($repo_root), ); push @build_steps, { id => 'xcat', @@ -402,15 +411,15 @@ if (!$skip_build) { my $genesis_home = "/tmp/mockbuild-all-$run_id/genesis-home"; # buildrpms.pl's rpmdev-setuptree only runs during env setup, not per build, so create the # rpmbuild tree ourselves for this per-target HOME (else $HOME/rpmbuild/SOURCES is missing). - my $mktree = join(' ', map { sh_quote("$genesis_home/rpmbuild/$_") } qw(SOURCES SPECS BUILD BUILDROOT RPMS SRPMS)); - my $cmd = "mkdir -p $mktree && HOME=" . sh_quote($genesis_home) . ' ' . join(' ', - 'perl', sh_quote("$xcat_src/buildrpms.pl"), + my $mktree = join(' ', map { shell_quote("$genesis_home/rpmbuild/$_") } qw(SOURCES SPECS BUILD BUILDROOT RPMS SRPMS)); + my $cmd = "mkdir -p $mktree && HOME=" . shell_quote($genesis_home) . ' ' . join(' ', + 'perl', shell_quote("$xcat_src/buildrpms.pl"), '--package', 'xCAT-genesis-base', - '--target', sh_quote($target), + '--target', shell_quote($target), '--nproc', int($nproc), '--force', '--verbose', - '--xcat_dep_path', sh_quote($repo_root), + '--xcat_dep_path', shell_quote($repo_root), ); push @build_steps, { id => 'genesis', @@ -537,8 +546,8 @@ if (!$skip_tarball) { my $cmd = join(' ', 'tar', '--sort=name', '--owner=0', '--group=0', "--mtime=\@$SOURCE_DATE_EPOCH", - '-C', sh_quote($run_root), - '-czf', sh_quote($tarball), + '-C', shell_quote($run_root), + '-czf', shell_quote($tarball), 'repo' ); run_step( @@ -549,8 +558,8 @@ if (!$skip_tarball) { my $srpm_cmd = join(' ', 'tar', '--sort=name', '--owner=0', '--group=0', "--mtime=\@$SOURCE_DATE_EPOCH", - '-C', sh_quote($run_root), - '-czf', sh_quote($srpm_tarball), + '-C', shell_quote($run_root), + '-czf', shell_quote($srpm_tarball), 'repo-src' ); run_step( @@ -637,8 +646,8 @@ sub deploy_target { sub createrepo_c_cmd { my ($dir) = @_; return 'createrepo_c --update --database ' - . '--revision ' . sh_quote($SOURCE_DATE_EPOCH) . ' --set-timestamp-to-revision ' - . sh_quote($dir); + . '--revision ' . shell_quote($SOURCE_DATE_EPOCH) . ' --set-timestamp-to-revision ' + . shell_quote($dir); } sub sign_and_index_repo { @@ -647,15 +656,15 @@ sub sign_and_index_repo { if ($gpg_sign && @rpms) { local $ENV{GNUPGHOME} = $gpg_home if $gpg_home; run_simple(qq(rpmsign --define "%_gpg_name $gpg_key_name" --addsign ) - . join(' ', map { sh_quote($_) } @rpms)); + . join(' ', map { shell_quote($_) } @rpms)); } run_simple(createrepo_c_cmd($dir)); if ($gpg_sign) { local $ENV{GNUPGHOME} = $gpg_home if $gpg_home; my $repomd = "$dir/repodata/repomd.xml"; unlink "$repomd.asc" if -f "$repomd.asc"; - run_simple(qq(gpg -a --detach-sign --default-key "$gpg_key_name" ) . sh_quote($repomd)); - run_simple(qq(gpg -a --export "$gpg_key_name" > ) . sh_quote("$repomd.key")); + run_simple(qq(gpg -a --detach-sign --default-key "$gpg_key_name" ) . shell_quote($repomd)); + run_simple(qq(gpg -a --export "$gpg_key_name" > ) . shell_quote("$repomd.key")); } } @@ -769,16 +778,6 @@ Notes: USAGE } -sub print_step { - my ($msg) = @_; - print "\n== $msg ==\n"; -} - -sub require_command { - my ($cmd) = @_; - run_simple("command -v " . sh_quote($cmd) . " >/dev/null 2>&1"); -} - sub run_simple { my ($cmd) = @_; my $rc = system($cmd); @@ -788,27 +787,6 @@ sub run_simple { } } -sub run_argv { - my (@command) = @_; - my $rc = system(@command); - if ($rc != 0) { - my $exit = $rc == -1 ? 255 : ($rc >> 8); - die "Command failed (rc=$exit): @command\n"; - } -} - -sub capture { - my ($cmd) = @_; - my $out = `$cmd`; - my $rc = $?; - if ($rc != 0) { - my $exit = $rc == -1 ? 255 : ($rc >> 8); - die "Command failed (rc=$exit): $cmd\n$out\n"; - } - chomp $out; - return $out; -} - sub run_step { my (%args) = @_; my $step = $args{step} // 'Run command'; @@ -829,12 +807,12 @@ sub run_step { my $full_cmd = $cmd; if ($cwd) { - $full_cmd = "cd " . sh_quote($cwd) . " && $cmd"; + $full_cmd = "cd " . shell_quote($cwd) . " && $cmd"; } if ($log) { my $log_dir = dirname($log); make_path($log_dir) if !-d $log_dir; - $full_cmd .= " > " . sh_quote($log) . " 2>&1"; + $full_cmd .= " > " . shell_quote($log) . " 2>&1"; } my $rc = system($full_cmd); @@ -1006,15 +984,6 @@ sub assert_genesis_release_copied { } } -sub checksum_sets_match { - my ($left, $right) = @_; - return 0 unless keys(%{$left}) == keys(%{$right}); - for my $name (keys %{$left}) { - return 0 unless exists($right->{$name}) && $left->{$name} eq $right->{$name}; - } - return 1; -} - sub collect_rpms { my (%args) = @_; my $roots = $args{roots} // []; @@ -1125,14 +1094,14 @@ sub resolve_mock_cfg { rocky => 'rocky', ); my $candidate = "${os_id}+epel-${rel}-${arch}"; - my $rc = system("mock -r " . sh_quote($candidate) . " --print-root-path >/dev/null 2>&1"); + my $rc = system("mock -r " . shell_quote($candidate) . " --print-root-path >/dev/null 2>&1"); if ($rc == 0) { return $candidate; } if (exists $short_forms{$os_id}) { my $short = $short_forms{$os_id}; $candidate = "${short}+epel-${rel}-${arch}"; - $rc = system("mock -r " . sh_quote($candidate) . " --print-root-path >/dev/null 2>&1"); + $rc = system("mock -r " . shell_quote($candidate) . " --print-root-path >/dev/null 2>&1"); if ($rc == 0) { print "Mock config resolved (short form): $candidate\n"; return $candidate; @@ -1193,7 +1162,7 @@ sub acquire_output_lock { if (mkdir $lock) { $HELD_LOCK = $lock; $LOCK_OWNER_PID = $$; - my $host = capture('uname -n') || 'unknown'; + my $host = capture_command('uname', '-n') || 'unknown'; if (open my $fh, '>', "$lock/owner") { print {$fh} "host=$host\npid=$$\nepoch=" . time() . "\n"; close $fh; @@ -1250,19 +1219,3 @@ sub uniq { my %seen; return grep { defined($_) && !$seen{$_}++ } @_; } - -sub slurp_chomp { - my ($path) = @_; - open my $fh, '<', $path or die "Cannot read $path: $!\n"; - my $line = <$fh>; - close $fh; - chomp $line if defined $line; - return $line // ''; -} - -sub sh_quote { - my ($s) = @_; - $s = '' if !defined $s; - $s =~ s/'/'"'"'/g; - return "'$s'"; -}