mirror of
https://github.com/xcat2/xcat-core.git
synced 2026-10-06 17:46:55 +00:00
Daniel Hilst
f5e3f7775b
The CI run is now three phases in order: the perl and bash unit tests, the ci_test set, then the DHCP on-wire cases. The wire cases carry dhcp_wire and deliberately not ci_test. They are the only cases that do not leave the machine alone while they run -- a veth pair appears, site.dhcpinterfaces and site.dhcpbackend change, the DHCP daemon is stopped and started -- and although each one restores all of it from a trap, a ci_test case running in between would be sharing a management node that is mid-reconfiguration. Holding them out of the set is also what stops them being run twice. run_dhcp_wire_test runs the whole set once per backend installed: against ISC, then against Kea, each pass bracketed by `dhcpfixture.sh backend-setup` and `backend-teardown`. It reports separately from the fast regression test and counts case runs rather than cases, since each case is run once per backend and each failure carries the backend name. The phase writes its own xcattest configuration without a dhcpbackend key: xcattest applies [Table_site] before every case, so the regression conf's `dhcpbackend=isc` would have reset the backend case by case and left the run silently testing ISC twice.
981 lines
37 KiB
Perl
981 lines
37 KiB
Perl
# IBM(c) 2022 EPL license http://www.eclipse.org/legal/epl-v10.html
|
|
|
|
use strict;
|
|
use warnings;
|
|
use Getopt::Long;
|
|
use Data::Dumper;
|
|
use Time::Local;
|
|
use File::Basename;
|
|
use File::Path;
|
|
use File::Find;
|
|
use LWP::UserAgent;
|
|
use HTTP::Request;
|
|
use Encode;
|
|
use Encode::CN;
|
|
use JSON;
|
|
use URI::Escape;
|
|
use LWP::Simple;
|
|
use Cwd;
|
|
|
|
use Term::ANSIColor qw(:constants);
|
|
$Term::ANSIColor::AUTORESET = 1;
|
|
|
|
#---Global attributes---
|
|
my $rst = 0;
|
|
my $retries = 5; # Try this many times to get response
|
|
my $check_result_str="``CI CHECK RESULT`` : ";
|
|
my $last_func_start = timelocal(localtime());
|
|
my $GITHUB_API = "https://api.github.com";
|
|
|
|
# The workflow starts us in the checked out source tree. The unit tests under
|
|
# xCAT-test/unit resolve xCAT modules and fixture files relative to that tree
|
|
# through FindBin, so they can only be run from a source tree. Take a copy
|
|
# before building and run the unit tests out of the copy.
|
|
#
|
|
# This used to be mandatory rather than tidy: build-ubunturepo set
|
|
# local_core_repo_path="$curdir/../../xcat-core"
|
|
# which, under the work/<repo>/<repo> layout GitHub checks out into, resolved to
|
|
# the checkout's own parent, and it rm -rf'd that path to make room for the apt
|
|
# repository -- destroying the tree the tests need. builddebs.pl writes under
|
|
# dist/debs INSIDE the checkout and restores every file it edits, so the copy is
|
|
# now only isolating the tests from build residue.
|
|
my $srcdir = getcwd();
|
|
my $unitsrc = ($ENV{'RUNNER_TEMP'} ? $ENV{'RUNNER_TEMP'} : "/tmp") . "/xcat-core-unitsrc";
|
|
|
|
# Cases whose output is printed even when they pass. A passing case is normally
|
|
# silent, which is the right default for 250 of them but makes it impossible to
|
|
# tell from the log whether a case did real work or skipped everything -- a
|
|
# distinction that matters for cases wrapping prove, since prove exits 0 either
|
|
# way. Name a case here to see its output; empty the list for the quiet
|
|
# behaviour.
|
|
my @verbose_cases = qw(integration_tests);
|
|
|
|
#--------------------------------------------------------
|
|
# Fuction name: runcmd
|
|
# Description: run a command after 'cmd' label in one case
|
|
# Attributes:
|
|
# Return code:
|
|
# $::RUNCMD_RC : the return code of command
|
|
# @$outref : the output of command
|
|
#--------------------------------------------------------
|
|
sub runcmd
|
|
{
|
|
my ($cmd) = @_;
|
|
my $rc = 0;
|
|
$::RUNCMD_RC = 0;
|
|
my $outref = [];
|
|
@$outref = `$cmd 2>&1`;
|
|
if ($?)
|
|
{
|
|
$rc = $?;
|
|
$rc = $rc >> 8;
|
|
$::RUNCMD_RC = $rc;
|
|
}
|
|
chomp(@$outref);
|
|
return @$outref;
|
|
|
|
}
|
|
|
|
#--------------------------------------------------------
|
|
# Fuction name: get_files_recursive
|
|
# Description: Search all file in one directory recursively
|
|
# Attributes:
|
|
# $dir (input attribute)
|
|
# The target scan directory
|
|
# $files_path_ref (output attribute)
|
|
# the reference of array where save all vaild files under $dir
|
|
# Return code:
|
|
#--------------------------------------------------------
|
|
sub get_files_recursive
|
|
{
|
|
my $dir = shift;
|
|
my $files_path_ref = shift;
|
|
|
|
my $fd = undef;
|
|
if(!opendir($fd, $dir)){
|
|
print "[get_files_recursive]: failed to open $dir :$!\n";
|
|
return 1;
|
|
}
|
|
|
|
for (; ;)
|
|
{
|
|
my $direntry = readdir($fd);
|
|
last unless (defined($direntry));
|
|
next if ($direntry =~ m/^\.\w*/);
|
|
next if ($direntry eq '..');
|
|
my $target = "$dir/$direntry";
|
|
if (-d $target) {
|
|
get_files_recursive($target, $files_path_ref);
|
|
} else {
|
|
push(@{$files_path_ref}, glob("$target\n"));
|
|
}
|
|
}
|
|
closedir($fd);
|
|
return 0;
|
|
}
|
|
|
|
#--------------------------------------------------------
|
|
# Fuction name: check_pr_format
|
|
# Description:
|
|
# Attributes:
|
|
# Return code:
|
|
#--------------------------------------------------------
|
|
sub check_pr_format{
|
|
if($ENV{'TRAVIS_EVENT_TYPE'} eq "pull_request"){
|
|
my $pr_url = "$GITHUB_API/repos/$ENV{'TRAVIS_REPO_SLUG'}/pulls/$ENV{'TRAVIS_PULL_REQUEST'}";
|
|
my $pr_url_resp;
|
|
my $counter = 1;
|
|
while($counter <= $retries) {
|
|
$pr_url_resp = get($pr_url);
|
|
if ($pr_url_resp) {
|
|
last; # Got response, no more retries
|
|
} else {
|
|
sleep($counter*2); # Sleep and try again
|
|
print "[check_pr_format] $counter Did not get response, sleeping ". $counter*2 . "\n";
|
|
$counter++;
|
|
}
|
|
}
|
|
unless ($pr_url_resp) {
|
|
print "[check_pr_format] After $retries retries, not able to get response from $pr_url \n";
|
|
# Failed after trying a few times, return error
|
|
return $counter;
|
|
}
|
|
my $pr_content = decode_json($pr_url_resp);
|
|
my $pr_title = $pr_content->{title};
|
|
my $pr_body = $pr_content->{body};
|
|
my $pr_milestone = $pr_content->{milestone};
|
|
my $pr_labels_len = @{$pr_content->{labels}};
|
|
|
|
#print "[check_pr_format] Dumper pr_content:\n";
|
|
#print Dumper $pr_content;
|
|
print "[check_pr_format] pr title = $pr_title\n";
|
|
print "[check_pr_format] pr body = $pr_body \n";
|
|
|
|
my $checkrst="";
|
|
if(! $pr_title){
|
|
$checkrst.="Missing title.";
|
|
}
|
|
if(! $pr_body){
|
|
$checkrst.="Missing description.";
|
|
}
|
|
|
|
if(! $pr_milestone){
|
|
$checkrst.="Missing milestone.";
|
|
}
|
|
|
|
if(! $pr_labels_len){
|
|
$checkrst.="Missing labels.";
|
|
}
|
|
|
|
# Guard against root user making commits
|
|
$checkrst.=check_commit_owner('root');
|
|
|
|
if(length($checkrst) == 0){
|
|
$check_result_str .= "> **PR FORMAT CORRECT**";
|
|
#send_back_comment("$check_result_str");
|
|
}else{
|
|
# Warning if missing milestone or labels, others are errors
|
|
if($checkrst =~ /milestone/ || $checkrst =~ /labels/){
|
|
$check_result_str .= "> **PR FORMAT WARNING** : $checkrst";
|
|
#send_back_comment("$check_result_str");
|
|
}else{
|
|
$check_result_str .= "> **PR FORMAT ERROR** : $checkrst";
|
|
#send_back_comment("$check_result_str");
|
|
return 1;
|
|
}
|
|
}
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
#--------------------------------------------------------
|
|
# Fuction name: check_commit_owner
|
|
# Description: Verify commits are not done by specified user
|
|
# Attributes: user login to reject
|
|
# Return:
|
|
# Error string -User rejected,
|
|
# Empty string -User not rejected
|
|
#--------------------------------------------------------
|
|
sub check_commit_owner{
|
|
my $invalid_user = shift;
|
|
if($ENV{'TRAVIS_EVENT_TYPE'} eq "pull_request"){
|
|
my $commits_content;
|
|
my $commits_len = 0;
|
|
my $json = new JSON;
|
|
my $commits_url = "$GITHUB_API/repos/$ENV{'TRAVIS_REPO_SLUG'}/pulls/$ENV{'TRAVIS_PULL_REQUEST'}/commits";
|
|
my $commits_url_resp;
|
|
my $counter = 1;
|
|
while($counter <= $retries) {
|
|
$commits_url_resp = get($commits_url);
|
|
if ($commits_url_resp) {
|
|
last; # Got response, no more retries
|
|
} else {
|
|
sleep($counter*2); # Sleep and try again
|
|
print "[check_commit_owner] $counter Did not get response, sleeping ". $counter*2 . "\n";
|
|
$counter++;
|
|
}
|
|
}
|
|
if ($commits_url_resp) {
|
|
$commits_content = $json->decode($commits_url_resp);
|
|
$commits_len = @$commits_content;
|
|
} else {
|
|
print "[check_commit_owner] After $retries retries, not able to get response from $commits_url \n";
|
|
return "Unable to verify login of committer.";
|
|
}
|
|
|
|
if($commits_len > 0) {
|
|
foreach my $commit (@{$commits_content}){
|
|
my $committer = $commit->{committer};
|
|
my $committer_login = $committer->{login};
|
|
print "[check_commit_owner] Committer login $committer_login \n";
|
|
if($committer_login =~ /^$invalid_user$/) {
|
|
# Committer logins matches
|
|
return "Commits by $invalid_user not allowed";
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return "";
|
|
}
|
|
#--------------------------------------------------------
|
|
# Fuction name: send_back_comment
|
|
# Description: Append to comment of the PR passed $message
|
|
# Attributes: Message to append to PR
|
|
# Return code:
|
|
#--------------------------------------------------------
|
|
sub send_back_comment{
|
|
my $message = shift;
|
|
|
|
my $comment_url = "$GITHUB_API/repos/$ENV{'TRAVIS_REPO_SLUG'}/issues/$ENV{'TRAVIS_PULL_REQUEST'}/comments";
|
|
my $json = new JSON;
|
|
my $comment_len = 0;
|
|
my $comment_content;
|
|
my $comment_url_resp;
|
|
my $counter = 1;
|
|
while($counter <= $retries) {
|
|
$comment_url_resp = get($comment_url);
|
|
if ($comment_url_resp) {
|
|
last; # Got response, no more retries
|
|
} else {
|
|
sleep($counter*2); # Sleep and try again
|
|
print "[send_back_comment] $counter Did not get response, sleeping ". $counter*2 . "\n";
|
|
$counter++;
|
|
}
|
|
}
|
|
unless ($comment_url_resp) {
|
|
print "[send_back_comment] After $retries retries, not able to get response from $comment_url \n";
|
|
# Failed after trying a few times, return
|
|
return;
|
|
}
|
|
print "\n\n>>>>>Dumper comment_url_resp:\n";
|
|
print Dumper $comment_url_resp;
|
|
|
|
$comment_content = $json->decode($comment_url_resp);
|
|
$comment_len = @$comment_content;
|
|
|
|
my $post_url = $comment_url;
|
|
my $post_method = "POST";
|
|
if($comment_len > 0){
|
|
foreach my $comment (@{$comment_content}){
|
|
if($comment->{'body'} =~ /CI CHECK RESULT/) {
|
|
$post_url = $comment->{'url'};
|
|
$post_method = "PATCH";
|
|
}
|
|
}
|
|
}
|
|
|
|
print "[send_back_comment] method = $post_method to $post_url. Message = $message\n";
|
|
if ( $ENV{'xcatbotuser'} and $ENV{'xcatbotpw'}) {
|
|
`curl -u "$ENV{'xcatbotuser'}:$ENV{'xcatbotpw'}" -X $post_method -d '{"body":"$message"}' $post_url`;
|
|
}
|
|
else {
|
|
print "Not able to update pull request with message: $message\n";
|
|
}
|
|
}
|
|
|
|
#--------------------------------------------------------
|
|
# Fuction name: preserve_source_tree
|
|
# Description: Copy the checkout aside before the build destroys it, so the
|
|
# unit tests still have a source tree to run against afterwards.
|
|
# Must be called before build_xcat_core().
|
|
# Attributes:
|
|
# Return code: 0 Success 1 Failed
|
|
#--------------------------------------------------------
|
|
sub preserve_source_tree{
|
|
my $cmd = "rm -rf $unitsrc && cp -a $srcdir $unitsrc";
|
|
print "[preserve_source_tree] running $cmd\n";
|
|
my @output = runcmd("$cmd");
|
|
if($::RUNCMD_RC){
|
|
print RED "[preserve_source_tree] $cmd ....[Failed]\n";
|
|
print Dumper \@output;
|
|
return 1;
|
|
}
|
|
|
|
@output = runcmd("find $unitsrc/xCAT-test/unit -name '*.t' | wc -l");
|
|
my $perl_count = $output[0];
|
|
@output = runcmd("find $unitsrc/xCAT-test/bats -name '*.bats' 2>/dev/null | wc -l");
|
|
my $bats_count = $output[0];
|
|
print "[preserve_source_tree] preserved $srcdir in $unitsrc ($perl_count Perl unit tests, $bats_count BATS tests)\n";
|
|
return 0;
|
|
}
|
|
|
|
#--------------------------------------------------------
|
|
# Fuction name: build_xcat_core
|
|
# Description:
|
|
# Attributes:
|
|
# Return code:
|
|
#--------------------------------------------------------
|
|
sub build_xcat_core{
|
|
my @output;
|
|
|
|
my $cmd = "sudo ./builddebs.pl --force";
|
|
@output = runcmd("$cmd");
|
|
if($::RUNCMD_RC){
|
|
my $lastline = $output[-1];
|
|
$lastline =~ s/[\r\n\t\\"']*//g;
|
|
print "[build_xcat_core] $cmd ....[Failed]\n";
|
|
$check_result_str .= "> **BUILD ERROR**, Please click ``Details`` label in ``Merge pull request`` box for detailed information";
|
|
print $check_result_str;
|
|
return 1;
|
|
}else{
|
|
print "[build_xcat_core] $cmd ....[Pass]\n";
|
|
$check_result_str .= "> **BUILD SUCCESSFUL** ";
|
|
print $check_result_str;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
#--------------------------------------------------------
|
|
# Fuction name: install_xcat
|
|
# Description:
|
|
# Attributes:
|
|
# Return code:
|
|
#--------------------------------------------------------
|
|
sub install_xcat{
|
|
|
|
my $repo = "$srcdir/dist/debs/xcat-core";
|
|
unless (-x "$repo/mklocalrepo.sh") {
|
|
print RED "[install_xcat] $repo/mklocalrepo.sh missing -- did the build run?\n";
|
|
$check_result_str .= "> **INSTALL XCAT ERROR** : the build produced no apt repository ";
|
|
print $check_result_str;
|
|
return 1;
|
|
}
|
|
|
|
my @cmds = ("sudo $repo/mklocalrepo.sh",
|
|
"sudo chmod 777 /etc/apt/sources.list",
|
|
"sudo echo \"deb [arch=amd64 allow-insecure=yes] http://xcat.org/files/xcat/repos/apt/latest/xcat-dep noble main\" >> /etc/apt/sources.list",
|
|
"sudo echo \"deb [arch=ppc64el allow-insecure=yes] http://xcat.org/files/xcat/repos/apt/latest/xcat-dep noble main\" >> /etc/apt/sources.list",
|
|
"sudo timeout 600 apt-get -qq -o Acquire::Retries=3 -o Acquire::http::Timeout=30 --allow-insecure-repositories update");
|
|
chdir $ENV{RUNNER_WORKSPACE};;
|
|
|
|
my @output;
|
|
foreach my $cmd (@cmds){
|
|
print "[install_xcat] running $cmd\n";
|
|
@output = runcmd("$cmd");
|
|
if($::RUNCMD_RC){
|
|
print RED "[install_xcat] $cmd. ...[Failed]\n";
|
|
print "[install_xcat] error message:\n";
|
|
print Dumper \@output;
|
|
$check_result_str .= "> **INSTALL XCAT ERROR** : Please click ``Details`` label in ``Merge pull request`` box for detailed information ";
|
|
print $check_result_str;
|
|
return 1;
|
|
}
|
|
}
|
|
|
|
my $cmd = "sudo timeout 1200 env DEBIAN_FRONTEND=noninteractive apt-get -o Acquire::Retries=3 -o Acquire::http::Timeout=30 install -y xcat --allow-remove-essential --allow-unauthenticated";
|
|
@output = runcmd("$cmd");
|
|
if($::RUNCMD_RC){
|
|
my $lastline = $output[-1];
|
|
$lastline =~ s/[\r\n\t\\"']*//g;
|
|
print "[install_xcat] $cmd ....[Failed]\n";
|
|
print ">>>>>Dumper the output of '$cmd'\n";
|
|
print Dumper \@output;
|
|
$check_result_str .= "> **INSTALL XCAT ERROR** : Please click ``Details`` label in ``Merge pull request`` box for detailed information";
|
|
print $check_result_str;
|
|
return 1;
|
|
}else{
|
|
print "[install_xcat] $cmd ....[Pass]\n";
|
|
|
|
print "\n------Config xcat and verify xcat is working correctly-----\n";
|
|
@cmds = ("sudo -s /opt/xcat/share/xcat/scripts/setup-local-client.sh -f runner",
|
|
"sudo -s /opt/xcat/sbin/chtab priority=1.1 policy.name=runner policy.rule=allow",
|
|
". /etc/profile.d/xcat.sh && tabdump policy",
|
|
". /etc/profile.d/xcat.sh && tabdump site",
|
|
". /etc/profile.d/xcat.sh && lsxcatd -a",
|
|
"ls /opt/xcat/sbin",
|
|
"service xcatd status");
|
|
my $ret = 0;
|
|
foreach my $cmd (@cmds){
|
|
print "\n[install_xcat] running $cmd.....\n";
|
|
@output = runcmd("$cmd");
|
|
print Dumper \@output;
|
|
if($::RUNCMD_RC){
|
|
print RED "[install_xcat] $cmd. ...[Failed]\n";
|
|
#print Dumper \@output;
|
|
$ret = 1;
|
|
}else{
|
|
print "[install_xcat] $cmd....[Pass]\n";
|
|
}
|
|
}
|
|
$cmd = "sudo timeout 1200 env DEBIAN_FRONTEND=noninteractive apt-get -o Acquire::Retries=3 -o Acquire::http::Timeout=30 install -y xcat-probe --allow-remove-essential --allow-unauthenticated";
|
|
@output = runcmd("$cmd");
|
|
if($::RUNCMD_RC){
|
|
print RED "[install_xcat] $cmd ....[Failed]\n";
|
|
print Dumper \@output;
|
|
$ret = 1;
|
|
}else{
|
|
print "[install_xcat] $cmd ....[Pass]:\n";
|
|
}
|
|
|
|
if($ret){
|
|
$check_result_str .= "> **INSTALL XCAT ERROR** : Please click ``Details`` label in ``Merge pull request`` box for detailed information\n";
|
|
print $check_result_str;
|
|
return 1;
|
|
}
|
|
|
|
$check_result_str .= "> **INSTALL XCAT SUCCESSFUL**";
|
|
print $check_result_str;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
|
|
#--------------------------------------------------------
|
|
# Fuction name: run_unit_tests
|
|
# Description: Run every Perl unit test under xCAT-test/unit with prove.
|
|
# Runs against the pre-build copy of the source tree taken by
|
|
# preserve_source_tree(): the tests reach for xCAT modules and
|
|
# fixture files through FindBin, so the installed copy under
|
|
# /opt/xcat/share/xcat/tools/autotest is not enough for them.
|
|
# Attributes:
|
|
# Return code: 0 all tests passed, 1 otherwise
|
|
#--------------------------------------------------------
|
|
sub run_unit_tests{
|
|
my $cmd = "cd $unitsrc && prove -r xCAT-test/unit";
|
|
print "[run_unit_tests] running $cmd\n";
|
|
my @output = runcmd("$cmd");
|
|
print Dumper \@output;
|
|
if($::RUNCMD_RC){
|
|
print RED "[run_unit_tests] $cmd ....[Failed]\n";
|
|
$check_result_str .= "> **UNIT TESTS Failed** : Please click ``Details`` label in ``Merge pull request`` box for detailed information\n";
|
|
print $check_result_str;
|
|
return 1;
|
|
}
|
|
|
|
print "[run_unit_tests] $cmd ....[Pass]\n";
|
|
$check_result_str .= "> **UNIT TESTS Successful**\n";
|
|
print $check_result_str;
|
|
return 0;
|
|
}
|
|
|
|
#--------------------------------------------------------
|
|
# Fuction name: run_bats_tests
|
|
# Description: Run shell-script unit tests under xCAT-test/bats.
|
|
# Runs against the pre-build copy of the source tree taken by
|
|
# preserve_source_tree(), like the Perl unit tests.
|
|
# Attributes:
|
|
# Return code: 0 all tests passed, 1 otherwise
|
|
#--------------------------------------------------------
|
|
sub run_bats_tests{
|
|
my $testdir = "$unitsrc/xCAT-test/bats";
|
|
my @output = runcmd("find $testdir -name '*.bats' -print -quit 2>/dev/null");
|
|
if (!@output) {
|
|
print "[run_bats_tests] no BATS tests found under $testdir\n";
|
|
return 0;
|
|
}
|
|
|
|
my $cmd = "cd $unitsrc && bats -r xCAT-test/bats";
|
|
print "[run_bats_tests] running $cmd\n";
|
|
@output = runcmd("$cmd");
|
|
print Dumper \@output;
|
|
if($::RUNCMD_RC){
|
|
print RED "[run_bats_tests] $cmd ....[Failed]\n";
|
|
$check_result_str .= "> **BATS TESTS Failed** : Please click ``Details`` label in ``Merge pull request`` box for detailed information\n";
|
|
print $check_result_str;
|
|
return 1;
|
|
}
|
|
|
|
print "[run_bats_tests] $cmd ....[Pass]\n";
|
|
$check_result_str .= "> **BATS TESTS Successful**\n";
|
|
print $check_result_str;
|
|
return 0;
|
|
}
|
|
|
|
#--------------------------------------------------------
|
|
# Fuction name: run_dhcptest_unit_tests
|
|
# Description: Run the dhcptest offline checks: its Python unit tests and
|
|
# `dhcptest validate` over every shipped .conf. Both are
|
|
# unprivileged and touch no network -- the wire scenarios need a
|
|
# provisioning NIC and a DHCP server, so they are not run here.
|
|
# Skipped when python3 is absent rather than failing the build.
|
|
# Attributes:
|
|
# Return code: 0 all checks passed, 1 otherwise
|
|
#--------------------------------------------------------
|
|
sub run_dhcptest_unit_tests{
|
|
my $testdir = "$unitsrc/xCAT-test/dhcptest";
|
|
if (! -d $testdir) {
|
|
print "[run_dhcptest_unit_tests] no dhcptest found under $testdir\n";
|
|
return 0;
|
|
}
|
|
runcmd("which python3 2>/dev/null");
|
|
if($::RUNCMD_RC){
|
|
print "[run_dhcptest_unit_tests] python3 is not installed, skipping\n";
|
|
return 0;
|
|
}
|
|
|
|
my $cmd = "cd $testdir && python3 -m unittest discover -s tests"
|
|
. " && python3 src/dhcptest validate conf/*.conf";
|
|
print "[run_dhcptest_unit_tests] running $cmd\n";
|
|
my @output = runcmd("$cmd");
|
|
print Dumper \@output;
|
|
if($::RUNCMD_RC){
|
|
print RED "[run_dhcptest_unit_tests] $cmd ....[Failed]\n";
|
|
$check_result_str .= "> **DHCPTEST TESTS Failed** : Please click ``Details`` label in ``Merge pull request`` box for detailed information\n";
|
|
print $check_result_str;
|
|
return 1;
|
|
}
|
|
|
|
print "[run_dhcptest_unit_tests] $cmd ....[Pass]\n";
|
|
$check_result_str .= "> **DHCPTEST TESTS Successful**\n";
|
|
print $check_result_str;
|
|
return 0;
|
|
}
|
|
|
|
#--------------------------------------------------------
|
|
# Fuction name: check_syntax
|
|
# Description:
|
|
# Attributes:
|
|
# Return code:
|
|
#--------------------------------------------------------
|
|
sub check_syntax{
|
|
my @output;
|
|
my @syntax_err;
|
|
my $ret = 0;
|
|
|
|
my @target_dirs=("/opt/xcat",
|
|
"/install");
|
|
foreach my $dir (@target_dirs){
|
|
my @files = ();
|
|
get_files_recursive("$dir", \@files);
|
|
|
|
foreach my $file (@files) {
|
|
next if($file =~ /\/opt\/xcat\/share\/xcat\/netboot\/genesis\//);
|
|
next if($file =~ /\/opt\/xcat\/probe\//);
|
|
next if($file =~ /\/opt\/xcat\/share\/xcat\/tools\/autotest\/unit\//);
|
|
|
|
@output = runcmd("file $file");
|
|
if($output[0] =~ /perl /i){
|
|
@output = runcmd("sudo bash -c '. /etc/profile.d/xcat.sh && perl -I /opt/xcat/lib/perl -I /opt/xcat/lib -I /usr/lib/perl5 -I /usr/share/perl -c $file'");
|
|
if($::RUNCMD_RC){
|
|
push @syntax_err, @output;
|
|
$ret = 1;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if(@syntax_err){
|
|
print "[check_syntax] syntax checking ....[Failed]\n";
|
|
print "[check_syntax] Dumper error message:\n";
|
|
print Dumper @syntax_err;
|
|
$check_result_str .= "> **CODE SYNTAX ERROR** : Please click ``Details`` label in ``Merge pull request`` box for detailed information";
|
|
print $check_result_str;
|
|
}else{
|
|
print "[check_syntax] syntax checking ....[Pass]\n";
|
|
$check_result_str .= "> **CODE SYNTAX CORRECT**";
|
|
print $check_result_str;
|
|
}
|
|
|
|
return $ret;
|
|
}
|
|
|
|
#--------------------------------------------------------
|
|
# Fuction name: write_regression_conf
|
|
# Description: write an xcattest configuration file and return its path
|
|
# Attributes: $name - the file name, under the working directory
|
|
# $pin_backend - set site.dhcpbackend in [Table_site]
|
|
#
|
|
# xcattest applies [Table_site] before every case it runs, so a file that
|
|
# pins dhcpbackend pins it again on each invocation. That is what the fast
|
|
# regression set wants -- one known backend for the whole run -- and exactly
|
|
# what the DHCP wire phase must not have: it selects the backend itself, once
|
|
# per pass, and a conf file that reset it would undo the kea pass case by
|
|
# case and leave the run silently testing isc twice.
|
|
# Return code: the path written
|
|
#--------------------------------------------------------
|
|
sub write_regression_conf{
|
|
my ($name, $pin_backend) = @_;
|
|
|
|
my $hostname = `hostname`;
|
|
chomp($hostname);
|
|
print "hostname = $hostname\n";
|
|
|
|
my $conf_file = "$ENV{'PWD'}/$name";
|
|
system "sudo touch $conf_file";
|
|
system "sudo chmod 777 $conf_file";
|
|
open(my $fh, '>', $conf_file) or die "Could not open test configuration file $!";
|
|
print $fh "[System]\nMN=$hostname\n[Table_site]\nkey=domain\nvalue=pok.stglabs.ibm.com\n";
|
|
print $fh "key=dhcpbackend\nvalue=isc\n" if($pin_backend);
|
|
close($fh);
|
|
|
|
print "Dumper $name:\n";
|
|
my @output = runcmd("cat $conf_file");
|
|
print Dumper \@output;
|
|
|
|
return $conf_file;
|
|
}
|
|
|
|
#--------------------------------------------------------
|
|
# Fuction name: list_cases
|
|
# Description: the names of the cases matching an xcattest label expression
|
|
# Attributes: $expression - e.g. "ci_test-dhcp_wire"
|
|
# Return code: the list of case names, empty if none match or the query failed
|
|
#--------------------------------------------------------
|
|
sub list_cases{
|
|
my $expression = shift;
|
|
my $cmd = "sudo bash -c '. /etc/profile.d/xcat.sh && xcattest -s \"$expression\" -l'";
|
|
my @cases = runcmd("$cmd");
|
|
if($::RUNCMD_RC){
|
|
print RED "[list_cases] $cmd ....[Failed]\n";
|
|
print Dumper \@cases;
|
|
return ();
|
|
}
|
|
print "[list_cases] $expression:\n";
|
|
print Dumper \@cases;
|
|
# "There are no cases match search expression ..." is a sentence, not a case.
|
|
return grep { /^\S+$/ } @cases;
|
|
}
|
|
|
|
#--------------------------------------------------------
|
|
# Fuction name: run_cases
|
|
# Description: run a list of cases, one xcattest invocation each
|
|
# Attributes: $conf_file - the regression configuration
|
|
# $cases - the case names
|
|
# $suffix - appended to a failing case name in the summary, so
|
|
# a case run more than once says which run failed
|
|
# $counts - hashref accumulating pass/fail across every call
|
|
# $tag - the phase name to print log lines under
|
|
# Return code: none; results land in $counts and @{$counts->{failed}}
|
|
#--------------------------------------------------------
|
|
sub run_cases{
|
|
my ($conf_file, $cases, $suffix, $counts, $tag) = @_;
|
|
$tag = "run_fast_regression_test" unless($tag);
|
|
my $x = 0;
|
|
foreach my $case (@{$cases}){
|
|
++$x;
|
|
my $cmd = "sudo bash -c '. /etc/profile.d/xcat.sh && xcattest -f $conf_file -t $case'";
|
|
print "[$tag] run $x$suffix: $cmd\n";
|
|
my @output = runcmd("$cmd");
|
|
my $verbose = grep { $_ eq $case } @verbose_cases;
|
|
if($verbose){
|
|
print "[$tag] output of $case (listed in \@verbose_cases):\n";
|
|
print Dumper \@output;
|
|
}
|
|
for(my $i = $#output; $i>-1; --$i){
|
|
if($output[$i] =~ /------END::(.+)::Failed/){
|
|
push @{$counts->{failed}}, "$1$suffix";
|
|
++$counts->{fail};
|
|
print Dumper \@output unless($verbose);
|
|
last;
|
|
}elsif ($output[$i] =~ /------END::(.+)::Passed/){
|
|
++$counts->{pass};
|
|
last;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
#--------------------------------------------------------
|
|
# Fuction name: run_dhcp_wire_test
|
|
# Description: run the DHCP wire cases, once per backend installed here.
|
|
#
|
|
# This is the last phase of the run, after the ci_test set, and the wire
|
|
# cases are deliberately not labelled ci_test. They do not leave the machine
|
|
# alone while they run -- a veth pair appears, site.dhcpinterfaces and
|
|
# site.dhcpbackend change, the DHCP daemon is stopped and started -- and
|
|
# although each case restores all of it from a trap, a ci_test case running
|
|
# in between would be sharing a management node that is mid-reconfiguration.
|
|
# Keeping them out of the set is also what stops them being run twice.
|
|
#
|
|
# Which DHCP backend a management node runs is an implementation default --
|
|
# kea on the newer distros, isc on the older ones -- so a node booting on the
|
|
# same network must be told the same things either way. Testing whichever
|
|
# backend the runner happened to configure proves half of that and hides
|
|
# every drift between the two.
|
|
#
|
|
# The whole set is therefore run against one backend and then again against
|
|
# the other, rather than each case switching backends internally: the daemon
|
|
# is reconfigured once per pass instead of once per case, and each failure in
|
|
# the summary carries the backend it belongs to.
|
|
#
|
|
# dhcpfixture.sh brackets each pass -- backend-setup selects the backend and
|
|
# stops the other daemon, backend-teardown puts the site table back and
|
|
# restarts what was running before.
|
|
# Attributes:
|
|
# Return code: 0 if every case passed under every backend
|
|
#--------------------------------------------------------
|
|
sub run_dhcp_wire_test{
|
|
my @cases = list_cases("dhcp_wire");
|
|
unless(@cases){
|
|
print "[run_dhcp_wire_test] no DHCP wire cases to run\n";
|
|
$check_result_str .= "> **DHCP WIRE TEST Skipped**: no case carries the dhcp_wire label\n";
|
|
print $check_result_str;
|
|
return 0;
|
|
}
|
|
|
|
my $conf_file = write_regression_conf("dhcp-wire.conf", 0);
|
|
my $fixture = "/opt/xcat/share/xcat/tools/autotest/testcase/dhcptest/dhcpfixture.sh";
|
|
unless(-f $fixture){
|
|
print RED "[run_dhcp_wire_test] $fixture is missing, so the wire cases cannot be run per backend\n";
|
|
$check_result_str .= "> **DHCP WIRE TEST Failed**: $fixture is not installed\n";
|
|
print $check_result_str;
|
|
return 1;
|
|
}
|
|
|
|
my @backends = runcmd("sudo bash -c '. /etc/profile.d/xcat.sh && $fixture backends'");
|
|
@backends = split ' ', join ' ', @backends;
|
|
unless(@backends){
|
|
print RED "[run_dhcp_wire_test] no DHCP daemon is installed, so the wire cases have nothing to talk to\n";
|
|
$check_result_str .= "> **DHCP WIRE TEST Failed**: no DHCP daemon is installed\n";
|
|
print $check_result_str;
|
|
return 1;
|
|
}
|
|
print "[run_dhcp_wire_test] DHCP backends installed here: @backends\n";
|
|
|
|
my %counts = (pass => 0, fail => 0, failed => []);
|
|
foreach my $backend (@backends){
|
|
my $cmd = "sudo bash -c '. /etc/profile.d/xcat.sh && $fixture backend-setup $backend'";
|
|
my @output = runcmd("$cmd");
|
|
print Dumper \@output;
|
|
if($::RUNCMD_RC){
|
|
print RED "[run_dhcp_wire_test] cannot serve with $backend\n";
|
|
push @{$counts{failed}}, "backend-setup($backend)";
|
|
++$counts{fail};
|
|
next;
|
|
}
|
|
|
|
run_cases($conf_file, \@cases, "($backend)", \%counts, "run_dhcp_wire_test");
|
|
|
|
@output = runcmd("sudo bash -c '. /etc/profile.d/xcat.sh && $fixture backend-teardown $backend'");
|
|
print Dumper \@output;
|
|
}
|
|
|
|
my $backend_str = join(" and ", @backends);
|
|
if(@backends < 2){
|
|
print "[run_dhcp_wire_test] WARNING: only $backend_str is installed here, so the backends were not compared\n";
|
|
}
|
|
|
|
# Case runs rather than cases: each case is run once per backend.
|
|
my $casenum = $counts{pass} + $counts{fail};
|
|
if($counts{fail}){
|
|
my $log_str = join(",", @{$counts{failed}});
|
|
$check_result_str .= "> **DHCP WIRE TEST Failed** ($backend_str): Totalrun $casenum Passed $counts{pass} Failed $counts{fail} FailedCases: $log_str. Please click ``Details`` label in ``Merge pull request`` box for detailed information\n";
|
|
print $check_result_str;
|
|
return 1;
|
|
}
|
|
|
|
$check_result_str .= "> **DHCP WIRE TEST Successful** ($backend_str): Totalrun $casenum Passed $counts{pass} Failed $counts{fail}\n";
|
|
print $check_result_str;
|
|
return 0;
|
|
}
|
|
|
|
#--------------------------------------------------------
|
|
# Fuction name: run_fast_regression_test
|
|
# Description:
|
|
# Attributes:
|
|
# Return code:
|
|
#--------------------------------------------------------
|
|
sub run_fast_regression_test{
|
|
my $cmd = "sudo timeout 1200 env DEBIAN_FRONTEND=noninteractive apt-get -o Acquire::Retries=3 -o Acquire::http::Timeout=30 install -y xcat-test --allow-remove-essential --allow-unauthenticated";
|
|
my @output = runcmd("$cmd");
|
|
if($::RUNCMD_RC){
|
|
print RED "[run_fast_regression_test] $cmd ....[Failed]\n";
|
|
print Dumper \@output;
|
|
return 1;
|
|
}else{
|
|
print "[run_fast_regression_test] $cmd .....:\n";
|
|
print Dumper \@output;
|
|
}
|
|
|
|
$cmd = "sudo bash -c '. /etc/profile.d/xcat.sh && xcattest -h'";
|
|
@output = runcmd("$cmd");
|
|
if($::RUNCMD_RC){
|
|
print RED "[run_fast_regression_test] $cmd ....[Failed]\n";
|
|
print "[run_fast_regression_test] error dumper:\n";
|
|
print Dumper \@output;
|
|
return 1;
|
|
}else{
|
|
print "[run_fast_regression_test] $cmd .....:\n";
|
|
print Dumper \@output;
|
|
}
|
|
|
|
my $conf_file = write_regression_conf("regression.conf", 1);
|
|
|
|
$cmd = "sudo bash -c '. /etc/profile.d/xcat.sh && xcattest -s \"ci_test\" -l'";
|
|
my @caseslist = runcmd("$cmd");
|
|
if($::RUNCMD_RC){
|
|
print RED "[run_fast_regression_test] $cmd ....[Failed]\n";
|
|
print "[run_fast_regression_test] error dumper:\n";
|
|
print Dumper \@caseslist;
|
|
return 1;
|
|
}else{
|
|
print "[run_fast_regression_test] $cmd .....:\n";
|
|
print Dumper \@caseslist;
|
|
}
|
|
|
|
# The DHCP wire cases are not in this set: they reconfigure the DHCP server
|
|
# out from under whatever else is running, so they are labelled dhcp_wire
|
|
# instead and run as their own phase after this one. See run_dhcp_wire_test.
|
|
my $casenum = @caseslist;
|
|
my %counts = (pass => 0, fail => 0, failed => []);
|
|
run_cases($conf_file, \@caseslist, "", \%counts);
|
|
|
|
my $passnum = $counts{pass};
|
|
my $failnum = $counts{fail};
|
|
my @failcase = @{$counts{failed}};
|
|
|
|
if($failnum){
|
|
my $log_str = join (",", @failcase );
|
|
$check_result_str .= "> **FAST REGRESSION TEST Failed**: Totalcase $casenum Passed $passnum Failed $failnum FailedCases: $log_str. Please click ``Details`` label in ``Merge pull request`` box for detailed information\n";
|
|
print $check_result_str;
|
|
return 1;
|
|
}else{
|
|
$check_result_str .= "> **FAST REGRESSION TEST Successful**: Totalcase $casenum Passed $passnum Failed $failnum\n";
|
|
print $check_result_str;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
#--------------------------------------------------------
|
|
# Fuction name: mark_time
|
|
# Description:
|
|
# Attributes:
|
|
# Return code:
|
|
#--------------------------------------------------------
|
|
sub mark_time{
|
|
my $func_name=shift;
|
|
my $nowtime = timelocal(localtime());
|
|
my $nowtime_str = scalar(localtime());
|
|
my $duration = $nowtime - $last_func_start;
|
|
$last_func_start = $nowtime;
|
|
print "[mark_time] $nowtime_str, ElapsedTime of $func_name is $duration s\n";
|
|
}
|
|
|
|
#===============Main Process=============================
|
|
|
|
my @os_info = runcmd("cat /etc/os-release");
|
|
print "Current OS information:\n";
|
|
print Dumper \@os_info;
|
|
|
|
my @perl_vserion = runcmd("perl -v");
|
|
print "Current perl information:\n";
|
|
print Dumper \@perl_vserion;
|
|
|
|
my @disk = runcmd("df -h");
|
|
print "Disk information:\n";
|
|
print Dumper \@disk;
|
|
|
|
# Hacking the netmask. Not sure if we need to recover it after finish xcattest
|
|
# Note: Here has an assumption from Travis VM: only 1 UP Ethernet interface available (CHANGEME if it not as is)
|
|
my @intfinfo = runcmd("ip -o link |grep 'link/ether'|grep 'state UP' |awk -F ':' '{print \$2}'|head -1");
|
|
foreach my $nic (@intfinfo) {
|
|
print "Hacking the netmask length to 16 if it is 32: $nic\n";
|
|
runcmd("ip -4 addr show $nic|grep 'inet'|grep -q '/32' && sudo ip addr add \$(hostname -I|awk '{print \$1}')/16 dev $nic");
|
|
}
|
|
my @ipinfo = runcmd("ip addr");
|
|
print "Networking information:\n";
|
|
print Dumper \@ipinfo;
|
|
|
|
#Start to build xcat core
|
|
|
|
#Save the source tree before the build deletes it, the unit tests need it later
|
|
print GREEN "\n------ Preserving the source tree for the unit tests ------\n";
|
|
$rst = preserve_source_tree();
|
|
if($rst){
|
|
print RED "Preserving the source tree failed\n";
|
|
exit $rst;
|
|
}
|
|
mark_time("preserve_source_tree");
|
|
|
|
print GREEN "\n------ Building xCAT core package ------\n";
|
|
$rst = build_xcat_core();
|
|
if($rst){
|
|
print RED "Build of xCAT core package failed\n";
|
|
exit $rst;
|
|
}
|
|
mark_time("build_xcat_core");
|
|
|
|
#Start to install xcat
|
|
print GREEN "\n------Installing xCAT ------\n";
|
|
$rst = install_xcat();
|
|
if($rst){
|
|
print RED "Install of xCAT failed\n";
|
|
exit $rst;
|
|
}
|
|
mark_time("install_xcat");
|
|
|
|
#Run the xCAT-test unit tests. They need the perl dependencies xCAT pulls in
|
|
#(Net::DNS, XML::Simple) and a usable xCAT database, so they run after install.
|
|
print GREEN "\n------Running xCAT-test unit tests ------\n";
|
|
$rst = run_unit_tests();
|
|
if($rst){
|
|
print RED "Run of xCAT-test unit tests failed\n";
|
|
exit $rst;
|
|
}
|
|
mark_time("run_unit_tests");
|
|
|
|
#Run shell-script unit tests.
|
|
print GREEN "\n------Running xCAT-test BATS tests ------\n";
|
|
$rst = run_bats_tests();
|
|
if($rst){
|
|
print RED "Run of xCAT-test BATS tests failed\n";
|
|
exit $rst;
|
|
}
|
|
mark_time("run_bats_tests");
|
|
|
|
#Check the syntax of changing code
|
|
print GREEN "\n------ Checking the syntax of changed code------\n";
|
|
$rst = check_syntax();
|
|
if($rst){
|
|
print RED "Check syntax of changed code failed\n";
|
|
exit $rst;
|
|
}
|
|
mark_time("check_syntax");
|
|
|
|
#run fast regression
|
|
print GREEN "\n------Running fast regression test ------\n";
|
|
$rst = run_fast_regression_test();
|
|
if($rst){
|
|
print RED "Run of fast regression test failed\n";
|
|
exit $rst;
|
|
}
|
|
mark_time("run_fast_regression_test");
|
|
|
|
#Run the dhcptest offline checks -- unit tests plus `dhcptest validate` -- last:
|
|
#they exercise the source tree rather than the installed copy, and nothing else
|
|
#in the run depends on them.
|
|
print GREEN "\n------Running xCAT-test dhcptest checks ------\n";
|
|
$rst = run_dhcptest_unit_tests();
|
|
if($rst){
|
|
print RED "Run of xCAT-test dhcptest checks failed\n";
|
|
exit $rst;
|
|
}
|
|
mark_time("run_dhcptest_unit_tests");
|
|
|
|
#The DHCP wire cases go last, once per backend installed. They are the only
|
|
#phase that reconfigures the DHCP server and restarts the daemon, so nothing
|
|
#that follows them could be trusted to be running against the machine it was
|
|
#given -- and the ci_test set above must not be sharing a management node that
|
|
#is mid-reconfiguration, which is why they are not labelled ci_test.
|
|
print GREEN "\n------Running DHCP on-wire test on every backend ------\n";
|
|
$rst = run_dhcp_wire_test();
|
|
if($rst){
|
|
print RED "Run of DHCP on-wire test failed\n";
|
|
exit $rst;
|
|
}
|
|
mark_time("run_dhcp_wire_test");
|
|
|
|
exit 0;
|