Merge pull request #81 from VersatusHPC/fix/openeuler-mock-dnf-minrate

fix(xcat-dep): every openEuler mock --init fails on a slow repo.openeuler.org

mock-configs/templates/openeuler-lts-xcat.tpl

@@ -7,6 +7,7 @@ keepcache=1
 reposdir=/dev/null
 logfile=/var/log/dnf.log
 retries=20
+timeout=300
 obsoletes=1
 gpgcheck=1
 assumeyes=1
@@ -16,16 +17,25 @@ install_weak_deps=0
 skip_if_unavailable=0
 protected_packages=
 """
+_openeuler_mirrors = ('https://mirrors.tuna.tsinghua.edu.cn/openeuler',
+                      'https://mirror.nju.edu.cn/openeuler',
+                      'https://repo.openeuler.org')
 _openeuler_root = 'https://repo.openeuler.org/openEuler-' + config_opts['openeuler_repository_release']
 _openeuler_arch = config_opts['target_arch']
 for _openeuler_repo in config_opts['openeuler_repositories']:
+    _openeuler_baseurls = ''
+    for _openeuler_mirror in _openeuler_mirrors:
+        _openeuler_baseurls += '{0}/openEuler-{1}/{2}/{3}/ '.format(
+            _openeuler_mirror, config_opts['openeuler_repository_release'],
+            _openeuler_repo, _openeuler_arch)
     config_opts['dnf.conf'] += """
 [{repo}]
 name=openEuler {release} {repo}
-baseurl={root}/{repo}/{arch}/
+baseurl={baseurls}
 enabled=1
 gpgcheck=1
 gpgkey={root}/OS/{arch}/RPM-GPG-KEY-openEuler
 skip_if_unavailable=0
 """.format(repo=_openeuler_repo, release=config_opts['openeuler_repository_release'],
-           root=_openeuler_root, arch=_openeuler_arch)
+           root=_openeuler_root, arch=_openeuler_arch,
+           baseurls=_openeuler_baseurls.strip())

native/fixtures/mock-configs.py

@@ -10,12 +10,21 @@ source = Path(sys.argv[1]).resolve()
 with tempfile.TemporaryDirectory() as directory:
     config_path = Path(directory)
     (config_path / 'templates').mkdir()
+    absent = []
     for parent in ('openeuler-20.03-sp4.tpl', 'openeuler-22.03-sp4.tpl', 'openeuler-24.03.tpl'):
-        shutil.copyfile(Path('/etc/mock/templates') / parent, config_path / 'templates' / parent)
+        source_template = Path('/etc/mock/templates') / parent
+        if not source_template.is_file():
+            absent.append(parent)
+            continue
+        shutil.copyfile(source_template, config_path / 'templates' / parent)
     shutil.copyfile(source / 'templates/openeuler-lts-xcat.tpl', config_path / 'templates/openeuler-lts-xcat.tpl')
-    result = {}
+    result = {'_absent_templates': absent}
     for wrapper in sorted(source.glob('openeuler-*.cfg')):
-        config = load_config(str(config_path), str(wrapper))
+        try:
+            config = load_config(str(config_path), str(wrapper))
+        except Exception as why:
+            result[wrapper.stem] = {'error': str(why)}
+            continue
         repos = configparser.ConfigParser(interpolation=None)
         repos.read_string(config['dnf.conf'])
         result[wrapper.stem] = {key: config[key] for key in ('root', 'target_arch', 'legal_host_arches', 'releasever', 'dist', 'use_bootstrap_image')}

native/mock-configs.t

@@ -21,10 +21,21 @@ open(my $pipe, '-|', 'python3', "$RealBin/fixtures/mock-configs.py", "$RealBin/.
 my $json = do {local $/; <$pipe>};
 close($pipe) or die "native mock config loader failed: $?";
 my $configs = decode_json($json);
+my $absent = delete $configs->{_absent_templates} || [];
+diag("base template absent on this host, its configs are skipped: @$absent") if @$absent;
 for my $cell (@cells) {
     my ($version, $release, $releasever, $arch) = @$cell;
     my $target = "openeuler-$version-$arch";
     my $config = $configs->{$target};
+    if ($config->{error}) {
+        my ($missing) = $config->{error} =~ m{templates/([^/\s]+\.tpl)};
+        if (defined($missing) && grep { $_ eq $missing } @$absent) {
+            note("$target skipped: $missing is not installed on this host");
+            next;
+        }
+        fail("$target loads: $config->{error}");
+        next;
+    }
     is($config->{root}, $target, "$target selects its own buildroot");
     is($config->{target_arch}, $arch, "$target selects its native architecture");
     is_deeply($config->{legal_host_arches}, [$arch], "$target requires a native host");
@@ -35,8 +46,18 @@ for my $cell (@cells) {
     my @names = $arch eq 'ppc64le' ? ('OS') : ('OS', 'everything', 'update');
     is_deeply([sort grep {$_ ne 'main'} keys %$repos], [sort @names], "$target selects only published native repositories");
     is($repos->{main}{gpgcheck}, '1', "$target requires native package signatures");
+    ok(!exists $repos->{main}{minrate}, "$target keeps dnf's rate abort, which is what tries a mirror");
+    cmp_ok($repos->{main}{timeout}, '>=', 300, "$target waits before it abandons a slow host");
     my $base = "https://repo.openeuler.org/openEuler-$release";
-    is_deeply([map {$repos->{$_}{baseurl}} @names], [map {"$base/$_/$arch/"} @names], "$target pins repository URLs to its exact release");
+    for my $repo (@names) {
+        my @urls = split ' ', $repos->{$repo}{baseurl};
+        isnt($urls[0], "$base/$repo/$arch/", "$target does not read $repo from the slowest host first");
+        ok(scalar(grep { $_ eq "$base/$repo/$arch/" } @urls),
+            "$target still carries the canonical $repo url");
+        cmp_ok(scalar @urls, '>', 1, "$target has a fallback for $repo");
+        ok(!grep({ $_ !~ m{/openEuler-\Q$release\E/\Q$repo\E/\Q$arch\E/$} } @urls),
+            "$target pins every $repo url to its exact release and architecture");
+    }
     is_deeply([map {$repos->{$_}{gpgkey}} @names], [map {"$base/OS/$arch/RPM-GPG-KEY-openEuler"} @names], "$target uses the release signing key");
     ok(!grep({$repos->{$_}{gpgcheck} ne '1' || $repos->{$_}{skip_if_unavailable} ne '0'} @names), "$target fails on unsigned packages or unavailable repositories");
 }