From 3530e44c02f88327b607aac5e8c68c0519b731da Mon Sep 17 00:00:00 2001 From: Harmen Stoppels Date: Thu, 19 Sep 2024 16:06:44 +0200 Subject: [PATCH] url join: fix oci scheme (#46483) * url.py: also special case oci scheme in join * avoid fetching keys from oci mirror --- lib/spack/spack/binary_distribution.py | 3 +++ lib/spack/spack/test/util/util_url.py | 2 +- lib/spack/spack/util/url.py | 6 +++--- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/binary_distribution.py b/lib/spack/spack/binary_distribution.py index 727acfdfb8..236746427e 100644 --- a/lib/spack/spack/binary_distribution.py +++ b/lib/spack/spack/binary_distribution.py @@ -2380,6 +2380,9 @@ def get_keys(install=False, trust=False, force=False, mirrors=None): for mirror in mirror_collection.values(): fetch_url = mirror.fetch_url + # TODO: oci:// does not support signing. + if fetch_url.startswith("oci://"): + continue keys_url = url_util.join( fetch_url, BUILD_CACHE_RELATIVE_PATH, BUILD_CACHE_KEYS_RELATIVE_PATH ) diff --git a/lib/spack/spack/test/util/util_url.py b/lib/spack/spack/test/util/util_url.py index 0929f98f71..1e698932ce 100644 --- a/lib/spack/spack/test/util/util_url.py +++ b/lib/spack/spack/test/util/util_url.py @@ -48,7 +48,7 @@ def test_relative_path_to_file_url(tmpdir): @pytest.mark.parametrize("resolve_href", [True, False]) -@pytest.mark.parametrize("scheme", ["http", "s3", "gs", "file"]) +@pytest.mark.parametrize("scheme", ["http", "s3", "gs", "file", "oci"]) def test_url_join_absolute(scheme, resolve_href): """Test that joining a URL with an absolute path works the same for schemes we care about, and whether we work in web browser mode or not.""" diff --git a/lib/spack/spack/util/url.py b/lib/spack/spack/util/url.py index 9337b103c8..e0fd8c18c0 100644 --- a/lib/spack/spack/util/url.py +++ b/lib/spack/spack/util/url.py @@ -79,7 +79,7 @@ def join(base: str, *components: str, resolve_href: bool = False, **kwargs) -> s 1. By default resolve_href=False, which makes the function like os.path.join: for example https://example.com/a/b + c/d = https://example.com/a/b/c/d. If resolve_href=True, the behavior is how a browser would resolve the URL: https://example.com/a/c/d. - 2. s3:// and gs:// URLs are joined like http:// URLs. + 2. s3://, gs://, oci:// URLs are joined like http:// URLs. 3. It accepts multiple components for convenience. Note that components[1:] are treated as literal path components and appended to components[0] separated by slashes.""" # Ensure a trailing slash in the path component of the base URL to get os.path.join-like @@ -93,8 +93,8 @@ def join(base: str, *components: str, resolve_href: bool = False, **kwargs) -> s try: # NOTE: we temporarily modify urllib internals so s3 and gs schemes are treated like http. # This is non-portable, and may be forward incompatible with future cpython versions. - urllib.parse.uses_netloc = [*uses_netloc, "s3", "gs"] - urllib.parse.uses_relative = [*uses_relative, "s3", "gs"] + urllib.parse.uses_netloc = [*uses_netloc, "s3", "gs", "oci"] + urllib.parse.uses_relative = [*uses_relative, "s3", "gs", "oci"] return urllib.parse.urljoin(base, "/".join(components), **kwargs) finally: urllib.parse.uses_netloc = uses_netloc