gitlab ci: install binary deps faster (#33248)

* Fast Gitlab CI job setup, and better legibility * Use a non-broken, recent GNU Make
2022-10-26 09:19:24 +02:00 · 2022-10-26 09:19:24 +02:00 · a2520e80c0
commit a2520e80c0
parent d039744a5b
19 changed files with 139 additions and 38 deletions
--- a/lib/spack/spack/ci.py
+++ b/lib/spack/spack/ci.py
@ -17,7 +17,7 @@
 import time
 import zipfile

-from six import iteritems
+from six import iteritems, string_types
 from six.moves.urllib.error import HTTPError, URLError
 from six.moves.urllib.parse import urlencode
 from six.moves.urllib.request import HTTPHandler, Request, build_opener
@ -1936,26 +1936,35 @@ def reproduce_ci_job(url, work_dir):
    print("".join(inst_list))


-def process_command(cmd, cmd_args, repro_dir):
+def process_command(name, commands, repro_dir):
    """
    Create a script for and run the command. Copy the script to the
    reproducibility directory.

    Arguments:
-        cmd (str): name of the command being processed
-        cmd_args (list): string arguments to pass to the command
+        name (str): name of the command being processed
+        commands (list): list of arguments for single command or list of lists of
+            arguments for multiple commands. No shell escape is performed.
        repro_dir (str): Job reproducibility directory

    Returns: the exit code from processing the command
    """
-    tty.debug("spack {0} arguments: {1}".format(cmd, cmd_args))
+    tty.debug("spack {0} arguments: {1}".format(name, commands))
+
+    if len(commands) == 0 or isinstance(commands[0], string_types):
+        commands = [commands]
+
+    # Create a string [command 1] && [command 2] && ... && [command n] with commands
+    # quoted using double quotes.
+    args_to_string = lambda args: " ".join('"{}"'.format(arg) for arg in args)
+    full_command = " && ".join(map(args_to_string, commands))

    # Write the command to a shell script
-    script = "{0}.sh".format(cmd)
+    script = "{0}.sh".format(name)
    with open(script, "w") as fd:
-        fd.write("#!/bin/bash\n\n")
-        fd.write("\n# spack {0} command\n".format(cmd))
-        fd.write(" ".join(['"{0}"'.format(i) for i in cmd_args]))
+        fd.write("#!/bin/sh\n\n")
+        fd.write("\n# spack {0} command\n".format(name))
+        fd.write(full_command)
        fd.write("\n")

    st = os.stat(script)
@ -1967,15 +1976,15 @@ def process_command(cmd, cmd_args, repro_dir):
    # Run the generated install.sh shell script as if it were being run in
    # a login shell.
    try:
-        cmd_process = subprocess.Popen(["bash", "./{0}".format(script)])
+        cmd_process = subprocess.Popen(["/bin/sh", "./{0}".format(script)])
        cmd_process.wait()
        exit_code = cmd_process.returncode
    except (ValueError, subprocess.CalledProcessError, OSError) as err:
-        tty.error("Encountered error running {0} script".format(cmd))
+        tty.error("Encountered error running {0} script".format(name))
        tty.error(err)
        exit_code = 1

-    tty.debug("spack {0} exited {1}".format(cmd, exit_code))
+    tty.debug("spack {0} exited {1}".format(name, exit_code))
    return exit_code


--- a/lib/spack/spack/cmd/ci.py
+++ b/lib/spack/spack/cmd/ci.py
@ -25,7 +25,8 @@
 section = "build"
 level = "long"

-CI_REBUILD_INSTALL_BASE_ARGS = ["spack", "--color=always", "--backtrace", "--verbose"]
+SPACK_COMMAND = "spack"
+MAKE_COMMAND = "make"
 INSTALL_FAIL_CODE = 1


@ -509,41 +510,88 @@ def ci_rebuild(args):
    # No hash match anywhere means we need to rebuild spec

    # Start with spack arguments
-    install_args = [base_arg for base_arg in CI_REBUILD_INSTALL_BASE_ARGS]
+    spack_cmd = [SPACK_COMMAND, "--color=always", "--backtrace", "--verbose"]

    config = cfg.get("config")
    if not config["verify_ssl"]:
-        install_args.append("-k")
+        spack_cmd.append("-k")

-    install_args.extend(
-        [
-            "install",
-            "--keep-stage",
-            "--use-buildcache",
-            "dependencies:only,package:never",
-        ]
-    )
+    install_args = []

    can_verify = spack_ci.can_verify_binaries()
    verify_binaries = can_verify and spack_is_pr_pipeline is False
    if not verify_binaries:
        install_args.append("--no-check-signature")

+    cdash_args = []
    if cdash_handler:
        # Add additional arguments to `spack install` for CDash reporting.
-        install_args.extend(cdash_handler.args())
+        cdash_args.extend(cdash_handler.args())

-    # A compiler action of 'FIND_ANY' means we are building a bootstrap
-    # compiler or one of its deps.
-    # TODO: when compilers are dependencies, we should include --no-add
-    if compiler_action != "FIND_ANY":
-        install_args.append("--no-add")
+    slash_hash = "/{}".format(job_spec.dag_hash())
+    deps_install_args = install_args
+    root_install_args = install_args + [
+        "--no-add",
+        "--keep-stage",
+        "--only=package",
+        "--use-buildcache=package:never,dependencies:only",
+        slash_hash,
+    ]

-    # Identify spec to install by hash
-    install_args.append("/{0}".format(job_spec.dag_hash()))
+    # ["x", "y"] -> "'x' 'y'"
+    args_to_string = lambda args: " ".join("'{}'".format(arg) for arg in args)
+
+    commands = [
+        # apparently there's a race when spack bootstraps? do it up front once
+        [
+            SPACK_COMMAND,
+            "-e",
+            env.path,
+            "bootstrap",
+            "now",
+        ],
+        [
+            SPACK_COMMAND,
+            "-e",
+            env.path,
+            "config",
+            "add",
+            "config:db_lock_timeout:120",  # 2 minutes for processes to fight for a db lock
+        ],
+        [
+            SPACK_COMMAND,
+            "-e",
+            env.path,
+            "env",
+            "depfile",
+            "-o",
+            "Makefile",
+            "--use-buildcache=package:never,dependencies:only",
+            "--make-target-prefix",
+            "ci",
+            slash_hash,  # limit to spec we're building
+        ],
+        [
+            # --output-sync requires GNU make 4.x.
+            # Old make errors when you pass it a flag it doesn't recognize,
+            # but it doesn't error or warn when you set unrecognized flags in
+            # this variable.
+            "export",
+            "GNUMAKEFLAGS=--output-sync=recurse",
+        ],
+        [
+            MAKE_COMMAND,
+            "SPACK={}".format(args_to_string(spack_cmd)),
+            "SPACK_COLOR=always",
+            "SPACK_INSTALL_FLAGS={}".format(args_to_string(deps_install_args)),
+            "-j$(nproc)",
+            "ci/.install-deps/{}".format(job_spec.dag_hash()),
+        ],
+        spack_cmd + ["install"] + root_install_args,
+    ]

    tty.debug("Installing {0} from source".format(job_spec.name))
-    install_exit_code = spack_ci.process_command("install", install_args, repro_dir)
+    install_exit_code = spack_ci.process_command("install", commands, repro_dir)

    # Now do the post-install tasks
    tty.debug("spack install exited {0}".format(install_exit_code))
--- a/lib/spack/spack/test/cmd/ci.py
+++ b/lib/spack/spack/test/cmd/ci.py
@ -916,11 +916,8 @@ def test_ci_rebuild_mock_success(
    pkg_name = "archive-files"
    rebuild_env = create_rebuild_env(tmpdir, pkg_name, broken_tests)

-    monkeypatch.setattr(
-        spack.cmd.ci,
-        "CI_REBUILD_INSTALL_BASE_ARGS",
-        ["echo"],
-    )
+    monkeypatch.setattr(spack.cmd.ci, "SPACK_COMMAND", "echo")
+    monkeypatch.setattr(spack.cmd.ci, "MAKE_COMMAND", "echo")

    with rebuild_env.env_dir.as_cwd():
        activate_rebuild_env(tmpdir, pkg_name, rebuild_env)
@ -965,7 +962,8 @@ def test_ci_rebuild(

        ci_cmd("rebuild", "--tests", fail_on_error=False)

-    monkeypatch.setattr(spack.cmd.ci, "CI_REBUILD_INSTALL_BASE_ARGS", ["notcommand"])
+    monkeypatch.setattr(spack.cmd.ci, "SPACK_COMMAND", "notcommand")
+    monkeypatch.setattr(spack.cmd.ci, "MAKE_COMMAND", "notcommand")
    monkeypatch.setattr(spack.cmd.ci, "INSTALL_FAIL_CODE", 127)

    with rebuild_env.env_dir.as_cwd():
--- a/share/spack/gitlab/cloud_pipelines/stacks/aws-ahug-aarch64/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/aws-ahug-aarch64/spack.yaml
@ -241,6 +241,9 @@ spack:
  gitlab-ci:

    script:
+      - curl -Lfs https://github.com/JuliaBinaryWrappers/GNUMake_jll.jl/releases/download/GNUMake-v4.3.0%2B0/GNUMake.v4.3.0.aarch64-linux-gnu.tar.gz -o gmake.tar.gz
+      - printf 'deb1dcae0eecdc7fce2902f294012ab5519629e6827204f1b9964dcfd3f74627 gmake.tar.gz' | sha256sum --check --strict --quiet
+      - tar -xzf gmake.tar.gz -C /usr bin/make 2> /dev/null
      - . "./share/spack/setup-env.sh"
      - spack --version
      - cd ${SPACK_CONCRETE_ENV_DIR}
--- a/share/spack/gitlab/cloud_pipelines/stacks/aws-ahug/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/aws-ahug/spack.yaml
@ -240,6 +240,9 @@ spack:
  gitlab-ci:

    script:
+      - curl -Lfs https://github.com/JuliaBinaryWrappers/GNUMake_jll.jl/releases/download/GNUMake-v4.3.0%2B0/GNUMake.v4.3.0.x86_64-linux-gnu.tar.gz -o gmake.tar.gz
+      - printf 'b019c4aa757503d442c906047f6b1c393bf8eeba71260d455537cfc708862249 gmake.tar.gz' | sha256sum --check --strict --quiet
+      - tar -xzf gmake.tar.gz -C /usr bin/make 2> /dev/null
      - . "./share/spack/setup-env.sh"
      - spack --version
      - cd ${SPACK_CONCRETE_ENV_DIR}
--- a/share/spack/gitlab/cloud_pipelines/stacks/aws-isc-aarch64/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/aws-isc-aarch64/spack.yaml
@ -148,6 +148,9 @@ spack:
  gitlab-ci:

    script:
+      - curl -Lfs https://github.com/JuliaBinaryWrappers/GNUMake_jll.jl/releases/download/GNUMake-v4.3.0%2B0/GNUMake.v4.3.0.aarch64-linux-gnu.tar.gz -o gmake.tar.gz
+      - printf 'deb1dcae0eecdc7fce2902f294012ab5519629e6827204f1b9964dcfd3f74627 gmake.tar.gz' | sha256sum --check --strict --quiet
+      - tar -xzf gmake.tar.gz -C /usr bin/make 2> /dev/null
      - . "./share/spack/setup-env.sh"
      - spack --version
      - cd ${SPACK_CONCRETE_ENV_DIR}
--- a/share/spack/gitlab/cloud_pipelines/stacks/aws-isc/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/aws-isc/spack.yaml
@ -159,6 +159,9 @@ spack:
  gitlab-ci:

    script:
+      - curl -Lfs https://github.com/JuliaBinaryWrappers/GNUMake_jll.jl/releases/download/GNUMake-v4.3.0%2B0/GNUMake.v4.3.0.x86_64-linux-gnu.tar.gz -o gmake.tar.gz
+      - printf 'b019c4aa757503d442c906047f6b1c393bf8eeba71260d455537cfc708862249  gmake.tar.gz' | sha256sum --check --strict --quiet
+      - tar -xzf gmake.tar.gz -C /usr bin/make 2> /dev/null
      - . "./share/spack/setup-env.sh"
      - spack --version
      - cd ${SPACK_CONCRETE_ENV_DIR}
--- a/share/spack/gitlab/cloud_pipelines/stacks/build_systems/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/build_systems/spack.yaml
@ -32,6 +32,9 @@ spack:

  gitlab-ci:
    script:
+      - curl -Lfs https://github.com/JuliaBinaryWrappers/GNUMake_jll.jl/releases/download/GNUMake-v4.3.0%2B0/GNUMake.v4.3.0.x86_64-linux-gnu.tar.gz -o gmake.tar.gz
+      - printf 'b019c4aa757503d442c906047f6b1c393bf8eeba71260d455537cfc708862249  gmake.tar.gz' | sha256sum --check --strict --quiet
+      - tar -xzf gmake.tar.gz -C /usr bin/make 2> /dev/null
      - . "./share/spack/setup-env.sh"
      - spack --version
      - cd ${SPACK_CONCRETE_ENV_DIR}
--- a/share/spack/gitlab/cloud_pipelines/stacks/data-vis-sdk/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/data-vis-sdk/spack.yaml
@ -47,6 +47,9 @@ spack:
  gitlab-ci:
    image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] }
    script:
+      - curl -Lfs https://github.com/JuliaBinaryWrappers/GNUMake_jll.jl/releases/download/GNUMake-v4.3.0%2B0/GNUMake.v4.3.0.x86_64-linux-gnu.tar.gz -o gmake.tar.gz
+      - printf 'b019c4aa757503d442c906047f6b1c393bf8eeba71260d455537cfc708862249  gmake.tar.gz' | sha256sum --check --strict --quiet
+      - tar -xzf gmake.tar.gz -C /usr bin/make 2> /dev/null
      - . "./share/spack/setup-env.sh"
      - spack --version
      - cd ${SPACK_CONCRETE_ENV_DIR}
--- a/share/spack/gitlab/cloud_pipelines/stacks/e4s-on-power/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-on-power/spack.yaml
@ -214,6 +214,7 @@ spack:
  gitlab-ci:

    script:
+      - curl -Lfs https://github.com/JuliaBinaryWrappers/GNUMake_jll.jl/releases/download/GNUMake-v4.3.0%2B0/GNUMake.v4.3.0.powerpc64le-linux-gnu.tar.gz | tar -xzf - -C /usr 2> /dev/null
      - . "./share/spack/setup-env.sh"
      - spack --version
      - cd ${SPACK_CONCRETE_ENV_DIR}
--- a/share/spack/gitlab/cloud_pipelines/stacks/e4s-oneapi/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-oneapi/spack.yaml
@ -266,6 +266,9 @@ spack:
  gitlab-ci:

    script:
+      - curl -Lfs https://github.com/JuliaBinaryWrappers/GNUMake_jll.jl/releases/download/GNUMake-v4.3.0%2B0/GNUMake.v4.3.0.x86_64-linux-gnu.tar.gz -o gmake.tar.gz
+      - printf 'b019c4aa757503d442c906047f6b1c393bf8eeba71260d455537cfc708862249  gmake.tar.gz' | sha256sum --check --strict --quiet
+      - tar -xzf gmake.tar.gz -C /usr bin/make 2> /dev/null
      - . "./share/spack/setup-env.sh"
      - spack --version
      - cd ${SPACK_CONCRETE_ENV_DIR}
--- a/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml
@ -253,6 +253,9 @@ spack:
  gitlab-ci:

    script:
+      - curl -Lfs https://github.com/JuliaBinaryWrappers/GNUMake_jll.jl/releases/download/GNUMake-v4.3.0%2B0/GNUMake.v4.3.0.x86_64-linux-gnu.tar.gz -o gmake.tar.gz
+      - printf 'b019c4aa757503d442c906047f6b1c393bf8eeba71260d455537cfc708862249  gmake.tar.gz' | sha256sum --check --strict --quiet
+      - tar -xzf gmake.tar.gz -C /usr bin/make 2> /dev/null
      - . "./share/spack/setup-env.sh"
      - spack --version
      - cd ${SPACK_CONCRETE_ENV_DIR}
--- a/share/spack/gitlab/cloud_pipelines/stacks/ml-cpu/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/ml-cpu/spack.yaml
@ -87,6 +87,9 @@ spack:

  gitlab-ci:
    script:
+      - curl -Lfs https://github.com/JuliaBinaryWrappers/GNUMake_jll.jl/releases/download/GNUMake-v4.3.0%2B0/GNUMake.v4.3.0.x86_64-linux-gnu.tar.gz -o gmake.tar.gz
+      - printf 'b019c4aa757503d442c906047f6b1c393bf8eeba71260d455537cfc708862249  gmake.tar.gz' | sha256sum --check --strict --quiet
+      - tar -xzf gmake.tar.gz -C /usr bin/make 2> /dev/null
      - . "./share/spack/setup-env.sh"
      - spack --version
      - cd ${SPACK_CONCRETE_ENV_DIR}
--- a/share/spack/gitlab/cloud_pipelines/stacks/ml-cuda/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/ml-cuda/spack.yaml
@ -90,6 +90,9 @@ spack:

  gitlab-ci:
    script:
+      - curl -Lfs https://github.com/JuliaBinaryWrappers/GNUMake_jll.jl/releases/download/GNUMake-v4.3.0%2B0/GNUMake.v4.3.0.x86_64-linux-gnu.tar.gz -o gmake.tar.gz
+      - printf 'b019c4aa757503d442c906047f6b1c393bf8eeba71260d455537cfc708862249  gmake.tar.gz' | sha256sum --check --strict --quiet
+      - tar -xzf gmake.tar.gz -C /usr bin/make 2> /dev/null
      - . "./share/spack/setup-env.sh"
      - spack --version
      - cd ${SPACK_CONCRETE_ENV_DIR}
--- a/share/spack/gitlab/cloud_pipelines/stacks/ml-rocm/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/ml-rocm/spack.yaml
@ -93,6 +93,9 @@ spack:

  gitlab-ci:
    script:
+      - curl -Lfs https://github.com/JuliaBinaryWrappers/GNUMake_jll.jl/releases/download/GNUMake-v4.3.0%2B0/GNUMake.v4.3.0.x86_64-linux-gnu.tar.gz -o gmake.tar.gz
+      - printf 'b019c4aa757503d442c906047f6b1c393bf8eeba71260d455537cfc708862249  gmake.tar.gz' | sha256sum --check --strict --quiet
+      - tar -xzf gmake.tar.gz -C /usr bin/make 2> /dev/null
      - . "./share/spack/setup-env.sh"
      - spack --version
      - cd ${SPACK_CONCRETE_ENV_DIR}
--- a/share/spack/gitlab/cloud_pipelines/stacks/radiuss-aws-aarch64/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/radiuss-aws-aarch64/spack.yaml
@ -56,6 +56,9 @@ spack:
  gitlab-ci:

    script:
+      - curl -Lfs https://github.com/JuliaBinaryWrappers/GNUMake_jll.jl/releases/download/GNUMake-v4.3.0%2B0/GNUMake.v4.3.0.aarch64-linux-gnu.tar.gz -o gmake.tar.gz
+      - printf 'deb1dcae0eecdc7fce2902f294012ab5519629e6827204f1b9964dcfd3f74627 gmake.tar.gz' | sha256sum --check --strict --quiet
+      - tar -xzf gmake.tar.gz -C /usr bin/make 2> /dev/null
      - . "./share/spack/setup-env.sh"
      - spack --version
      - cd ${SPACK_CONCRETE_ENV_DIR}
--- a/share/spack/gitlab/cloud_pipelines/stacks/radiuss-aws/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/radiuss-aws/spack.yaml
@ -61,6 +61,9 @@ spack:
  gitlab-ci:

    script:
+      - curl -Lfs https://github.com/JuliaBinaryWrappers/GNUMake_jll.jl/releases/download/GNUMake-v4.3.0%2B0/GNUMake.v4.3.0.x86_64-linux-gnu.tar.gz -o gmake.tar.gz
+      - printf 'b019c4aa757503d442c906047f6b1c393bf8eeba71260d455537cfc708862249  gmake.tar.gz' | sha256sum --check --strict --quiet
+      - tar -xzf gmake.tar.gz -C /usr bin/make 2> /dev/null
      - . "./share/spack/setup-env.sh"
      - spack --version
      - cd ${SPACK_CONCRETE_ENV_DIR}
--- a/share/spack/gitlab/cloud_pipelines/stacks/radiuss/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/radiuss/spack.yaml
@ -64,6 +64,9 @@ spack:
  gitlab-ci:
    image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] }
    script:
+      - curl -Lfs https://github.com/JuliaBinaryWrappers/GNUMake_jll.jl/releases/download/GNUMake-v4.3.0%2B0/GNUMake.v4.3.0.x86_64-linux-gnu.tar.gz -o gmake.tar.gz
+      - printf 'b019c4aa757503d442c906047f6b1c393bf8eeba71260d455537cfc708862249  gmake.tar.gz' | sha256sum --check --strict --quiet
+      - tar -xzf gmake.tar.gz -C /usr bin/make 2> /dev/null
      - . "./share/spack/setup-env.sh"
      - spack --version
      - cd ${SPACK_CONCRETE_ENV_DIR}
--- a/share/spack/gitlab/cloud_pipelines/stacks/tutorial/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/tutorial/spack.yaml
@ -63,6 +63,9 @@ spack:

  gitlab-ci:
    script:
+      - curl -Lfs https://github.com/JuliaBinaryWrappers/GNUMake_jll.jl/releases/download/GNUMake-v4.3.0%2B0/GNUMake.v4.3.0.x86_64-linux-gnu.tar.gz -o gmake.tar.gz
+      - printf 'b019c4aa757503d442c906047f6b1c393bf8eeba71260d455537cfc708862249  gmake.tar.gz' | sha256sum --check --strict --quiet
+      - tar -xzf gmake.tar.gz -C /usr bin/make 2> /dev/null
      - . "./share/spack/setup-env.sh"
      - spack --version
      - spack compiler find