gitlab ci: Better tagging of "service" jobs (#36846)

- Tag non-rebuild jobs to target a cheaper (and more highly available)
subset of runners.

- Add missing resource requests to these jobs as well.

lib/spack/spack/test/cmd/ci.py

@@ -2411,3 +2411,53 @@ def test_gitlab_ci_update(
             ci_root = yaml_contents["spack"]["ci"]
 
             assert "pipeline-gen" in ci_root
+
+
+def test_gitlab_config_scopes(
+    tmpdir, working_env, mutable_mock_env_path, mock_packages, ci_base_environment
+):
+    """Test pipeline generation with real configs included"""
+    configs_path = os.path.join(spack_paths.share_path, "gitlab", "cloud_pipelines", "configs")
+    filename = str(tmpdir.join("spack.yaml"))
+    with open(filename, "w") as f:
+        f.write(
+            """\
+spack:
+  config:
+    install_tree: {0}
+  include: [{1}]
+  view: false
+  specs:
+    - flatten-deps
+  mirrors:
+    some-mirror: https://my.fake.mirror
+  ci:
+    pipeline-gen:
+    - build-job:
+        image: "ecpe4s/ubuntu20.04-runner-x86_64:2023-01-01"
+        tags: ["some_tag"]
+""".format(
+                tmpdir.strpath, configs_path
+            )
+        )
+
+    with tmpdir.as_cwd():
+        env_cmd("create", "test", "./spack.yaml")
+        outputfile = str(tmpdir.join(".gitlab-ci.yml"))
+
+        with ev.read("test"):
+            ci_cmd("generate", "--output-file", outputfile)
+
+        with open(outputfile) as f:
+            contents = f.read()
+            yaml_contents = syaml.load(contents)
+
+            assert "rebuild-index" in yaml_contents
+            rebuild_job = yaml_contents["rebuild-index"]
+            assert "tags" in rebuild_job
+            assert "variables" in rebuild_job
+            rebuild_tags = rebuild_job["tags"]
+            rebuild_vars = rebuild_job["variables"]
+            assert all([t in rebuild_tags for t in ["spack", "service"]])
+            expected_vars = ["CI_JOB_SIZE", "KUBERNETES_CPU_REQUEST", "KUBERNETES_MEMORY_REQUEST"]
+            assert all([v in rebuild_vars for v in expected_vars])

share/spack/gitlab/cloud_pipelines/configs/ci.yaml

@@ -36,6 +36,27 @@ ci:
         - aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache
         - aws s3 cp /tmp/public_keys ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/_pgp --recursive --exclude "*" --include "*.pub"
 
+  - reindex-job:
+      tags: ["service"]
+      variables:
+        CI_JOB_SIZE: "medium"
+        KUBERNETES_CPU_REQUEST: "4000m"
+        KUBERNETES_MEMORY_REQUEST: "16G"
+
+  - cleanup-job:
+      tags: ["service"]
+      variables:
+        CI_JOB_SIZE: "small"
+        KUBERNETES_CPU_REQUEST: "500m"
+        KUBERNETES_MEMORY_REQUEST: "500M"
+
+  - noop-job:
+      tags: ["service"]
+      variables:
+        CI_JOB_SIZE: "small"
+        KUBERNETES_CPU_REQUEST: "500m"
+        KUBERNETES_MEMORY_REQUEST: "500M"
+
   - any-job:
       image: "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18"
       tags: ["spack"]