Gitlab CI: Windows Configs (#43967)

Add support for Gitlab CI on Windows

This PR adds the config changes required to configure and execute
Gitlab pipelines running Windows builds on Windows runners using
the existing Gitlab CI infrastructure (and newly added Windows 
infrastructure).

* Adds support for generating child pipelines dispatched to Windows runners
* Refactors the relevant pre-scripts, scripts, and post scripts to be compatible with Windows
* Adds Windows config section describing Windows jobs
* Adds VTK as Windows build stack (to be expanded later)
* Modifies proj to build on Windows
* Refactors Windows rpath symlinking to avoid system libs and externals

---------

Co-authored-by: Ryan Krattiger <ryan.krattiger@kitware.com>
Co-authored-by: Mike VanDenburgh <michael.vandenburgh@kitware.com>
Co-authored-by: Todd Gamblin <tgamblin@llnl.gov>
Co-authored-by: Scott Wittenburg <scott.wittenburg@kitware.com>

bin/spack.ps1

@@ -144,3 +144,5 @@ switch($SpackSubCommand)
     "unload" {Invoke-SpackLoad}
     default  {python "$Env:SPACK_ROOT/bin/spack" $SpackCMD_params $SpackSubCommand $SpackSubCommandArgs}
 }
+
+exit $LASTEXITCODE

lib/spack/llnl/util/filesystem.py

@@ -2510,6 +2510,12 @@ def establish_link(self):
 
         # for each binary install dir in self.pkg (i.e. pkg.prefix.bin, pkg.prefix.lib)
         # install a symlink to each dependent library
+
+        # do not rpath for system libraries included in the dag
+        # we should not be modifying libraries managed by the Windows system
+        # as this will negatively impact linker behavior and can result in permission
+        # errors if those system libs are not modifiable by Spack
+        if "windows-system" not in getattr(self.pkg, "tags", []):
             for library, lib_dir in itertools.product(self.rpaths, self.library_dependents):
                 self._link(library, lib_dir)
 

lib/spack/spack/ci.py

@@ -684,6 +684,22 @@ def generate_gitlab_ci_yaml(
             "instead.",
         )
 
+    def ensure_expected_target_path(path):
+        """Returns passed paths with all Windows path separators exchanged
+        for posix separators only if copy_only_pipeline is enabled
+
+        This is required as copy_only_pipelines are a unique scenario where
+        the generate job and child pipelines are run on different platforms.
+        To make this compatible w/ Windows, we cannot write Windows style path separators
+        that will be consumed on by the Posix copy job runner.
+
+        TODO (johnwparent): Refactor config + cli read/write to deal only in posix
+        style paths
+        """
+        if copy_only_pipeline and path:
+            path = path.replace("\\", "/")
+        return path
+
     pipeline_mirrors = spack.mirror.MirrorCollection(binary=True)
     deprecated_mirror_config = False
     buildcache_destination = None
@@ -807,7 +823,7 @@ def generate_gitlab_ci_yaml(
             if scope not in include_scopes and scope not in env_includes:
                 include_scopes.insert(0, scope)
         env_includes.extend(include_scopes)
-        env_yaml_root["spack"]["include"] = env_includes
+        env_yaml_root["spack"]["include"] = [ensure_expected_target_path(i) for i in env_includes]
 
         if "gitlab-ci" in env_yaml_root["spack"] and "ci" not in env_yaml_root["spack"]:
             env_yaml_root["spack"]["ci"] = env_yaml_root["spack"].pop("gitlab-ci")
@@ -1228,6 +1244,9 @@ def main_script_replacements(cmd):
             "SPACK_REBUILD_EVERYTHING": str(rebuild_everything),
             "SPACK_REQUIRE_SIGNING": os.environ.get("SPACK_REQUIRE_SIGNING", "False"),
         }
+        output_vars = output_object["variables"]
+        for item, val in output_vars.items():
+            output_vars[item] = ensure_expected_target_path(val)
 
         # TODO: Remove this block in Spack 0.23
         if deprecated_mirror_config and remote_mirror_override:
@@ -1284,7 +1303,6 @@ def main_script_replacements(cmd):
     sorted_output = {}
     for output_key, output_value in sorted(output_object.items()):
         sorted_output[output_key] = output_value
-
     if known_broken_specs_encountered:
         tty.error("This pipeline generated hashes known to be broken on develop:")
         display_broken_spec_messages(broken_specs_url, known_broken_specs_encountered)

lib/spack/spack/package_base.py

@@ -161,7 +161,11 @@ def windows_establish_runtime_linkage(self):
 
         Performs symlinking to incorporate rpath dependencies to Windows runtime search paths
         """
-        if sys.platform == "win32":
+        # If spec is an external, we should not be modifying its bin directory, as we would
+        # be doing in this method
+        # Spack should in general not modify things it has not installed
+        # we can reasonably expect externals to have their link interface properly established
+        if sys.platform == "win32" and not self.spec.external:
             self.win_rpath.add_library_dependent(*self.win_add_library_dependent())
             self.win_rpath.add_rpath(*self.win_add_rpath())
             self.win_rpath.establish_link()
@@ -2446,9 +2450,18 @@ def rpath(self):
 
         # on Windows, libraries of runtime interest are typically
         # stored in the bin directory
+        # Do not include Windows system libraries in the rpath interface
+        # these libraries are handled automatically by VS/VCVARS and adding
+        # Spack derived system libs into the link path or address space of a program
+        # can result in conflicting versions, which makes Spack packages less useable
         if sys.platform == "win32":
             rpaths = [self.prefix.bin]
-            rpaths.extend(d.prefix.bin for d in deps if os.path.isdir(d.prefix.bin))
+            rpaths.extend(
+                d.prefix.bin
+                for d in deps
+                if os.path.isdir(d.prefix.bin)
+                and "windows-system" not in getattr(d.package, "tags", [])
+            )
         else:
             rpaths = [self.prefix.lib, self.prefix.lib64]
             rpaths.extend(d.prefix.lib for d in deps if os.path.isdir(d.prefix.lib))

share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml

@@ -64,6 +64,11 @@ default:
     SPACK_TARGET_PLATFORM: "linux"
     SPACK_TARGET_ARCH: "ppc64le"
 
+.win64-msvc2019:
+  variables:
+    SPACK_TARGET_PLATFORM: "win64"
+    SPACK_TARGET_ARCH: "x86_64"
+
 ########################################
 # Job templates
 ########################################
@@ -72,6 +77,8 @@ default:
     PIPELINE_MIRROR_TEMPLATE: "single-src-protected-mirrors.yaml.in"
     # TODO: We can remove this when we drop the "deprecated" stack
     PUSH_BUILDCACHE_DEPRECATED: "${PROTECTED_MIRROR_PUSH_DOMAIN}/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}"
+    SPACK_CI_CONFIG_ROOT: "${CI_PROJECT_DIR}/share/spack/gitlab/cloud_pipelines/configs"
+    SPACK_CI_SCRIPTS_ROOT: "${CI_PROJECT_DIR}/share/spack/gitlab/cloud_pipelines/scripts"
 
   rules:
     - if: $SPACK_CI_DISABLE_STACKS =~ /.+/ && $SPACK_CI_STACK_NAME =~ $SPACK_CI_DISABLE_STACKS
@@ -114,16 +121,8 @@ default:
 .generate-common:
   stage: generate
   script:
-    - uname -a || true
-    - grep -E 'vendor|model name' /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true
-    - nproc || true
-    - cat /proc/loadavg || true
-    - cat /proc/meminfo | grep 'MemTotal\|MemFree' || true
-    - . "./share/spack/setup-env.sh"
     - spack --version
-    - cd share/spack/gitlab/cloud_pipelines/stacks/${SPACK_CI_STACK_NAME}
-    - spack env activate --without-view .
-    - export SPACK_CI_CONFIG_ROOT="${SPACK_ROOT}/share/spack/gitlab/cloud_pipelines/configs"
+    - spack env activate --without-view share/spack/gitlab/cloud_pipelines/stacks/${SPACK_CI_STACK_NAME}
     - spack
       --config-scope "${SPACK_CI_CONFIG_ROOT}"
       --config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}"
@@ -134,29 +133,25 @@ default:
       --config-scope "${SPACK_CI_CONFIG_ROOT}"
       --config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}"
       --config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}/${SPACK_TARGET_ARCH}"
-      ${CI_STACK_CONFIG_SCOPES}
       audit configs
-    - spack python -c "import os,sys; print(os.path.expandvars(sys.stdin.read()))"
-      < "${SPACK_CI_CONFIG_ROOT}/${PIPELINE_MIRROR_TEMPLATE}" > "${SPACK_CI_CONFIG_ROOT}/mirrors.yaml"
+    # Command below needs to be `spack python` due to naming differences accross platforms
+    - spack python ${SPACK_CI_SCRIPTS_ROOT}/common/expand_vars.py
+      "${SPACK_CI_CONFIG_ROOT}/${PIPELINE_MIRROR_TEMPLATE}"
+      "${SPACK_CI_CONFIG_ROOT}/mirrors.yaml"
     - spack config add -f "${SPACK_CI_CONFIG_ROOT}/mirrors.yaml"
-    - mkdir -p "${CI_PROJECT_DIR}/jobs_scratch_dir"
+    - mkdir "${CI_PROJECT_DIR}/jobs_scratch_dir"
     - spack
       --config-scope "${SPACK_CI_CONFIG_ROOT}"
       --config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}"
       --config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}/${SPACK_TARGET_ARCH}"
-      ${CI_STACK_CONFIG_SCOPES}
       config blame > "${CI_PROJECT_DIR}/jobs_scratch_dir/spack.yaml.blame"
     - spack -v  --color=always
       --config-scope "${SPACK_CI_CONFIG_ROOT}"
       --config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}"
       --config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}/${SPACK_TARGET_ARCH}"
-      ${CI_STACK_CONFIG_SCOPES}
       ci generate --check-index-only
       --artifacts-root "${CI_PROJECT_DIR}/jobs_scratch_dir"
       --output-file "${CI_PROJECT_DIR}/jobs_scratch_dir/cloud-ci-pipeline.yml"
-  after_script:
-    - cat /proc/loadavg || true
-    - cat /proc/meminfo | grep 'MemTotal\|MemFree' || true
   artifacts:
     paths:
       - "${CI_PROJECT_DIR}/jobs_scratch_dir"
@@ -179,6 +174,16 @@ default:
 # Generate without tags for cases using external runners
 .generate-base:
   extends: [ ".base-job", ".generate-common" ]
+  before_script:
+    - uname -a || true
+    - grep -E 'vendor|model name' /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true
+    - nproc || true
+    - cat /proc/loadavg || true
+    - cat /proc/meminfo | grep 'MemTotal\|MemFree' || true
+    - . "./share/spack/setup-env.sh"
+  after_script:
+    - cat /proc/loadavg || true
+    - cat /proc/meminfo | grep 'MemTotal\|MemFree' || true
 
 .generate-x86_64:
   extends: [ ".generate-base" ]
@@ -196,6 +201,25 @@ default:
   extends: [ ".generate-base" ]
   tags: ["spack", "public", "medium", "neoverse_v2"]
 
+.generate-win64:
+  extends: [ ".base-job", ".generate-common" ]
+  before_script:
+    - $ErrorActionOld=$ErrorActionPreference
+    - $ErrorActionPreference="SilentlyContinue"
+    - python -c"import psutil;print(psutil.getloadavg())"
+    - (Get-WmiObject Win32_PhysicalMemory | measure-object Capacity -sum).sum/1kb
+    - $ErrorActionPreference=$ErrorActionOld
+    - . .\share\spack\setup-env.ps1
+  after_script:
+    - $ErrorActionOld=$ErrorActionPreference
+    - $ErrorActionPreference="SilentlyContinue"
+    - python -c"import psutil;print(psutil.getloadavg())"
+    - (Get-WmiObject Win32_PhysicalMemory | measure-object Capacity -sum).sum/1kb
+    - $ErrorActionPreference=$ErrorActionOld
+
+  tags: ["spack", "public", "medium", "x86_64-win"]
+  image: "ghcr.io/johnwparent/windows-server21h2:sha-c749cf3"
+
 .generate-deprecated:
   extends: [ ".base-job" ]
   stage: generate
@@ -859,6 +883,15 @@ aws-pcluster-build-neoverse_v1:
     - echo $PATH
     - module avail
     - module list
+    - uname -a || true
+    - grep -E 'vendor|model name' /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true
+    - nproc || true
+    - cat /proc/loadavg || true
+    - cat /proc/meminfo | grep 'MemTotal\|MemFree' || true
+    - . "./share/spack/setup-env.sh"
+  after_script:
+    - cat /proc/loadavg || true
+    - cat /proc/meminfo | grep 'MemTotal\|MemFree' || true
 
 .generate-cray-rhel:
   tags: [ "cray-rhel-zen4", "public" ]
@@ -912,3 +945,25 @@ e4s-cray-sles-build:
   needs:
     - artifacts: True
       job: e4s-cray-sles-generate
+
+#######################################
+# Windows Visualization Tools
+#######################################
+.windows-vis:
+  extends: [".win64-msvc2019"]
+  variables:
+    SPACK_CI_STACK_NAME: windows-vis
+
+windows-vis-generate:
+  extends: [ ".generate-win64", ".windows-vis" ]
+
+windows-vis-build:
+  extends: [ ".build", ".windows-vis"]
+  trigger:
+    include:
+      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
+        job: windows-vis-generate
+    strategy: depend
+  needs:
+    - artifacts: True
+      job: windows-vis-generate

share/spack/gitlab/cloud_pipelines/configs/win64/ci.yaml

@@ -0,0 +1,18 @@
+ci:
+  pipeline-gen:
+  - build-job:
+      after_script::
+      - Write-Output "Done"
+
+      before_script::
+      - fsutil 8dot3name set C:\ 0
+      - . .\share\spack\setup-env.ps1
+      - If (Test-Path -path C:\\key\intermediate_ci_signing_key.gpg) { spack.ps1 gpg trust C:\\key\intermediate_ci_signing_key.gpg }
+      - If (Test-Path -path C:\\key\spack_public_key.gpg) { spack.ps1 gpg trust C:\\key\spack_public_key.gpg }
+
+      script::
+      - spack.ps1 env activate --without-view ${SPACK_CONCRETE_ENV_DIR}
+      - spack.ps1 config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{hash}'"
+      - mkdir ${SPACK_ARTIFACTS_ROOT}/user_data
+      - spack.ps1 --backtrace ci rebuild | Tee-Object -FilePath "${env:SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt" 2>&1 | Tee-Object -FilePath "${env:SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt"
+      image: "ghcr.io/johnwparent/windows-server21h2:sha-c749cf3"

share/spack/gitlab/cloud_pipelines/configs/win64/config.yaml

@@ -0,0 +1,10 @@
+config:
+  build_stage::
+    - 'C:/spack stage'
+  install_tree:
+    root: "C:/spack install"
+    # Path lengths on windows doesn't support much padding
+    padded_length: 0
+    # Reduce the projections to only including the hash to avoid path length issues
+    projections:
+      all: '{hash}'

share/spack/gitlab/cloud_pipelines/configs/win64/packages.yaml

@@ -0,0 +1,25 @@
+packages:
+  all:
+    target: [x86_64]
+  tbb:
+    require: "intel-tbb"
+  cmake:
+    externals:
+    - spec: cmake@3.28.0-msvc1
+      prefix: "C:\\Program Files\\Microsoft Visual Studio\\2022\\Community\\Common7\\IDE\\CommonExtensions\\Microsoft\\CMake\\CMake"
+    buildable: False
+  ninja:
+    externals:
+    - spec: ninja@1.11.0
+      prefix: "C:\\Program Files\\Microsoft Visual Studio\\2022\\Community\\Common7\\IDE\\CommonExtensions\\Microsoft\\CMake\\Ninja"
+    buildable: False
+  wgl:
+    externals:
+    - spec: wgl@10.0.22621 plat=x64
+      prefix: "C:\\Program Files (x86)\\Windows Kits\\10"
+    buildable: False
+  win-sdk:
+    externals:
+    - spec: win-sdk@10.0.22621 plat=x64
+      prefix: "C:\\Program Files (x86)\\Windows Kits\\10"
+    buildable: False

share/spack/gitlab/cloud_pipelines/configs/win64/x86_64/ci.yaml

@@ -0,0 +1,4 @@
+ci:
+  pipeline-gen:
+  - build-job:
+      tags: [x86_64-win]

share/spack/gitlab/cloud_pipelines/configs/win64/x86_64/packages.yaml

@@ -0,0 +1,3 @@
+packages:
+  all:
+    target: [x86_64]

share/spack/gitlab/cloud_pipelines/scripts/common/expand_vars.py

@@ -0,0 +1,10 @@
+import argparse
+import os
+
+parser = argparse.ArgumentParser()
+parser.add_argument("input", type=argparse.FileType("r"))
+parser.add_argument("out", type=argparse.FileType("w"))
+
+args = parser.parse_args()
+
+args.out.write(os.path.expandvars(args.input.read()))

share/spack/gitlab/cloud_pipelines/stacks/windows-vis/spack.yaml

@@ -0,0 +1,12 @@
+# Windows Visualization Stack
+# maintainers:
+# - John Parent (@johnwparent)
+# - Ryan Krattiger (@kwryankrattiger)
+
+spack:
+  view: false
+  specs:
+  - vtk
+
+  cdash:
+    build-group: Windows Visualization (Kitware)

var/spack/repos/builtin/packages/proj/package.py

@@ -87,15 +87,14 @@ class Proj(CMakePackage, AutotoolsPackage):
         when="@6.2:9.1",
     )
 
-    patch("proj.cmakelists.5.0.patch", when="@5.0")
-    patch("proj.cmakelists.5.1.patch", when="@5.1:5.2")
-
     # https://proj.org/install.html#build-requirements
     with when("build_system=cmake"):
         # CMake 3.19 refactored the FindTiff module interface, update older proj's
         # to be compatible with this "new" interface
         # patch replaces the TIFF_LIBRARY variable (no longer used) with TIFF_LIBRARIES
-        patch("proj-8.1-cmake-3.29-new-tiff-interface.patch", when="+tiff @:9.1.0 ^cmake@3.19:")
+        patch("proj-8.1-cmake-3.29-new-tiff-interface.patch", when="+tiff @7:9.1.0 ^cmake@3.19:")
+        patch("proj.cmakelists.5.0.patch", when="@5.0")
+        patch("proj.cmakelists.5.1.patch", when="@5.1:5.2")
         conflicts("cmake@3.19:", when="@:7")
         depends_on("cmake@3.9:", when="@6:", type="build")
         depends_on("cmake@3.5:", when="@5", type="build")

var/spack/repos/builtin/packages/wgl/package.py

@@ -14,7 +14,7 @@ class Wgl(Package):
 
     homepage = "https://learn.microsoft.com/en-us/windows/win32/opengl/wgl-and-windows-reference"
     has_code = False
-    tags = ["windows"]
+    tags = ["windows", "windows-system"]
     # hard code the extension as shared lib
     libraries = ["OpenGL32.Lib"]
 
@@ -36,6 +36,10 @@ class Wgl(Package):
     version("10.0.26639")
     version("10.0.20348")
 
+    variant(
+        "plat", values=("x64", "x86", "arm", "arm64"), default="x64", description="Toolchain arch"
+    )
+
     # As per https://github.com/spack/spack/pull/31748 this provisory version represents
     # an arbitrary openGL version designed for maximum compatibility with calling packages
     # this current version simply reflects the latest OpenGL vesion available at the time of
@@ -68,6 +72,16 @@ def determine_version(cls, lib):
         ver_str = re.search(version_match_pat, lib)
         return ver_str if not ver_str else Version(ver_str.group())
 
+    @classmethod
+    def determine_variants(cls, libs, ver_str):
+        """Allow for determination of toolchain arch for detected WGL"""
+        variants = []
+        for lib in libs:
+            base, lib_name = os.path.split(lib)
+            _, arch = os.path.split(base)
+            variants.append("plat=%s" % arch)
+        return variants
+
     def _spec_arch_to_sdk_arch(self):
         spec_arch = str(self.spec.architecture.target.microarchitecture.family).lower()
         _64bit = "64" in spec_arch

var/spack/repos/builtin/packages/win-sdk/package.py

@@ -19,7 +19,7 @@ class WinSdk(Package):
 
     homepage = "https://developer.microsoft.com/en-us/windows/downloads/windows-sdk/"
     has_code = False
-    tags = ["windows"]
+    tags = ["windows", "windows-system"]
 
     # The sdk has many libraries and executables. Record one for detection purposes
     libraries = ["rcdll.dll"]

var/spack/repos/builtin/packages/win-wdk/package.py

@@ -18,7 +18,7 @@ class WinWdk(Package):
     """
 
     homepage = "https://learn.microsoft.com/en-us/windows-hardware/drivers/"
-    tags = ["windows"]
+    tags = ["windows", "windows-system"]
 
     # The wdk has many libraries and executables. Record one for detection purposes
     libraries = ["mmos.lib"]