hlrs-software-stack/spack
7
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Gitlab CI: Windows Configs (#43967)

Browse source 
Add support for Gitlab CI on Windows

This PR adds the config changes required to configure and execute
Gitlab pipelines running Windows builds on Windows runners using
the existing Gitlab CI infrastructure (and newly added Windows 
infrastructure).

* Adds support for generating child pipelines dispatched to Windows runners
* Refactors the relevant pre-scripts, scripts, and post scripts to be compatible with Windows
* Adds Windows config section describing Windows jobs
* Adds VTK as Windows build stack (to be expanded later)
* Modifies proj to build on Windows
* Refactors Windows rpath symlinking to avoid system libs and externals

---------

Co-authored-by: Ryan Krattiger <ryan.krattiger@kitware.com>
Co-authored-by: Mike VanDenburgh <michael.vandenburgh@kitware.com>
Co-authored-by: Todd Gamblin <tgamblin@llnl.gov>
Co-authored-by: Scott Wittenburg <scott.wittenburg@kitware.com>
This commit is contained in:
John W. Parent 2024-05-16 19:00:02 -04:00 committed by Harmen Stoppels
parent a79b1bd9af
commit a70ea11e69
16 changed files with 220 additions and 31 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
bin/spack.ps1
View file

@ -144,3 +144,5 @@ switch($SpackSubCommand)
"unload" {Invoke-SpackLoad}
default {python "$Env:SPACK_ROOT/bin/spack" $SpackCMD_params $SpackSubCommand $SpackSubCommandArgs}
}
exit $LASTEXITCODE

10
lib/spack/llnl/util/filesystem.py
View file

@ -2510,8 +2510,14 @@ def establish_link(self):
# for each binary install dir in self.pkg (i.e. pkg.prefix.bin, pkg.prefix.lib)
# install a symlink to each dependent library
for library, lib_dir in itertools.product(self.rpaths, self.library_dependents):
self._link(library, lib_dir)
# do not rpath for system libraries included in the dag
# we should not be modifying libraries managed by the Windows system
# as this will negatively impact linker behavior and can result in permission
# errors if those system libs are not modifiable by Spack
if "windows-system" not in getattr(self.pkg, "tags", []):
for library, lib_dir in itertools.product(self.rpaths, self.library_dependents):
self._link(library, lib_dir)
@system_path_filter

22
lib/spack/spack/ci.py
View file

@ -684,6 +684,22 @@ def generate_gitlab_ci_yaml(
"instead.",
)
def ensure_expected_target_path(path):
"""Returns passed paths with all Windows path separators exchanged
for posix separators only if copy_only_pipeline is enabled
This is required as copy_only_pipelines are a unique scenario where
the generate job and child pipelines are run on different platforms.
To make this compatible w/ Windows, we cannot write Windows style path separators
that will be consumed on by the Posix copy job runner.
TODO (johnwparent): Refactor config + cli read/write to deal only in posix
style paths
"""
if copy_only_pipeline and path:
path = path.replace("\\", "/")
return path
pipeline_mirrors = spack.mirror.MirrorCollection(binary=True)
deprecated_mirror_config = False
buildcache_destination = None
@ -807,7 +823,7 @@ def generate_gitlab_ci_yaml(
if scope not in include_scopes and scope not in env_includes:
include_scopes.insert(0, scope)
env_includes.extend(include_scopes)
env_yaml_root["spack"]["include"] = env_includes
env_yaml_root["spack"]["include"] = [ensure_expected_target_path(i) for i in env_includes]
if "gitlab-ci" in env_yaml_root["spack"] and "ci" not in env_yaml_root["spack"]:
env_yaml_root["spack"]["ci"] = env_yaml_root["spack"].pop("gitlab-ci")
@ -1228,6 +1244,9 @@ def main_script_replacements(cmd):
"SPACK_REBUILD_EVERYTHING": str(rebuild_everything),
"SPACK_REQUIRE_SIGNING": os.environ.get("SPACK_REQUIRE_SIGNING", "False"),
}
output_vars = output_object["variables"]
for item, val in output_vars.items():
output_vars[item] = ensure_expected_target_path(val)
# TODO: Remove this block in Spack 0.23
if deprecated_mirror_config and remote_mirror_override:
@ -1284,7 +1303,6 @@ def main_script_replacements(cmd):
sorted_output = {}
for output_key, output_value in sorted(output_object.items()):
sorted_output[output_key] = output_value
if known_broken_specs_encountered:
tty.error("This pipeline generated hashes known to be broken on develop:")
display_broken_spec_messages(broken_specs_url, known_broken_specs_encountered)

17
lib/spack/spack/package_base.py
View file

@ -161,7 +161,11 @@ def windows_establish_runtime_linkage(self):
Performs symlinking to incorporate rpath dependencies to Windows runtime search paths
"""
if sys.platform == "win32":
# If spec is an external, we should not be modifying its bin directory, as we would
# be doing in this method
# Spack should in general not modify things it has not installed
# we can reasonably expect externals to have their link interface properly established
if sys.platform == "win32" and not self.spec.external:
self.win_rpath.add_library_dependent(*self.win_add_library_dependent())
self.win_rpath.add_rpath(*self.win_add_rpath())
self.win_rpath.establish_link()
@ -2446,9 +2450,18 @@ def rpath(self):
# on Windows, libraries of runtime interest are typically
# stored in the bin directory
# Do not include Windows system libraries in the rpath interface
# these libraries are handled automatically by VS/VCVARS and adding
# Spack derived system libs into the link path or address space of a program
# can result in conflicting versions, which makes Spack packages less useable
if sys.platform == "win32":
rpaths = [self.prefix.bin]
rpaths.extend(d.prefix.bin for d in deps if os.path.isdir(d.prefix.bin))
rpaths.extend(
d.prefix.bin
for d in deps
if os.path.isdir(d.prefix.bin)
and "windows-system" not in getattr(d.package, "tags", [])
)
else:
rpaths = [self.prefix.lib, self.prefix.lib64]
rpaths.extend(d.prefix.lib for d in deps if os.path.isdir(d.prefix.lib))

91
share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml
View file

@ -64,6 +64,11 @@ default:
SPACK_TARGET_PLATFORM: "linux"
SPACK_TARGET_ARCH: "ppc64le"
.win64-msvc2019:
variables:
SPACK_TARGET_PLATFORM: "win64"
SPACK_TARGET_ARCH: "x86_64"
########################################
# Job templates
########################################
@ -72,6 +77,8 @@ default:
PIPELINE_MIRROR_TEMPLATE: "single-src-protected-mirrors.yaml.in"
# TODO: We can remove this when we drop the "deprecated" stack
PUSH_BUILDCACHE_DEPRECATED: "${PROTECTED_MIRROR_PUSH_DOMAIN}/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}"
SPACK_CI_CONFIG_ROOT: "${CI_PROJECT_DIR}/share/spack/gitlab/cloud_pipelines/configs"
SPACK_CI_SCRIPTS_ROOT: "${CI_PROJECT_DIR}/share/spack/gitlab/cloud_pipelines/scripts"
rules:
- if: $SPACK_CI_DISABLE_STACKS =~ /.+/ && $SPACK_CI_STACK_NAME =~ $SPACK_CI_DISABLE_STACKS
@ -114,16 +121,8 @@ default:
.generate-common:
stage: generate
script:
- uname -a || true
- grep -E 'vendor|model name' /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true
- nproc || true
- cat /proc/loadavg || true
- cat /proc/meminfo | grep 'MemTotal\|MemFree' || true
- . "./share/spack/setup-env.sh"
- spack --version
- cd share/spack/gitlab/cloud_pipelines/stacks/${SPACK_CI_STACK_NAME}
- spack env activate --without-view .
- export SPACK_CI_CONFIG_ROOT="${SPACK_ROOT}/share/spack/gitlab/cloud_pipelines/configs"
- spack env activate --without-view share/spack/gitlab/cloud_pipelines/stacks/${SPACK_CI_STACK_NAME}
- spack
--config-scope "${SPACK_CI_CONFIG_ROOT}"
--config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}"
@ -134,29 +133,25 @@ default:
--config-scope "${SPACK_CI_CONFIG_ROOT}"
--config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}"
--config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}/${SPACK_TARGET_ARCH}"
${CI_STACK_CONFIG_SCOPES}
audit configs
- spack python -c "import os,sys; print(os.path.expandvars(sys.stdin.read()))"
< "${SPACK_CI_CONFIG_ROOT}/${PIPELINE_MIRROR_TEMPLATE}" > "${SPACK_CI_CONFIG_ROOT}/mirrors.yaml"
# Command below needs to be `spack python` due to naming differences accross platforms
- spack python ${SPACK_CI_SCRIPTS_ROOT}/common/expand_vars.py
"${SPACK_CI_CONFIG_ROOT}/${PIPELINE_MIRROR_TEMPLATE}"
"${SPACK_CI_CONFIG_ROOT}/mirrors.yaml"
- spack config add -f "${SPACK_CI_CONFIG_ROOT}/mirrors.yaml"
- mkdir -p "${CI_PROJECT_DIR}/jobs_scratch_dir"
- mkdir "${CI_PROJECT_DIR}/jobs_scratch_dir"
- spack
--config-scope "${SPACK_CI_CONFIG_ROOT}"
--config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}"
--config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}/${SPACK_TARGET_ARCH}"
${CI_STACK_CONFIG_SCOPES}
config blame > "${CI_PROJECT_DIR}/jobs_scratch_dir/spack.yaml.blame"
- spack -v --color=always
--config-scope "${SPACK_CI_CONFIG_ROOT}"
--config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}"
--config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}/${SPACK_TARGET_ARCH}"
${CI_STACK_CONFIG_SCOPES}
ci generate --check-index-only
--artifacts-root "${CI_PROJECT_DIR}/jobs_scratch_dir"
--output-file "${CI_PROJECT_DIR}/jobs_scratch_dir/cloud-ci-pipeline.yml"
after_script:
- cat /proc/loadavg || true
- cat /proc/meminfo | grep 'MemTotal\|MemFree' || true
artifacts:
paths:
- "${CI_PROJECT_DIR}/jobs_scratch_dir"
@ -179,6 +174,16 @@ default:
# Generate without tags for cases using external runners
.generate-base:
extends: [ ".base-job", ".generate-common" ]
before_script:
- uname -a || true
- grep -E 'vendor|model name' /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true
- nproc || true
- cat /proc/loadavg || true
- cat /proc/meminfo | grep 'MemTotal\|MemFree' || true
- . "./share/spack/setup-env.sh"
after_script:
- cat /proc/loadavg || true
- cat /proc/meminfo | grep 'MemTotal\|MemFree' || true
.generate-x86_64:
extends: [ ".generate-base" ]
@ -196,6 +201,25 @@ default:
extends: [ ".generate-base" ]
tags: ["spack", "public", "medium", "neoverse_v2"]
.generate-win64:
extends: [ ".base-job", ".generate-common" ]
before_script:
- $ErrorActionOld=$ErrorActionPreference
- $ErrorActionPreference="SilentlyContinue"
- python -c"import psutil;print(psutil.getloadavg())"
- (Get-WmiObject Win32_PhysicalMemory | measure-object Capacity -sum).sum/1kb
- $ErrorActionPreference=$ErrorActionOld
- . .\share\spack\setup-env.ps1
after_script:
- $ErrorActionOld=$ErrorActionPreference
- $ErrorActionPreference="SilentlyContinue"
- python -c"import psutil;print(psutil.getloadavg())"
- (Get-WmiObject Win32_PhysicalMemory | measure-object Capacity -sum).sum/1kb
- $ErrorActionPreference=$ErrorActionOld
tags: ["spack", "public", "medium", "x86_64-win"]
image: "ghcr.io/johnwparent/windows-server21h2:sha-c749cf3"
.generate-deprecated:
extends: [ ".base-job" ]
stage: generate
@ -859,6 +883,15 @@ aws-pcluster-build-neoverse_v1:
- echo $PATH
- module avail
- module list
- uname -a || true
- grep -E 'vendor|model name' /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true
- nproc || true
- cat /proc/loadavg || true
- cat /proc/meminfo | grep 'MemTotal\|MemFree' || true
- . "./share/spack/setup-env.sh"
after_script:
- cat /proc/loadavg || true
- cat /proc/meminfo | grep 'MemTotal\|MemFree' || true
.generate-cray-rhel:
tags: [ "cray-rhel-zen4", "public" ]
@ -912,3 +945,25 @@ e4s-cray-sles-build:
needs:
- artifacts: True
job: e4s-cray-sles-generate
#######################################
# Windows Visualization Tools
#######################################
.windows-vis:
extends: [".win64-msvc2019"]
variables:
SPACK_CI_STACK_NAME: windows-vis
windows-vis-generate:
extends: [ ".generate-win64", ".windows-vis" ]
windows-vis-build:
extends: [ ".build", ".windows-vis"]
trigger:
include:
- artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
job: windows-vis-generate
strategy: depend
needs:
- artifacts: True
job: windows-vis-generate

18
share/spack/gitlab/cloud_pipelines/configs/win64/ci.yaml Normal file
View file

@ -0,0 +1,18 @@
ci:
pipeline-gen:
- build-job:
after_script::
- Write-Output "Done"
before_script::
- fsutil 8dot3name set C:\ 0
- . .\share\spack\setup-env.ps1
- If (Test-Path -path C:\\key\intermediate_ci_signing_key.gpg) { spack.ps1 gpg trust C:\\key\intermediate_ci_signing_key.gpg }
- If (Test-Path -path C:\\key\spack_public_key.gpg) { spack.ps1 gpg trust C:\\key\spack_public_key.gpg }
script::
- spack.ps1 env activate --without-view ${SPACK_CONCRETE_ENV_DIR}
- spack.ps1 config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{hash}'"
- mkdir ${SPACK_ARTIFACTS_ROOT}/user_data
- spack.ps1 --backtrace ci rebuild | Tee-Object -FilePath "${env:SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt" 2>&1 | Tee-Object -FilePath "${env:SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt"
image: "ghcr.io/johnwparent/windows-server21h2:sha-c749cf3"

10
share/spack/gitlab/cloud_pipelines/configs/win64/config.yaml Normal file
View file

@ -0,0 +1,10 @@
config:
build_stage::
- 'C:/spack stage'
install_tree:
root: "C:/spack install"
# Path lengths on windows doesn't support much padding
padded_length: 0
# Reduce the projections to only including the hash to avoid path length issues
projections:
all: '{hash}'

25
share/spack/gitlab/cloud_pipelines/configs/win64/packages.yaml Normal file
View file

@ -0,0 +1,25 @@
packages:
all:
target: [x86_64]
tbb:
require: "intel-tbb"
cmake:
externals:
- spec: cmake@3.28.0-msvc1
prefix: "C:\\Program Files\\Microsoft Visual Studio\\2022\\Community\\Common7\\IDE\\CommonExtensions\\Microsoft\\CMake\\CMake"
buildable: False
ninja:
externals:
- spec: ninja@1.11.0
prefix: "C:\\Program Files\\Microsoft Visual Studio\\2022\\Community\\Common7\\IDE\\CommonExtensions\\Microsoft\\CMake\\Ninja"
buildable: False
wgl:
externals:
- spec: wgl@10.0.22621 plat=x64
prefix: "C:\\Program Files (x86)\\Windows Kits\\10"
buildable: False
win-sdk:
externals:
- spec: win-sdk@10.0.22621 plat=x64
prefix: "C:\\Program Files (x86)\\Windows Kits\\10"
buildable: False

4
share/spack/gitlab/cloud_pipelines/configs/win64/x86_64/ci.yaml Normal file
View file

@ -0,0 +1,4 @@
ci:
pipeline-gen:
- build-job:
tags: [x86_64-win]

3
share/spack/gitlab/cloud_pipelines/configs/win64/x86_64/packages.yaml Normal file
View file

@ -0,0 +1,3 @@
packages:
all:
target: [x86_64]

10
share/spack/gitlab/cloud_pipelines/scripts/common/expand_vars.py Normal file
View file

@ -0,0 +1,10 @@
import argparse
import os
parser = argparse.ArgumentParser()
parser.add_argument("input", type=argparse.FileType("r"))
parser.add_argument("out", type=argparse.FileType("w"))
args = parser.parse_args()
args.out.write(os.path.expandvars(args.input.read()))

12
share/spack/gitlab/cloud_pipelines/stacks/windows-vis/spack.yaml Normal file
View file

@ -0,0 +1,12 @@
# Windows Visualization Stack
# maintainers:
# - John Parent (@johnwparent)
# - Ryan Krattiger (@kwryankrattiger)
spack:
view: false
specs:
- vtk
cdash:
build-group: Windows Visualization (Kitware)

7
var/spack/repos/builtin/packages/proj/package.py
View file

@ -87,15 +87,14 @@ class Proj(CMakePackage, AutotoolsPackage):
when="@6.2:9.1",
)
patch("proj.cmakelists.5.0.patch", when="@5.0")
patch("proj.cmakelists.5.1.patch", when="@5.1:5.2")
# https://proj.org/install.html#build-requirements
with when("build_system=cmake"):
# CMake 3.19 refactored the FindTiff module interface, update older proj's
# to be compatible with this "new" interface
# patch replaces the TIFF_LIBRARY variable (no longer used) with TIFF_LIBRARIES
patch("proj-8.1-cmake-3.29-new-tiff-interface.patch", when="+tiff @:9.1.0 ^cmake@3.19:")
patch("proj-8.1-cmake-3.29-new-tiff-interface.patch", when="+tiff @7:9.1.0 ^cmake@3.19:")
patch("proj.cmakelists.5.0.patch", when="@5.0")
patch("proj.cmakelists.5.1.patch", when="@5.1:5.2")
conflicts("cmake@3.19:", when="@:7")
depends_on("cmake@3.9:", when="@6:", type="build")
depends_on("cmake@3.5:", when="@5", type="build")

16
var/spack/repos/builtin/packages/wgl/package.py
View file

@ -14,7 +14,7 @@ class Wgl(Package):
homepage = "https://learn.microsoft.com/en-us/windows/win32/opengl/wgl-and-windows-reference"
has_code = False
tags = ["windows"]
tags = ["windows", "windows-system"]
# hard code the extension as shared lib
libraries = ["OpenGL32.Lib"]
@ -36,6 +36,10 @@ class Wgl(Package):
version("10.0.26639")
version("10.0.20348")
variant(
"plat", values=("x64", "x86", "arm", "arm64"), default="x64", description="Toolchain arch"
)
# As per https://github.com/spack/spack/pull/31748 this provisory version represents
# an arbitrary openGL version designed for maximum compatibility with calling packages
# this current version simply reflects the latest OpenGL vesion available at the time of
@ -68,6 +72,16 @@ def determine_version(cls, lib):
ver_str = re.search(version_match_pat, lib)
return ver_str if not ver_str else Version(ver_str.group())
@classmethod
def determine_variants(cls, libs, ver_str):
"""Allow for determination of toolchain arch for detected WGL"""
variants = []
for lib in libs:
base, lib_name = os.path.split(lib)
_, arch = os.path.split(base)
variants.append("plat=%s" % arch)
return variants
def _spec_arch_to_sdk_arch(self):
spec_arch = str(self.spec.architecture.target.microarchitecture.family).lower()
_64bit = "64" in spec_arch

2
var/spack/repos/builtin/packages/win-sdk/package.py
View file

@ -19,7 +19,7 @@ class WinSdk(Package):
homepage = "https://developer.microsoft.com/en-us/windows/downloads/windows-sdk/"
has_code = False
tags = ["windows"]
tags = ["windows", "windows-system"]
# The sdk has many libraries and executables. Record one for detection purposes
libraries = ["rcdll.dll"]

2
var/spack/repos/builtin/packages/win-wdk/package.py
View file

@ -18,7 +18,7 @@ class WinWdk(Package):
"""
homepage = "https://learn.microsoft.com/en-us/windows-hardware/drivers/"
tags = ["windows"]
tags = ["windows", "windows-system"]
# The wdk has many libraries and executables. Record one for detection purposes
libraries = ["mmos.lib"]