Spack on Windows package ports

CMake - Windows Bootstrap (#25825)

Remove hardcoded cmake compiler (#26410)

Revert breaking cmake changes
Ensure no autotools on Windows

Perl on Windows (#26612)

Python source build windows (#26313)

Reconfigure sysconf for Windows

Python2.6 compatibility

Fxixup new sbang tests for windows

Ruby support (#28287)

Add NASM support (#28319)

Add mock Ninja package for testing
This commit is contained in:
John Parent 2021-10-22 14:00:02 -04:00 committed by Peter Scheibel
parent 90c773488c
commit 3a994032f8
27 changed files with 646 additions and 197 deletions

View file

@ -15,8 +15,12 @@ popd
:: Check if Python is on the PATH
if not defined python_pf_ver (
(for /f "delims=" %%F in ('where python.exe') do (set python_pf_ver=%%F) ) 2> NUL
(for /f "delims=" %%F in ('where python.exe') do (
set python_pf_ver=%%F
goto :found_python
) ) 2> NUL
)
:found_python
if not defined python_pf_ver (
:: If not, look for Python from the Spack installer
:get_builtin
@ -35,7 +39,7 @@ if not defined python_pf_ver (
:: Python is already on the path
set py_exe=!python_pf_ver!
(for /F "tokens=* USEBACKQ" %%F in (
`!py_exe! --version`) do (set "output=%%F")) 2>NUL
`"!py_exe!" --version`) do (set "output=%%F")) 2>NUL
if not "!output:Microsoft Store=!"=="!output!" goto :get_builtin
goto :exitpoint
)

View file

@ -1587,14 +1587,6 @@ When given the option of adjusting your ``PATH``, choose the ``Git from the
command line and also from 3rd-party software`` option. This will automatically
update your ``PATH`` variable to include the ``git`` command.
""""
Perl
""""
Perl is a flexible and feature-rich programming language that comes built-in
on Unix boxes but needs to be installed externally for Windows users. Fortunately,
you can find the most recent release at https://www.perl.org/get.html.
""""
NASM
""""

View file

@ -146,7 +146,7 @@ def process_stacktrace(countback):
file_list = []
for frame in st:
# Check that the file is a spack file
if frame[0].find("/spack") >= 0:
if frame[0].find(os.path.sep + "spack") >= 0:
file_list.append(frame[0])
# We use commonprefix to find what the spack 'root' directory is.
root_dir = os.path.commonprefix(file_list)

View file

@ -528,7 +528,9 @@ def _set_variables_for_single_module(pkg, module):
m.cmake = Executable('cmake')
m.ctest = MakeExecutable('ctest', jobs)
# Standard build system arguments
if os.name == 'nt':
m.nmake = Executable('nmake')
# Standard CMake arguments
m.std_cmake_args = spack.build_systems.cmake.CMakePackage._std_args(pkg)
m.std_meson_args = spack.build_systems.meson.MesonPackage._std_args(pkg)
m.std_pip_args = spack.build_systems.python.PythonPackage._std_args(pkg)

View file

@ -14,7 +14,7 @@
from llnl.util.filesystem import force_remove, working_dir
from spack.build_environment import InstallError
from spack.directives import depends_on
from spack.directives import conflicts, depends_on
from spack.operating_systems.mac_os import macos_version
from spack.package import PackageBase, run_after, run_before
from spack.util.executable import Executable
@ -104,6 +104,7 @@ def patch_config_files(self):
depends_on('gnuconfig', type='build', when='target=ppc64le:')
depends_on('gnuconfig', type='build', when='target=aarch64:')
depends_on('gnuconfig', type='build', when='target=riscv64:')
conflicts('platform=windows')
@property
def _removed_la_files_log(self):

View file

@ -93,15 +93,11 @@ class CMakePackage(PackageBase):
#: See https://cmake.org/cmake/help/latest/manual/cmake-generators.7.html
#: for more information.
variant('generator',
default='Make' if sys.platform != 'win32' else 'Ninja',
description='Build system to generate',
values=('Make', 'Ninja'))
generator = "Unix Makefiles"
depends_on('ninja', when='generator=Ninja')
generatorMap = {'Make': 'Unix Makefiles',
'Ninja': 'Ninja'}
if sys.platform == 'win32':
generator = "Ninja"
depends_on('ninja')
# https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html
variant('build_type', default='RelWithDebInfo',
@ -150,10 +146,10 @@ def _std_args(pkg):
"""Computes the standard cmake arguments for a generic package"""
try:
pkg.generator = pkg.spec.variants['generator'].value
except KeyError:
pkg.generator = 'Make' if sys.platform != 'win32' else 'Ninja'
primary_generator = CMakePackage.generatorMap[pkg.generator]
if not pkg.generator:
raise AttributeError
except AttributeError:
pkg.generator = CMakePackage.generator
try:
build_type = pkg.spec.variants['build_type'].value
@ -167,22 +163,16 @@ def _std_args(pkg):
define = CMakePackage.define
args = [
'-G', primary_generator,
'-G', pkg.generator,
define('CMAKE_INSTALL_PREFIX', pkg.prefix.replace('\\', '/')),
define('CMAKE_BUILD_TYPE', build_type),
define('CMAKE_C_COMPILER:FILEPATH', pkg.compiler.cc.replace('\\', '/')),
define('CMAKE_CXX_COMPILER:FILEPATH', pkg.compiler.cxx.replace('\\', '/'))
]
if pkg.compiler.fc is not None:
args.append(define('CMAKE_Fortran_COMPILER:FILEPATH',
pkg.compiler.fc.replace('\\', '/')))
# CMAKE_INTERPROCEDURAL_OPTIMIZATION only exists for CMake >= 3.9
if pkg.spec.satisfies('^cmake@3.9:'):
args.append(define('CMAKE_INTERPROCEDURAL_OPTIMIZATION', ipo))
if primary_generator == 'Unix Makefiles':
if pkg.generator == 'Unix Makefiles':
args.append(define('CMAKE_VERBOSE_MAKEFILE', True))
if platform.mac_ver()[0]:

View file

@ -130,7 +130,7 @@ def colorize(c, prefix):
# in the future - so this manipulation might be fragile
if nodetype.lower() == 'function':
name_parts.append(item)
key_end = os.path.join(*[x[1] for x in key_parts])
key_end = os.path.join(*key_parts[-1][1].split('/'))
key = next(f for f in files if f.endswith(key_end))
tests[key].add(tuple(x[1] for x in name_parts))
elif nodetype.lower() == 'class':

View file

@ -4,13 +4,37 @@
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import os
import re
import subprocess
import sys
from typing import List # novm
from distutils.version import StrictVersion
from typing import Dict, List, Set # novm
import spack.operating_systems.windows_os
import spack.util.executable
from spack.compiler import Compiler
from spack.error import SpackError
avail_fc_version = set() # type: Set[str]
fc_path = dict() # type: Dict[str, str]
fortran_mapping = {
'2021.3.0': '19.29.30133',
'2021.2.1': '19.28.29913',
'2021.2.0': '19.28.29334',
'2021.1.0': '19.28.29333',
}
def get_valid_fortran_pth(comp_ver):
cl_ver = str(comp_ver).split('@')[1]
sort_fn = lambda fc_ver: StrictVersion(fc_ver)
sort_fc_ver = sorted(list(avail_fc_version), key=sort_fn)
for ver in sort_fc_ver:
if ver in fortran_mapping:
if StrictVersion(cl_ver) <= StrictVersion(fortran_mapping[ver]):
return fc_path[ver]
return None
class Msvc(Compiler):
@ -46,6 +70,8 @@ class Msvc(Compiler):
# file based on compiler executable path.
def __init__(self, *args, **kwargs):
new_pth = [pth if pth else get_valid_fortran_pth(args[0]) for pth in args[3]]
args[3][:] = new_pth
super(Msvc, self).__init__(*args, **kwargs)
if os.getenv("ONEAPI_ROOT"):
# If this found, it sets all the vars
@ -65,6 +91,12 @@ def verbose_flag(self):
def pic_flag(self):
return ""
@property
def msvc_version(self):
ver = re.search(Msvc.version_regex, self.cc).group(1)
ver = "".join(ver.split('.')[:2])[:-1]
return "MSVC" + ver
def setup_custom_environment(self, pkg, env):
"""Set environment variables for MSVC using the
Microsoft-provided script."""
@ -81,44 +113,40 @@ def setup_custom_environment(self, pkg, env):
if sys.version_info[0] >= 3:
out = out.decode('utf-16le', errors='replace') # novermin
int_env = { # novermin
key.lower(): value
for key, _, value in
int_env = dict((key.lower(), value) for key, _, value in
(line.partition('=') for line in out.splitlines())
if key and value
}
if key and value)
if 'path' in int_env:
env.set_path('PATH', int_env['path'].split(';'))
env.set_path('INCLUDE', int_env.get('include', '').split(';'))
env.set_path('LIB', int_env.get('lib', '').split(';'))
env.set('CC', self.cc)
env.set('CXX', self.cxx)
env.set('FC', self.fc)
env.set('F77', self.f77)
else:
# Should not this be an exception?
print("Cannot pull msvc compiler information in Python 2.6 or below")
# fc_version only loads the ifx compiler into the first MSVC stanza;
# if there are other versions of Microsoft VS installed and detected, they
# will only have cl.exe as the C/C++ compiler
@classmethod
def fc_version(cls, fc):
# We're using intel for the Fortran compilers, which exist if
# ONEAPI_ROOT is a meaningful variable
fc_ver = cls.default_version(fc)
avail_fc_version.add(fc_ver)
fc_path[fc_ver] = fc
if os.getenv("ONEAPI_ROOT"):
try:
sps = spack.operating_systems.windows_os.WindowsOs.compiler_search_paths
except Exception:
print("sps not found.")
raise
try:
except AttributeError:
raise SpackError("Windows compiler search paths not established")
clp = spack.util.executable.which_string("cl", path=sps)
except Exception:
print("cl not found.")
raise
ver = cls.default_version(clp)
return ver
else:
return cls.default_version(fc)
ver = fc_ver
return ver
@classmethod
def f77_version(cls, f77):

View file

@ -12,7 +12,6 @@
import sys
import warnings
import llnl.util.filesystem
import llnl.util.tty

View file

@ -353,8 +353,9 @@ def _existing_url(self, url):
# Telling urllib to check if url is accessible
try:
url, headers, response = spack.util.web.read_from_url(url)
except spack.util.web.SpackWebError:
msg = "Urllib fetch failed to verify url {0}".format(url)
except spack.util.web.SpackWebError as werr:
msg = "Urllib fetch failed to verify url\
{0}\n with error {1}".format(url, werr)
raise FailedDownloadError(url, msg)
return (response.getcode() is None or response.getcode() == 200)

View file

@ -2,6 +2,8 @@
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import platform
import llnl.util.lang
import spack.error
@ -143,3 +145,6 @@ def oses():
for o in sorted(self.operating_sys.values()):
yield o._cmp_iter
yield oses
def is_64bit(self):
return platform.machine().endswith('64')

View file

@ -131,7 +131,7 @@ def test_custom_store_in_environment(mutable_config, tmpdir):
# Don't trigger evaluation here
with spack.bootstrap.ensure_bootstrap_configuration():
pass
assert str(spack.store.root) == '/tmp/store'
assert str(spack.store.root) == sep + os.path.join('tmp', 'store')
def test_nested_use_of_context_manager(mutable_config):

View file

@ -3,10 +3,12 @@
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import os
from spack.main import SpackCommand
spack_test = SpackCommand('unit-test')
cmd_test_py = 'lib/spack/spack/test/cmd/unit_test.py'
cmd_test_py = os.path.join('lib', 'spack', 'spack', 'test', 'cmd', 'unit_test.py')
def test_list():

View file

@ -3,6 +3,7 @@
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import re
import sys
import pytest

View file

@ -2,6 +2,7 @@
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import os
import os.path
import re
import shutil

View file

@ -11,6 +11,7 @@
import os
import shutil
import stat
import sys
import tempfile
import pytest
@ -193,6 +194,8 @@ def test_shebang_interpreter_regex(shebang, interpreter):
sbang.get_interpreter(shebang) == interpreter
@pytest.mark.skipif(sys.platform == 'win32',
reason="Not supported on Windows (yet)")
def test_shebang_handling(script_dir, sbang_line):
sbang.filter_shebangs_in_directory(script_dir.tempdir)
@ -339,6 +342,8 @@ def test_install_user_sbang(install_mockery, configure_user_perms):
run_test_install_sbang(False)
@pytest.mark.skipif(sys.platform == 'win32',
reason="Not supported on Windows (yet)")
def test_install_sbang_too_long(tmpdir):
root = str(tmpdir)
num_extend = sbang.system_shebang_limit - len(root) - len('/bin/sbang')
@ -357,6 +362,8 @@ def test_install_sbang_too_long(tmpdir):
assert 'cannot patch' in err
@pytest.mark.skipif(sys.platform == 'win32',
reason="Not supported on Windows (yet)")
def test_sbang_hook_skips_nonexecutable_blobs(tmpdir):
# Write a binary blob to non-executable.sh, with a long interpreter "path"
# consisting of invalid UTF-8. The latter is technically not really necessary for
@ -374,6 +381,8 @@ def test_sbang_hook_skips_nonexecutable_blobs(tmpdir):
assert b'sbang' not in f.readline()
@pytest.mark.skipif(sys.platform == 'win32',
reason="Not supported on Windows (yet)")
def test_sbang_handles_non_utf8_files(tmpdir):
# We have an executable with a copyright sign as filename
contents = (b'#!' + b'\xa9' * sbang.system_shebang_limit +
@ -408,6 +417,8 @@ def shebang_limits_system_8_spack_16():
sbang.spack_shebang_limit = spack_limit
@pytest.mark.skipif(sys.platform == 'win32',
reason="Not supported on Windows (yet)")
def test_shebang_exceeds_spack_shebang_limit(shebang_limits_system_8_spack_16, tmpdir):
"""Tests whether shebangs longer than Spack's limit are skipped"""
file = str(tmpdir.join('longer_than_spack_limit.sh'))
@ -421,6 +432,8 @@ def test_shebang_exceeds_spack_shebang_limit(shebang_limits_system_8_spack_16, t
assert b'sbang' not in f.read()
@pytest.mark.skipif(sys.platform == 'win32',
reason="Not supported on Windows (yet)")
def test_sbang_hook_handles_non_writable_files_preserving_permissions(tmpdir):
path = str(tmpdir.join('file.sh'))
with open(path, 'w') as f:

View file

@ -0,0 +1,15 @@
# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
from spack import *
class Ninja(Package):
"""Dummy Ninja Package"""
homepage = "https://ninja-build.org/"
url = "https://github.com/ninja-build/ninja/archive/v1.7.2.tar.gz"
version('1.10.2', sha256='ce35865411f0490368a8fc383f29071de6690cbadc27704734978221f25e2bed')

View file

@ -3,9 +3,13 @@
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import os
import re
import shutil
import sys
import spack.build_environment
from spack import *
class Cmake(Package):
@ -141,6 +145,12 @@ class Cmake(Package):
# https://gitlab.kitware.com/cmake/cmake/merge_requests/4075
patch('fix-xlf-ninja-mr-4075.patch', sha256="42d8b2163a2f37a745800ec13a96c08a3a20d5e67af51031e51f63313d0dedd1", when="@3.15.5")
generator = "Unix Makefiles"
if sys.platform == 'win32':
generator = "Ninja"
depends_on('ninja')
# We default ownlibs to true because it greatly speeds up the CMake
# build, and CMake is built frequently. Also, CMake is almost always
# a build dependency, and its libs will not interfere with others in
@ -149,7 +159,7 @@ class Cmake(Package):
variant('qt', default=False, description='Enables the build of cmake-gui')
variant('doc', default=False, description='Enables the generation of html and man page documentation')
variant('openssl', default=True, description="Enable openssl for curl bootstrapped by CMake when using +ownlibs")
variant('ncurses', default=True, description='Enables the build of the ncurses gui')
variant('ncurses', default=os.name != 'nt', description='Enables the build of the ncurses gui')
# See https://gitlab.kitware.com/cmake/cmake/-/issues/21135
conflicts('%gcc platform=darwin', when='@:3.17',
@ -241,12 +251,19 @@ def flag_handler(self, name, flags):
flags.append(self.compiler.cxx11_flag)
return (flags, None, None)
def setup_build_environment(self, env):
spec = self.spec
if '+openssl' in spec:
env.set('OPENSSL_ROOT_DIR', spec['openssl'].prefix)
def bootstrap_args(self):
spec = self.spec
args = [
'--prefix={0}'.format(self.prefix),
'--parallel={0}'.format(make_jobs)
]
args = []
if not os.name == 'nt':
args.extend(
['--prefix={0}'.format(self.prefix),
'--parallel={0}'.format(make_jobs)]
)
if '+ownlibs' in spec:
# Build and link to the CMake-provided third-party libraries
@ -271,6 +288,10 @@ def bootstrap_args(self):
# Now for CMake arguments to pass after the initial bootstrap
args.append('--')
else:
args.append('-DCMAKE_INSTALL_PREFIX=%s' % self.prefix)
if self.spec.satisfies('generator=Ninja'):
args.append('-GNinja')
args.append('-DCMAKE_BUILD_TYPE={0}'.format(
self.spec.variants['build_type'].value))
@ -297,20 +318,60 @@ def bootstrap_args(self):
return args
def winbootcmake(self, spec):
from spack import fetch_strategy, stage
urls = {
'3': ('https://cmake.org/files/v3.21/cmake-3.21.2-windows-x86_64.zip', "f21e72ede9d15070602b60b2c14dc779"),
'2': ('https://cmake.org/files/v2.8/cmake-2.8.4-win32-x86.zip', "a2525342e495518101381203bf4484c4")
}
if spec.satisfies('@3.0.2:'):
bootstrap_url = urls['3']
else:
bootstrap_url = urls['2']
remote = fetch_strategy.URLFetchStrategy(url=bootstrap_url[0],
checksum=bootstrap_url[1])
bootstrap_stage_path = os.path.join(self.stage.path, "cmake-bootstraper")
with stage.Stage(remote, path=bootstrap_stage_path) as bootstrap_stage:
remote.stage = bootstrap_stage
remote.fetch()
remote.check()
remote.expand()
shutil.move(bootstrap_stage.source_path, self.stage.source_path)
def cmake_bootstrap(self):
exe_prefix = self.stage.source_path
relative_cmake_exe = os.path.join('spack-src', 'bin', 'cmake.exe')
return Executable(os.path.join(exe_prefix, relative_cmake_exe))
def bootstrap(self, spec, prefix):
bootstrap_args = self.bootstrap_args()
if os.name == 'nt':
self.winbootcmake(spec)
bootstrap = self.cmake_bootstrap()
bootstrap_args.extend(['.'])
else:
bootstrap = Executable('./bootstrap')
bootstrap(*self.bootstrap_args())
bootstrap(*bootstrap_args)
def build(self, spec, prefix):
if self.generator == "Ninja":
ninja()
else:
make()
@run_after('build')
@on_package_attributes(run_tests=True)
def build_test(self):
# Some tests fail, takes forever
if self.generator == "Ninja":
ninja('test')
else:
make('test')
def install(self, spec, prefix):
if self.generator == "Ninja":
ninja('install')
else:
make('install')
if spec.satisfies('%fj'):

View file

@ -0,0 +1,18 @@
diff --git a/Mkfiles/msvc.mak b/Mkfiles/msvc1.mak
index 4f6121f..f0e6f4b 100644
--- a/Mkfiles/msvc.mak
+++ b/Mkfiles/msvc1.mak
@@ -228,10 +228,10 @@ WARNFILES = asm\warnings.c include\warnings.h doc\warnings.src
warnings:
$(RM_F) $(WARNFILES)
- $(MAKE) asm\warnings.time
+# $(MAKE) asm\warnings.time
-asm\warnings.time: $(ALLOBJ:.@OBJEXT@=.c)
- : > asm\warnings.time
+# asm\warnings.time: $(ALLOBJ:.@OBJEXT@=.c)
+# : > asm\warnings.time
$(MAKE) $(WARNFILES)
asm\warnings.c: asm\warnings.pl asm\warnings.time

View file

@ -5,17 +5,20 @@
from spack import *
is_windows = str(spack.platforms.host()) == 'windows'
class Nasm(AutotoolsPackage):
class Nasm(Package):
"""NASM (Netwide Assembler) is an 80x86 assembler designed for
portability and modularity. It includes a disassembler as well."""
homepage = "https://www.nasm.us"
url = "https://www.nasm.us/pub/nasm/releasebuilds/2.14.02/nasm-2.14.02.tar.xz"
url = "https://www.nasm.us/pub/nasm/releasebuilds/2.14.02/nasm-2.14.02.tar.gz"
list_url = "https://www.nasm.us/pub/nasm/releasebuilds"
list_depth = 1
a = '3caf6729c1073bf96629b57cee31eeb54f4f8129b01902c73428836550b30a3f'
version('2.15.05', sha256='3caf6729c1073bf96629b57cee31eeb54f4f8129b01902c73428836550b30a3f')
version('2.15.05', sha256='9182a118244b058651c576baa9d0366ee05983c4d4ae1d9ddd3236a9f2304997')
version('2.14.02', sha256='e24ade3e928f7253aa8c14aa44726d1edf3f98643f87c9d72ec1df44b26be8f5')
version('2.13.03', sha256='812ecfb0dcbc5bd409aaa8f61c7de94c5b8752a7b00c632883d15b2ed6452573')
version('2.11.06', sha256='90f60d95a15b8a54bf34d87b9be53da89ee3d6213ea739fb2305846f4585868a')
@ -24,9 +27,13 @@ class Nasm(AutotoolsPackage):
# https://bugzilla.nasm.us/show_bug.cgi?id=3392461
patch('https://src.fedoraproject.org/rpms/nasm/raw/0cc3eb244bd971df81a7f02bc12c5ec259e1a5d6/f/0001-Remove-invalid-pure_func-qualifiers.patch', level=1, sha256='ac9f315d204afa6b99ceefa1fe46d4eed2b8a23c7315d32d33c0f378d930e950', when='@2.13.03 %gcc@8:')
patch('msvc.mak.patch', when='platform=windows')
conflicts('%intel@:14', when='@2.14:',
msg="Intel 14 has immature C11 support")
phases = ['configure', 'build', 'install']
def patch(self):
# Remove flags not recognized by the NVIDIA compiler
if self.spec.satisfies('%nvhpc@:20.11'):
@ -34,3 +41,23 @@ def patch(self):
'CFLAGS="$pa_add_cflags__old_cflags"', 'configure')
filter_file(r'CFLAGS="\$pa_add_flags__old_flags -Werror=.*"',
'CFLAGS="$pa_add_flags__old_flags"', 'configure')
def configure(self, spec, prefix):
with working_dir(self.stage.source_path, create=True):
if not is_windows:
configure(['--prefix={0}'.format(self.prefix)])
def build(self, spec, prefix):
with working_dir(self.stage.source_path):
if is_windows:
touch('asm\\warnings.time')
nmake('/f', 'Mkfiles\\msvc.mak')
else:
make(['V=1'])
def install(self, spec, prefix):
with working_dir(self.stage.source_path):
if is_windows:
pass
else:
make(['install'])

View file

@ -71,7 +71,6 @@ class NetlibLapack(CMakePackage):
depends_on('blas', when='+external-blas')
depends_on('netlib-xblas+fortran+plain_blas', when='+xblas')
depends_on('python@2.7:', type='test')
depends_on('ninja@1.10.0:', when='platform=windows')
# We need to run every phase twice in order to get static and shared
# versions of the libraries. When ~shared, we run the default

View file

@ -5,6 +5,7 @@
import os
import re
import sys
import llnl.util.tty as tty
@ -96,6 +97,8 @@ class Openssl(Package): # Uses Fake Autotools, should subclass Package
depends_on('perl@5.14.0:', type=('build', 'test'))
depends_on('ca-certificates-mozilla', type=('build', 'run'), when='certs=mozilla')
conflicts('+dynamic', when=sys.platform != 'win32')
@classmethod
def determine_version(cls, exe):
output = Executable(exe)('version', output=str, error=str)
@ -151,7 +154,7 @@ def install(self, spec, prefix):
'CC=\"%s\"' % os.environ.get('SPACK_CC'),
'CXX=\"%s\"' % os.environ.get('SPACK_CXX'),
'%s' % shared_flag,
'VC-WIN64A')
'VC-WIN64A', ignore_quotes=True)
else:
config = Executable('./config')
config('--prefix=%s' % prefix,

View file

@ -11,15 +11,20 @@
# Author: Justin Too <justin@doubleotoo.com>
# Date: September 6, 2015
#
import os
import re
from contextlib import contextmanager
from llnl.util.lang import match_predicate
from llnl.util.symlink import symlink
from spack import *
from spack.operating_systems.mac_os import macos_version
host = spack.platforms.host()
is_windows = str(host) == 'windows'
class Perl(Package): # Perl doesn't use Autotools, it should subclass Package
"""Perl 5 is a highly capable, feature-rich programming language with over
@ -64,6 +69,7 @@ class Perl(Package): # Perl doesn't use Autotools, it should subclass Package
extendable = True
if not is_windows:
# Bind us below gdbm-1.20 due to API change: https://github.com/Perl/perl5/issues/18915
depends_on('gdbm@:1.19')
# :5.28 needs gdbm@:1:14.1: https://rt-archive.perl.org/perl5/Ticket/Display.html?id=133295
@ -181,6 +187,23 @@ def do_stage(self, mirror_only=False):
perm = os.stat(filename).st_mode
os.chmod(filename, perm | 0o200)
@property
def nmake_arguments(self):
args = []
if self.spec.satisfies('%msvc'):
args.append('CCTYPE=%s' % self.compiler.msvc_version)
else:
raise RuntimeError("Perl unsupported for non MSVC compilers on Windows")
args.append('INST_TOP="%s"' % self.prefix.replace('/', '\\'))
args.append("INST_ARCH=\\$(ARCHNAME)")
if self.spec.satisfies('~shared'):
args.append("ALL_STATIC=%s" % "define")
if self.spec.satisfies('~threads'):
args.extend(["USE_MULTI=undef", "USE_ITHREADS=undef", "USE_IMP_SYS=undef"])
if not host.is_64bit():
args.append("WIN64=undef")
return args
def configure_args(self):
spec = self.spec
prefix = self.prefix
@ -229,30 +252,69 @@ def configure_args(self):
return config_args
def configure(self, spec, prefix):
if is_windows:
return
configure = Executable('./Configure')
configure(*self.configure_args())
def build(self, spec, prefix):
if is_windows:
pass
else:
make()
@run_after('build')
@on_package_attributes(run_tests=True)
def build_test(self):
if is_windows:
win32_dir = os.path.join(self.stage.source_path, "win32")
with working_dir(win32_dir):
nmake('test', ignore_quotes=True)
else:
make('test')
def install(self, spec, prefix):
if is_windows:
win32_dir = os.path.join(self.stage.source_path, "win32")
with working_dir(win32_dir):
nmake('install', *self.nmake_arguments, ignore_quotes=True)
else:
make('install')
@run_after('install')
def symlink_windows(self):
if not is_windows:
return
win_install_path = os.path.join(self.prefix.bin, "MSWin32")
if host.is_64bit():
win_install_path += "-x64"
else:
win_install_path += "-x86"
if self.spec.satisfies("+threads"):
win_install_path += "-multi-thread"
else:
win_install_path += "-perlio"
for f in os.listdir(os.path.join(self.prefix.bin, win_install_path)):
lnk_path = os.path.join(self.prefix.bin, f)
src_path = os.path.join(win_install_path, f)
if not os.path.exists(lnk_path):
symlink(src_path, lnk_path)
@run_after('install')
def install_cpanm(self):
spec = self.spec
maker = make
win_prefix = ''
if is_windows:
maker = nmake
win_prefix = self.stage.source_path
if '+cpanm' in spec:
with working_dir(join_path('cpanm', 'cpanm')):
with working_dir(join_path(win_prefix, 'cpanm', 'cpanm')):
perl = spec['perl'].command
perl('Makefile.PL')
make()
make('install')
maker()
maker('install')
def _setup_dependent_env(self, env, dependent_spec, deptypes):
"""Set PATH and PERL5LIB to include the extension and
@ -295,6 +357,9 @@ def setup_dependent_package(self, module, dependent_spec):
mkdirp(module.perl_lib_dir)
def setup_build_environment(self, env):
if is_windows:
return
spec = self.spec
if (spec.version <= Version('5.34.0')
@ -321,7 +386,8 @@ def filter_config_dot_pm(self):
frustrates filter_file on some filesystems (NFSv4), so make them
temporarily writable.
"""
if is_windows:
return
kwargs = {'ignore_absent': True, 'backup': False, 'string': False}
# Find the actual path to the installed Config.pm file.
@ -409,8 +475,11 @@ def command(self):
Executable: the Perl command
"""
for ver in ('', self.spec.version):
path = os.path.join(self.prefix.bin, '{0}{1}'.format(
self.spec.name, ver))
ext = ''
if is_windows:
ext = '.exe'
path = os.path.join(self.prefix.bin, '{0}{1}{2}'.format(
self.spec.name, ver, ext))
if os.path.exists(path):
return Executable(path)
else:

View file

@ -0,0 +1,28 @@
diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat
index b5a44e3..52941c7 100644
--- a/PCbuild/get_externals.bat
+++ b/PCbuild/get_externals.bat
@@ -76,7 +76,7 @@ for %%e in (%libraries%) do (
echo.Fetching external binaries...
set binaries=
-if NOT "%IncludeLibffi%"=="false" set binaries=%binaries% libffi
+if NOT "%IncludeLibffi%"=="false" set binaries=%binaries% libffi-3.3.0
if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-1.1.1k-1
if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-8.6.9.0
if NOT "%IncludeSSLSrc%"=="false" set binaries=%binaries% nasm-2.11.06
diff --git a/PCbuild/python.props b/PCbuild/python.props
index 419d5eb..c66fb07 100644
--- a/PCbuild/python.props
+++ b/PCbuild/python.props
@@ -59,8 +59,8 @@
<sqlite3Dir>$(ExternalsDir)sqlite-3.35.5.0\</sqlite3Dir>
<bz2Dir>$(ExternalsDir)bzip2-1.0.6\</bz2Dir>
<lzmaDir>$(ExternalsDir)xz-5.2.2\</lzmaDir>
- <libffiDir>$(ExternalsDir)libffi\</libffiDir>
- <libffiOutDir>$(ExternalsDir)libffi\$(ArchName)\</libffiOutDir>
+ <libffiDir>$(ExternalsDir)libffi-3.3.0\</libffiDir>
+ <libffiOutDir>$(ExternalsDir)libffi-3.3.0\$(ArchName)\</libffiOutDir>
<libffiIncludeDir>$(libffiOutDir)include</libffiIncludeDir>
<opensslDir>$(ExternalsDir)openssl-1.1.1k\</opensslDir>
<opensslOutDir>$(ExternalsDir)openssl-bin-1.1.1k-1\$(ArchName)\</opensslOutDir>

View file

@ -4,10 +4,14 @@
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import glob
import inspect
import json
import os
import re
import subprocess
import sys
from distutils.dir_util import copy_tree
from shutil import copy
import llnl.util.tty as tty
from llnl.util.filesystem import get_filetype, path_contains_subdirectory
@ -18,8 +22,12 @@
from spack.util.environment import is_system_path
from spack.util.prefix import Prefix
arch_map = {"AMD64": "x64", "x86": "Win32",
"IA64": "Win32", "EM64T": "Win32"}
is_windows = os.name == 'nt'
class Python(AutotoolsPackage):
class Python(Package):
"""The Python programming language."""
homepage = "https://www.python.org/"
@ -29,11 +37,19 @@ class Python(AutotoolsPackage):
maintainers = ['adamjstewart', 'skosukhin', 'scheibelp', 'varioustoxins']
phases = ['configure', 'build', 'install']
#: phase
install_targets = ['install']
build_targets = []
version('3.10.2', sha256='3c0ede893011319f9b0a56b44953a3d52c7abf9657c23fb4bc9ced93b86e9c97')
version('3.10.1', sha256='b76117670e7c5064344b9c138e141a377e686b9063f3a8a620ff674fa8ec90d3')
version('3.10.0', sha256='c4e0cbad57c90690cb813fb4663ef670b4d0f587d8171e2c42bd4c9245bd2758')
version('3.9.10', sha256='1aa9c0702edbae8f6a2c95f70a49da8420aaa76b7889d3419c186bfc8c0e571e', preferred=True)
version('3.9.9', sha256='2cc7b67c1f3f66c571acc42479cdf691d8ed6b47bee12c9b68430413a17a44ea')
version('3.9.9', sha256='2cc7b67c1f3f66c571acc42479cdf691d8ed6b47bee12c9b68430413a17a44ea')
version('3.9.8', sha256='7447fb8bb270942d620dd24faa7814b1383b61fa99029a240025fd81c1db8283')
version('3.9.7', sha256='a838d3f9360d157040142b715db34f0218e535333696a5569dc6f854604eb9d1')
version('3.9.6', sha256='d0a35182e19e416fc8eae25a3dcd4d02d4997333e4ad1f2eee6010aadc3fe866')
@ -146,12 +162,12 @@ class Python(AutotoolsPackage):
description='Enable expensive build-time optimizations, if available'
)
# See https://legacy.python.org/dev/peps/pep-0394/
variant('pythoncmd', default=True,
variant('pythoncmd', default=not is_windows,
description="Symlink 'python3' executable to 'python' "
"(not PEP 394 compliant)")
# Optional Python modules
variant('readline', default=True, description='Build readline module')
variant('readline', default=not is_windows, description='Build readline module')
variant('ssl', default=True, description='Build ssl module')
variant('sqlite3', default=True, description='Build sqlite3 module')
variant('dbm', default=True, description='Build dbm module')
@ -166,6 +182,7 @@ class Python(AutotoolsPackage):
variant('tix', default=False, description='Build Tix module')
variant('ensurepip', default=True, description='Build ensurepip module', when='@2.7.9:2,3.4:')
if os.name != 'nt':
depends_on('pkgconfig@0.9.0:', type='build')
depends_on('gettext +libxml2', when='+libxml2')
depends_on('gettext ~libxml2', when='~libxml2')
@ -211,6 +228,7 @@ class Python(AutotoolsPackage):
patch('python-3.7.3-distutils-C++.patch', when='@3.7.3')
patch('python-3.7.4+-distutils-C++.patch', when='@3.7.4:')
patch('python-3.7.4+-distutils-C++-testsuite.patch', when='@3.7.4:')
patch('cpython-windows-externals.patch', when='@:3.9.6 platform=windows')
patch('tkinter.patch', when='@:2.8,3.3:3.7 platform=darwin')
# Patch the setup script to deny that tcl/x11 exists rather than allowing
@ -438,6 +456,70 @@ def flag_handler(self, name, flags):
# allow flags to be passed through compiler wrapper
return (flags, None, None)
@property
def configure_directory(self):
"""Returns the directory where 'configure' resides.
:return: directory where to find configure
"""
return self.stage.source_path
@property
def build_directory(self):
"""Override to provide another place to build the package"""
return self.configure_directory
@property
def plat_arch(self):
arch = platform.machine()
if arch in arch_map:
arch = arch_map[arch]
return arch
@property
def win_build_params(self):
args = []
args.append("-p %s" % self.plat_arch)
if self.spec.satisfies('+debug'):
args.append('-d')
if self.spec.satisfies('~ctypes'):
args.append('--no-ctypes')
if self.spec.satisfies('~ssl'):
args.append('--no-ssl')
if self.spec.satisfies('~tkinter'):
args.append('--no-tkinter')
return args
def win_installer(self, prefix):
proj_root = self.stage.source_path
pcbuild_root = os.path.join(proj_root, "PCbuild")
build_root = os.path.join(pcbuild_root, platform.machine().lower())
include_dir = os.path.join(proj_root, "Include")
copy_tree(include_dir, prefix.include)
doc_dir = os.path.join(proj_root, "Doc")
copy_tree(doc_dir, prefix.Doc)
tools_dir = os.path.join(proj_root, "Tools")
copy_tree(tools_dir, prefix.Tools)
lib_dir = os.path.join(proj_root, "Lib")
copy_tree(lib_dir, prefix.Lib)
pyconfig = os.path.join(proj_root, "PC", "pyconfig.h")
copy(pyconfig, prefix.include)
shared_libraries = []
shared_libraries.extend(glob.glob("%s\\*.exe" % build_root))
shared_libraries.extend(glob.glob("%s\\*.dll" % build_root))
shared_libraries.extend(glob.glob("%s\\*.pyd" % build_root))
os.makedirs(prefix.DLLs)
for lib in shared_libraries:
file_name = os.path.basename(lib)
if file_name.endswith(".exe") or\
(file_name.endswith(".dll") and "python" in file_name)\
or "vcruntime" in file_name:
copy(lib, prefix)
else:
copy(lib, prefix.DLLs)
static_libraries = glob.glob("%s\\*.lib")
for lib in static_libraries:
copy(lib, prefix.libs)
def configure_args(self):
spec = self.spec
config_args = []
@ -549,6 +631,54 @@ def configure_args(self):
return config_args
def configure(self, spec, prefix):
"""Runs configure with the arguments specified in
:meth:`~spack.build_systems.autotools.AutotoolsPackage.configure_args`
and an appropriately set prefix.
"""
with working_dir(self.build_directory, create=True):
if is_windows:
pass
else:
options = getattr(self, 'configure_flag_args', [])
options += ['--prefix={0}'.format(prefix)]
options += self.configure_args()
configure(*options)
def build(self, spec, prefix):
"""Makes the build targets specified by
:py:attr:``~.AutotoolsPackage.build_targets``
"""
# Windows builds use a batch script to drive
# configure and build in one step
with working_dir(self.build_directory):
if is_windows:
pcbuild_root = os.path.join(self.stage.source_path, "PCbuild")
builder_cmd = os.path.join(pcbuild_root, 'build.bat')
try:
subprocess.check_output( # novermin
" ".join([builder_cmd] + self.win_build_params),
stderr=subprocess.STDOUT
)
except subprocess.CalledProcessError as e:
raise ProcessError("Process exited with status %d" % e.returncode,
long_message=e.output.decode('utf-8'))
else:
# See https://autotools.io/automake/silent.html
params = ['V=1']
params += self.build_targets
inspect.getmodule(self).make(*params)
def install(self, spec, prefix):
"""Makes the install targets specified by
:py:attr:``~.AutotoolsPackage.install_targets``
"""
with working_dir(self.build_directory):
if is_windows:
self.win_installer(prefix)
else:
inspect.getmodule(self).make(*self.install_targets)
@run_after('install')
def filter_compilers(self):
"""Run after install to tell the configuration files and Makefiles
@ -557,7 +687,8 @@ def filter_compilers(self):
If this isn't done, they'll have CC and CXX set to Spack's generic
cc and c++. We want them to be bound to whatever compiler
they were built with."""
if is_windows:
return
kwargs = {'ignore_absent': True, 'backup': False, 'string': True}
filenames = [
@ -570,6 +701,8 @@ def filter_compilers(self):
@run_after('install')
def symlink(self):
if is_windows:
return
spec = self.spec
prefix = self.prefix
@ -759,6 +892,26 @@ def config_vars(self):
Returns:
dict: variable definitions
"""
# Some values set by sysconfig may not always exist on Windows, so
# compute Windows alternatives
def repair_win_sysconf(conf):
if is_windows:
conf["LIBDIR"] = os.path.join(conf["LIBDEST"], "..", "libs")
conf["LIBPL"] = conf["LIBDIR"]
conf["PYTHONFRAMEWORKPREFIX"] = ""
conf["LDLIBRARY"] = "python" + conf["VERSION"] + ".dll"
conf["LIBRARY"] = "python" + conf["VERSION"] + ".lib"
conf["CC"] = ""
conf["CXX"] = ""
conf["LDSHARED"] = ""
conf["LDCXXSHARED"] = ""
return conf
# TODO: distutils is deprecated in Python 3.10 and will be removed in
# Python 3.12, find a different way to access this information.
# Also, calling the python executable disallows us from cross-compiling,
# so we want to try to avoid that if possible.
cmd = """
import json
from sysconfig import (
@ -823,7 +976,7 @@ def config_vars(self):
config.update(json.loads(self.command('-c', cmd, output=str)))
except (ProcessError, RuntimeError):
pass
self._config_vars[dag_hash] = config
self._config_vars[dag_hash] = repair_win_sysconf(config)
return self._config_vars[dag_hash]
def get_sysconfigdata_name(self):
@ -865,6 +1018,9 @@ def libs(self):
# In Ubuntu 16.04.6 and python 2.7.12 from the system, lib could be
# in LBPL
# https://mail.python.org/pipermail/python-dev/2013-April/125733.html
# LIBPL does not exist in Windows, avoid uneccesary KeyError while allowing
# later failures.
# Return empty string rather than none so os.path doesn't complain
libpl = self.config_vars['LIBPL']
# The system Python installation on macOS and Homebrew installations
@ -881,7 +1037,7 @@ def libs(self):
if '+shared' in self.spec:
ldlibrary = self.config_vars['LDLIBRARY']
win_bin_dir = self.config_vars['BINDIR']
if os.path.exists(os.path.join(libdir, ldlibrary)):
return LibraryList(os.path.join(libdir, ldlibrary))
elif os.path.exists(os.path.join(libpl, ldlibrary)):
@ -891,6 +1047,9 @@ def libs(self):
elif macos_developerdir and \
os.path.exists(os.path.join(macos_developerdir, ldlibrary)):
return LibraryList(os.path.join(macos_developerdir, ldlibrary))
elif is_windows and \
os.path.exists(os.path.join(win_bin_dir, ldlibrary)):
return LibraryList(os.path.join(win_bin_dir, ldlibrary))
else:
msg = 'Unable to locate {0} libraries in {1}'
raise RuntimeError(msg.format(ldlibrary, libdir))
@ -1075,7 +1234,7 @@ def setup_dependent_build_environment(self, env, dependent_spec):
# fact that LDSHARED is set in the environment, therefore we export
# the variable only if the new value is different from what we got
# from the sysconfigdata file:
if config_link != new_link:
if config_link != new_link and not is_windows:
env.set(link_var, new_link)
def setup_dependent_run_environment(self, env, dependent_spec):
@ -1211,7 +1370,8 @@ def deactivate(self, ext_pkg, view, **args):
))
def add_files_to_view(self, view, merge_map):
bin_dir = self.spec.prefix.bin
bin_dir = self.spec.prefix.bin if os.name != 'nt'\
else self.spec.prefix
for src, dst in merge_map.items():
if not path_contains_subdirectory(src, bin_dir):
view.link(src, dst, spec=self.spec)
@ -1243,7 +1403,8 @@ def add_files_to_view(self, view, merge_map):
view.link(new_link_target, dst, spec=self.spec)
def remove_files_from_view(self, view, merge_map):
bin_dir = self.spec.prefix.bin
bin_dir = self.spec.prefix.bin if os.name != 'nt'\
else self.spec.prefix
for src, dst in merge_map.items():
if not path_contains_subdirectory(src, bin_dir):
view.remove_file(src, dst)

View file

@ -5,8 +5,12 @@
import re
from spack import *
class Ruby(AutotoolsPackage):
is_windows = str(spack.platforms.host()) == 'windows'
class Ruby(Package):
"""A dynamic, open source programming language with a focus on
simplicity and productivity."""
@ -27,23 +31,22 @@ class Ruby(AutotoolsPackage):
version('2.5.3', sha256='9828d03852c37c20fa333a0264f2490f07338576734d910ee3fd538c9520846c')
version('2.2.0', sha256='7671e394abfb5d262fbcd3b27a71bf78737c7e9347fa21c39e58b0bb9c4840fc')
if not is_windows:
variant('openssl', default=True, description="Enable OpenSSL support")
variant('readline', default=False, description="Enable Readline support")
extendable = True
depends_on('pkgconfig', type=('build'))
depends_on('libffi')
depends_on('zlib')
depends_on('libx11', when='@:2.3')
depends_on('tcl', when='@:2.3')
depends_on('tk', when='@:2.3')
depends_on('readline', when='+readline')
depends_on('zlib')
with when('+openssl'):
depends_on('openssl@:1')
depends_on('openssl@:1.0', when='@:2.3')
extendable = True
phases = ['autoreconf', 'configure', 'build', 'install']
# Known build issues when Avira antivirus software is running:
# https://github.com/rvm/rvm/issues/4313#issuecomment-374020379
# TODO: add check for this and warn user
@ -116,6 +119,32 @@ def setup_dependent_package(self, module, dependent_spec):
module.gem = Executable(self.prefix.bin.gem)
module.rake = Executable(self.prefix.bin.rake)
def configure(self, spec, prefix):
with working_dir(self.build_directory, create=True):
if is_windows:
Executable("win32\\configure.bat", "--prefix=%s" % self.prefix)
else:
options = getattr(self, 'configure_flag_args', [])
options += ['--prefix={0}'.format(prefix)]
options += self.configure_args()
configure(*options)
def build(self, spec, prefix):
with working_dir(self.build_directory):
if is_windows:
nmake()
else:
params = ['V=1']
params += self.build_targets
make(*params)
def install(self, spec, prefix):
with working_dir(self.build_directory):
if is_windows:
nmake('install')
else:
make(*self.install_targets)
@run_after('install')
def post_install(self):
""" RubyGems updated their SSL certificates at some point, so

View file

@ -6,7 +6,11 @@
# Although zlib comes with a configure script, it does not use Autotools
# The AutotoolsPackage causes zlib to fail to build with PGI
class Zlib(CMakePackage):
import glob
import os
class Zlib(Package):
"""A free, general-purpose, legally unencumbered lossless
data-compression library.
"""
@ -37,16 +41,29 @@ def libs(self):
['libz'], root=self.prefix, recursive=True, shared=shared
)
def cmake_args(self):
args = ['-DBUILD_SHARED_LIBS:BOOL=' +
('ON' if self._building_shared else 'OFF')]
return args
def win_install(self):
build_dir = self.stage.source_path
install_tree = {}
install_tree["bin"] = glob.glob(os.path.join(build_dir, "*.dll"))
install_tree["lib"] = glob.glob(os.path.join(build_dir, "*.lib"))
compose_src_path = lambda x: os.path.join(build_dir, x)
install_tree["include"] = [compose_src_path("zlib.h"),
compose_src_path("zconf.h")]
install_tree["share"] = {"man": {"man3": [compose_src_path("zlib.3")]}}
@property
def build_directory(self):
return join_path(self.stage.source_path,
'spack-build-shared' if self._building_shared
else 'spack-build-static')
def installtree(dst, tree):
for inst_dir in tree:
if type(tree[inst_dir]) is list:
install_dst = getattr(dst, inst_dir)
try:
os.makedirs(install_dst)
except OSError:
pass
for file in tree[inst_dir]:
copy(file, install_dst)
else:
installtree(getattr(dst, inst_dir), tree[inst_dir])
installtree(self.prefix, install_tree)
def setup_build_environment(self, env):
if '+pic' in self.spec:
@ -54,27 +71,10 @@ def setup_build_environment(self, env):
if '+optimize' in self.spec:
env.append_flags('CFLAGS', '-O2')
# Build, install, and check both static and shared versions of the
# libraries when +shared
@when('+shared platform=windows')
def cmake(self, spec, prefix):
for self._building_shared in (False, True):
super(Zlib, self).cmake(spec, prefix)
@when('+shared platform=windows')
def build(self, spec, prefix):
for self._building_shared in (False, True):
super(Zlib, self).build(spec, prefix)
@when('+shared platform=windows')
def check(self):
for self._building_shared in (False, True):
super(Zlib, self).check()
def install(self, spec, prefix):
if 'platform=windows' in self.spec and '+shared' in self.spec:
for self._building_shared in (False, True):
super(Zlib, self).install(spec, prefix)
if 'platform=windows' in self.spec:
nmake('-f' 'win32\\Makefile.msc')
self.win_install()
else:
config_args = []
if '~shared' in spec: