From 3a994032f8143a2bdbfe5c61baeb12b1bd2fed60 Mon Sep 17 00:00:00 2001 From: John Parent Date: Fri, 22 Oct 2021 14:00:02 -0400 Subject: [PATCH] Spack on Windows package ports CMake - Windows Bootstrap (#25825) Remove hardcoded cmake compiler (#26410) Revert breaking cmake changes Ensure no autotools on Windows Perl on Windows (#26612) Python source build windows (#26313) Reconfigure sysconf for Windows Python2.6 compatibility Fxixup new sbang tests for windows Ruby support (#28287) Add NASM support (#28319) Add mock Ninja package for testing --- bin/spack_cmd.bat | 8 +- lib/spack/docs/getting_started.rst | 8 - lib/spack/llnl/util/tty/__init__.py | 2 +- lib/spack/spack/build_environment.py | 4 +- lib/spack/spack/build_systems/autotools.py | 3 +- lib/spack/spack/build_systems/cmake.py | 30 +-- lib/spack/spack/cmd/unit_test.py | 2 +- lib/spack/spack/compilers/msvc.py | 70 ++++-- lib/spack/spack/detection/path.py | 1 - lib/spack/spack/fetch_strategy.py | 5 +- lib/spack/spack/platforms/_platform.py | 5 + lib/spack/spack/test/bootstrap.py | 2 +- lib/spack/spack/test/cmd/unit_test.py | 4 +- lib/spack/spack/test/cmd/url.py | 1 + lib/spack/spack/test/relocate.py | 1 + lib/spack/spack/test/sbang.py | 13 + .../builtin.mock/packages/ninja/package.py | 15 ++ .../repos/builtin/packages/cmake/package.py | 125 +++++++--- .../builtin/packages/nasm/msvc.mak.patch | 18 ++ .../repos/builtin/packages/nasm/package.py | 33 ++- .../builtin/packages/netlib-lapack/package.py | 1 - .../repos/builtin/packages/openssl/package.py | 5 +- .../repos/builtin/packages/perl/package.py | 107 ++++++-- .../python/cpython-windows-externals.patch | 28 +++ .../repos/builtin/packages/python/package.py | 231 +++++++++++++++--- .../repos/builtin/packages/ruby/package.py | 61 +++-- .../repos/builtin/packages/zlib/package.py | 60 ++--- 27 files changed, 646 insertions(+), 197 deletions(-) create mode 100644 var/spack/repos/builtin.mock/packages/ninja/package.py create mode 100644 var/spack/repos/builtin/packages/nasm/msvc.mak.patch create mode 100644 var/spack/repos/builtin/packages/python/cpython-windows-externals.patch diff --git a/bin/spack_cmd.bat b/bin/spack_cmd.bat index 5ba685fa46..56b6024665 100644 --- a/bin/spack_cmd.bat +++ b/bin/spack_cmd.bat @@ -15,8 +15,12 @@ popd :: Check if Python is on the PATH if not defined python_pf_ver ( -(for /f "delims=" %%F in ('where python.exe') do (set python_pf_ver=%%F) ) 2> NUL +(for /f "delims=" %%F in ('where python.exe') do ( + set python_pf_ver=%%F + goto :found_python + ) ) 2> NUL ) +:found_python if not defined python_pf_ver ( :: If not, look for Python from the Spack installer :get_builtin @@ -35,7 +39,7 @@ if not defined python_pf_ver ( :: Python is already on the path set py_exe=!python_pf_ver! (for /F "tokens=* USEBACKQ" %%F in ( - `!py_exe! --version`) do (set "output=%%F")) 2>NUL + `"!py_exe!" --version`) do (set "output=%%F")) 2>NUL if not "!output:Microsoft Store=!"=="!output!" goto :get_builtin goto :exitpoint ) diff --git a/lib/spack/docs/getting_started.rst b/lib/spack/docs/getting_started.rst index a904f0416a..8ba0a799a9 100644 --- a/lib/spack/docs/getting_started.rst +++ b/lib/spack/docs/getting_started.rst @@ -1587,14 +1587,6 @@ When given the option of adjusting your ``PATH``, choose the ``Git from the command line and also from 3rd-party software`` option. This will automatically update your ``PATH`` variable to include the ``git`` command. -"""" -Perl -"""" - -Perl is a flexible and feature-rich programming language that comes built-in -on Unix boxes but needs to be installed externally for Windows users. Fortunately, -you can find the most recent release at https://www.perl.org/get.html. - """" NASM """" diff --git a/lib/spack/llnl/util/tty/__init__.py b/lib/spack/llnl/util/tty/__init__.py index b244b8a58d..5b8729d7ec 100644 --- a/lib/spack/llnl/util/tty/__init__.py +++ b/lib/spack/llnl/util/tty/__init__.py @@ -146,7 +146,7 @@ def process_stacktrace(countback): file_list = [] for frame in st: # Check that the file is a spack file - if frame[0].find("/spack") >= 0: + if frame[0].find(os.path.sep + "spack") >= 0: file_list.append(frame[0]) # We use commonprefix to find what the spack 'root' directory is. root_dir = os.path.commonprefix(file_list) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 7888dee139..3281a24013 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -528,7 +528,9 @@ def _set_variables_for_single_module(pkg, module): m.cmake = Executable('cmake') m.ctest = MakeExecutable('ctest', jobs) - # Standard build system arguments + if os.name == 'nt': + m.nmake = Executable('nmake') + # Standard CMake arguments m.std_cmake_args = spack.build_systems.cmake.CMakePackage._std_args(pkg) m.std_meson_args = spack.build_systems.meson.MesonPackage._std_args(pkg) m.std_pip_args = spack.build_systems.python.PythonPackage._std_args(pkg) diff --git a/lib/spack/spack/build_systems/autotools.py b/lib/spack/spack/build_systems/autotools.py index 46a3ccddf7..a644c92622 100644 --- a/lib/spack/spack/build_systems/autotools.py +++ b/lib/spack/spack/build_systems/autotools.py @@ -14,7 +14,7 @@ from llnl.util.filesystem import force_remove, working_dir from spack.build_environment import InstallError -from spack.directives import depends_on +from spack.directives import conflicts, depends_on from spack.operating_systems.mac_os import macos_version from spack.package import PackageBase, run_after, run_before from spack.util.executable import Executable @@ -104,6 +104,7 @@ def patch_config_files(self): depends_on('gnuconfig', type='build', when='target=ppc64le:') depends_on('gnuconfig', type='build', when='target=aarch64:') depends_on('gnuconfig', type='build', when='target=riscv64:') + conflicts('platform=windows') @property def _removed_la_files_log(self): diff --git a/lib/spack/spack/build_systems/cmake.py b/lib/spack/spack/build_systems/cmake.py index 979a6bb024..591d73a5c4 100644 --- a/lib/spack/spack/build_systems/cmake.py +++ b/lib/spack/spack/build_systems/cmake.py @@ -93,15 +93,11 @@ class CMakePackage(PackageBase): #: See https://cmake.org/cmake/help/latest/manual/cmake-generators.7.html #: for more information. - variant('generator', - default='Make' if sys.platform != 'win32' else 'Ninja', - description='Build system to generate', - values=('Make', 'Ninja')) + generator = "Unix Makefiles" - depends_on('ninja', when='generator=Ninja') - - generatorMap = {'Make': 'Unix Makefiles', - 'Ninja': 'Ninja'} + if sys.platform == 'win32': + generator = "Ninja" + depends_on('ninja') # https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html variant('build_type', default='RelWithDebInfo', @@ -150,10 +146,10 @@ def _std_args(pkg): """Computes the standard cmake arguments for a generic package""" try: - pkg.generator = pkg.spec.variants['generator'].value - except KeyError: - pkg.generator = 'Make' if sys.platform != 'win32' else 'Ninja' - primary_generator = CMakePackage.generatorMap[pkg.generator] + if not pkg.generator: + raise AttributeError + except AttributeError: + pkg.generator = CMakePackage.generator try: build_type = pkg.spec.variants['build_type'].value @@ -167,22 +163,16 @@ def _std_args(pkg): define = CMakePackage.define args = [ - '-G', primary_generator, + '-G', pkg.generator, define('CMAKE_INSTALL_PREFIX', pkg.prefix.replace('\\', '/')), define('CMAKE_BUILD_TYPE', build_type), - define('CMAKE_C_COMPILER:FILEPATH', pkg.compiler.cc.replace('\\', '/')), - define('CMAKE_CXX_COMPILER:FILEPATH', pkg.compiler.cxx.replace('\\', '/')) ] - if pkg.compiler.fc is not None: - args.append(define('CMAKE_Fortran_COMPILER:FILEPATH', - pkg.compiler.fc.replace('\\', '/'))) - # CMAKE_INTERPROCEDURAL_OPTIMIZATION only exists for CMake >= 3.9 if pkg.spec.satisfies('^cmake@3.9:'): args.append(define('CMAKE_INTERPROCEDURAL_OPTIMIZATION', ipo)) - if primary_generator == 'Unix Makefiles': + if pkg.generator == 'Unix Makefiles': args.append(define('CMAKE_VERBOSE_MAKEFILE', True)) if platform.mac_ver()[0]: diff --git a/lib/spack/spack/cmd/unit_test.py b/lib/spack/spack/cmd/unit_test.py index 0faa05b834..05c308b84c 100644 --- a/lib/spack/spack/cmd/unit_test.py +++ b/lib/spack/spack/cmd/unit_test.py @@ -130,7 +130,7 @@ def colorize(c, prefix): # in the future - so this manipulation might be fragile if nodetype.lower() == 'function': name_parts.append(item) - key_end = os.path.join(*[x[1] for x in key_parts]) + key_end = os.path.join(*key_parts[-1][1].split('/')) key = next(f for f in files if f.endswith(key_end)) tests[key].add(tuple(x[1] for x in name_parts)) elif nodetype.lower() == 'class': diff --git a/lib/spack/spack/compilers/msvc.py b/lib/spack/spack/compilers/msvc.py index d0ae6921ea..f51583caf4 100644 --- a/lib/spack/spack/compilers/msvc.py +++ b/lib/spack/spack/compilers/msvc.py @@ -4,13 +4,37 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os +import re import subprocess import sys -from typing import List # novm +from distutils.version import StrictVersion +from typing import Dict, List, Set # novm import spack.operating_systems.windows_os import spack.util.executable from spack.compiler import Compiler +from spack.error import SpackError + +avail_fc_version = set() # type: Set[str] +fc_path = dict() # type: Dict[str, str] + +fortran_mapping = { + '2021.3.0': '19.29.30133', + '2021.2.1': '19.28.29913', + '2021.2.0': '19.28.29334', + '2021.1.0': '19.28.29333', +} + + +def get_valid_fortran_pth(comp_ver): + cl_ver = str(comp_ver).split('@')[1] + sort_fn = lambda fc_ver: StrictVersion(fc_ver) + sort_fc_ver = sorted(list(avail_fc_version), key=sort_fn) + for ver in sort_fc_ver: + if ver in fortran_mapping: + if StrictVersion(cl_ver) <= StrictVersion(fortran_mapping[ver]): + return fc_path[ver] + return None class Msvc(Compiler): @@ -46,6 +70,8 @@ class Msvc(Compiler): # file based on compiler executable path. def __init__(self, *args, **kwargs): + new_pth = [pth if pth else get_valid_fortran_pth(args[0]) for pth in args[3]] + args[3][:] = new_pth super(Msvc, self).__init__(*args, **kwargs) if os.getenv("ONEAPI_ROOT"): # If this found, it sets all the vars @@ -65,6 +91,12 @@ def verbose_flag(self): def pic_flag(self): return "" + @property + def msvc_version(self): + ver = re.search(Msvc.version_regex, self.cc).group(1) + ver = "".join(ver.split('.')[:2])[:-1] + return "MSVC" + ver + def setup_custom_environment(self, pkg, env): """Set environment variables for MSVC using the Microsoft-provided script.""" @@ -81,44 +113,40 @@ def setup_custom_environment(self, pkg, env): if sys.version_info[0] >= 3: out = out.decode('utf-16le', errors='replace') # novermin - int_env = { # novermin - key.lower(): value - for key, _, value in - (line.partition('=') for line in out.splitlines()) - if key and value - } + int_env = dict((key.lower(), value) for key, _, value in + (line.partition('=') for line in out.splitlines()) + if key and value) if 'path' in int_env: env.set_path('PATH', int_env['path'].split(';')) env.set_path('INCLUDE', int_env.get('include', '').split(';')) env.set_path('LIB', int_env.get('lib', '').split(';')) + + env.set('CC', self.cc) + env.set('CXX', self.cxx) + env.set('FC', self.fc) + env.set('F77', self.f77) else: # Should not this be an exception? print("Cannot pull msvc compiler information in Python 2.6 or below") - # fc_version only loads the ifx compiler into the first MSVC stanza; - # if there are other versions of Microsoft VS installed and detected, they - # will only have cl.exe as the C/C++ compiler - @classmethod def fc_version(cls, fc): # We're using intel for the Fortran compilers, which exist if # ONEAPI_ROOT is a meaningful variable + fc_ver = cls.default_version(fc) + avail_fc_version.add(fc_ver) + fc_path[fc_ver] = fc if os.getenv("ONEAPI_ROOT"): try: sps = spack.operating_systems.windows_os.WindowsOs.compiler_search_paths - except Exception: - print("sps not found.") - raise - try: - clp = spack.util.executable.which_string("cl", path=sps) - except Exception: - print("cl not found.") - raise + except AttributeError: + raise SpackError("Windows compiler search paths not established") + clp = spack.util.executable.which_string("cl", path=sps) ver = cls.default_version(clp) - return ver else: - return cls.default_version(fc) + ver = fc_ver + return ver @classmethod def f77_version(cls, f77): diff --git a/lib/spack/spack/detection/path.py b/lib/spack/spack/detection/path.py index f280af4520..ee98e23841 100644 --- a/lib/spack/spack/detection/path.py +++ b/lib/spack/spack/detection/path.py @@ -12,7 +12,6 @@ import sys import warnings - import llnl.util.filesystem import llnl.util.tty diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index f9823fb648..2435da4b2f 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -353,8 +353,9 @@ def _existing_url(self, url): # Telling urllib to check if url is accessible try: url, headers, response = spack.util.web.read_from_url(url) - except spack.util.web.SpackWebError: - msg = "Urllib fetch failed to verify url {0}".format(url) + except spack.util.web.SpackWebError as werr: + msg = "Urllib fetch failed to verify url\ + {0}\n with error {1}".format(url, werr) raise FailedDownloadError(url, msg) return (response.getcode() is None or response.getcode() == 200) diff --git a/lib/spack/spack/platforms/_platform.py b/lib/spack/spack/platforms/_platform.py index b68b7f1299..a19381b8d4 100644 --- a/lib/spack/spack/platforms/_platform.py +++ b/lib/spack/spack/platforms/_platform.py @@ -2,6 +2,8 @@ # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +import platform + import llnl.util.lang import spack.error @@ -143,3 +145,6 @@ def oses(): for o in sorted(self.operating_sys.values()): yield o._cmp_iter yield oses + + def is_64bit(self): + return platform.machine().endswith('64') diff --git a/lib/spack/spack/test/bootstrap.py b/lib/spack/spack/test/bootstrap.py index 8b72a98ad3..d753e2cdfa 100644 --- a/lib/spack/spack/test/bootstrap.py +++ b/lib/spack/spack/test/bootstrap.py @@ -131,7 +131,7 @@ def test_custom_store_in_environment(mutable_config, tmpdir): # Don't trigger evaluation here with spack.bootstrap.ensure_bootstrap_configuration(): pass - assert str(spack.store.root) == '/tmp/store' + assert str(spack.store.root) == sep + os.path.join('tmp', 'store') def test_nested_use_of_context_manager(mutable_config): diff --git a/lib/spack/spack/test/cmd/unit_test.py b/lib/spack/spack/test/cmd/unit_test.py index 276996501b..aa31282fdb 100644 --- a/lib/spack/spack/test/cmd/unit_test.py +++ b/lib/spack/spack/test/cmd/unit_test.py @@ -3,10 +3,12 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +import os + from spack.main import SpackCommand spack_test = SpackCommand('unit-test') -cmd_test_py = 'lib/spack/spack/test/cmd/unit_test.py' +cmd_test_py = os.path.join('lib', 'spack', 'spack', 'test', 'cmd', 'unit_test.py') def test_list(): diff --git a/lib/spack/spack/test/cmd/url.py b/lib/spack/spack/test/cmd/url.py index 204f3a81b1..1e4c0d391a 100644 --- a/lib/spack/spack/test/cmd/url.py +++ b/lib/spack/spack/test/cmd/url.py @@ -3,6 +3,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import re +import sys import pytest diff --git a/lib/spack/spack/test/relocate.py b/lib/spack/spack/test/relocate.py index a64052c7c8..4ef9ae4bb3 100644 --- a/lib/spack/spack/test/relocate.py +++ b/lib/spack/spack/test/relocate.py @@ -2,6 +2,7 @@ # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +import os import os.path import re import shutil diff --git a/lib/spack/spack/test/sbang.py b/lib/spack/spack/test/sbang.py index aa9d9b10dd..f0cc1835d9 100644 --- a/lib/spack/spack/test/sbang.py +++ b/lib/spack/spack/test/sbang.py @@ -11,6 +11,7 @@ import os import shutil import stat +import sys import tempfile import pytest @@ -193,6 +194,8 @@ def test_shebang_interpreter_regex(shebang, interpreter): sbang.get_interpreter(shebang) == interpreter +@pytest.mark.skipif(sys.platform == 'win32', + reason="Not supported on Windows (yet)") def test_shebang_handling(script_dir, sbang_line): sbang.filter_shebangs_in_directory(script_dir.tempdir) @@ -339,6 +342,8 @@ def test_install_user_sbang(install_mockery, configure_user_perms): run_test_install_sbang(False) +@pytest.mark.skipif(sys.platform == 'win32', + reason="Not supported on Windows (yet)") def test_install_sbang_too_long(tmpdir): root = str(tmpdir) num_extend = sbang.system_shebang_limit - len(root) - len('/bin/sbang') @@ -357,6 +362,8 @@ def test_install_sbang_too_long(tmpdir): assert 'cannot patch' in err +@pytest.mark.skipif(sys.platform == 'win32', + reason="Not supported on Windows (yet)") def test_sbang_hook_skips_nonexecutable_blobs(tmpdir): # Write a binary blob to non-executable.sh, with a long interpreter "path" # consisting of invalid UTF-8. The latter is technically not really necessary for @@ -374,6 +381,8 @@ def test_sbang_hook_skips_nonexecutable_blobs(tmpdir): assert b'sbang' not in f.readline() +@pytest.mark.skipif(sys.platform == 'win32', + reason="Not supported on Windows (yet)") def test_sbang_handles_non_utf8_files(tmpdir): # We have an executable with a copyright sign as filename contents = (b'#!' + b'\xa9' * sbang.system_shebang_limit + @@ -408,6 +417,8 @@ def shebang_limits_system_8_spack_16(): sbang.spack_shebang_limit = spack_limit +@pytest.mark.skipif(sys.platform == 'win32', + reason="Not supported on Windows (yet)") def test_shebang_exceeds_spack_shebang_limit(shebang_limits_system_8_spack_16, tmpdir): """Tests whether shebangs longer than Spack's limit are skipped""" file = str(tmpdir.join('longer_than_spack_limit.sh')) @@ -421,6 +432,8 @@ def test_shebang_exceeds_spack_shebang_limit(shebang_limits_system_8_spack_16, t assert b'sbang' not in f.read() +@pytest.mark.skipif(sys.platform == 'win32', + reason="Not supported on Windows (yet)") def test_sbang_hook_handles_non_writable_files_preserving_permissions(tmpdir): path = str(tmpdir.join('file.sh')) with open(path, 'w') as f: diff --git a/var/spack/repos/builtin.mock/packages/ninja/package.py b/var/spack/repos/builtin.mock/packages/ninja/package.py new file mode 100644 index 0000000000..3ed308b3c0 --- /dev/null +++ b/var/spack/repos/builtin.mock/packages/ninja/package.py @@ -0,0 +1,15 @@ +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +from spack import * + + +class Ninja(Package): + """Dummy Ninja Package""" + + homepage = "https://ninja-build.org/" + url = "https://github.com/ninja-build/ninja/archive/v1.7.2.tar.gz" + + version('1.10.2', sha256='ce35865411f0490368a8fc383f29071de6690cbadc27704734978221f25e2bed') diff --git a/var/spack/repos/builtin/packages/cmake/package.py b/var/spack/repos/builtin/packages/cmake/package.py index e91b6eacdc..e431dc62e5 100644 --- a/var/spack/repos/builtin/packages/cmake/package.py +++ b/var/spack/repos/builtin/packages/cmake/package.py @@ -3,9 +3,13 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +import os import re +import shutil +import sys import spack.build_environment +from spack import * class Cmake(Package): @@ -141,6 +145,12 @@ class Cmake(Package): # https://gitlab.kitware.com/cmake/cmake/merge_requests/4075 patch('fix-xlf-ninja-mr-4075.patch', sha256="42d8b2163a2f37a745800ec13a96c08a3a20d5e67af51031e51f63313d0dedd1", when="@3.15.5") + generator = "Unix Makefiles" + + if sys.platform == 'win32': + generator = "Ninja" + depends_on('ninja') + # We default ownlibs to true because it greatly speeds up the CMake # build, and CMake is built frequently. Also, CMake is almost always # a build dependency, and its libs will not interfere with others in @@ -149,7 +159,7 @@ class Cmake(Package): variant('qt', default=False, description='Enables the build of cmake-gui') variant('doc', default=False, description='Enables the generation of html and man page documentation') variant('openssl', default=True, description="Enable openssl for curl bootstrapped by CMake when using +ownlibs") - variant('ncurses', default=True, description='Enables the build of the ncurses gui') + variant('ncurses', default=os.name != 'nt', description='Enables the build of the ncurses gui') # See https://gitlab.kitware.com/cmake/cmake/-/issues/21135 conflicts('%gcc platform=darwin', when='@:3.17', @@ -241,36 +251,47 @@ def flag_handler(self, name, flags): flags.append(self.compiler.cxx11_flag) return (flags, None, None) + def setup_build_environment(self, env): + spec = self.spec + if '+openssl' in spec: + env.set('OPENSSL_ROOT_DIR', spec['openssl'].prefix) + def bootstrap_args(self): spec = self.spec - args = [ - '--prefix={0}'.format(self.prefix), - '--parallel={0}'.format(make_jobs) - ] + args = [] + if not os.name == 'nt': + args.extend( + ['--prefix={0}'.format(self.prefix), + '--parallel={0}'.format(make_jobs)] + ) - if '+ownlibs' in spec: - # Build and link to the CMake-provided third-party libraries - args.append('--no-system-libs') + if '+ownlibs' in spec: + # Build and link to the CMake-provided third-party libraries + args.append('--no-system-libs') + else: + # Build and link to the Spack-installed third-party libraries + args.append('--system-libs') + + if spec.satisfies('@3.2:'): + # jsoncpp requires CMake to build + # use CMake-provided library to avoid circular dependency + args.append('--no-system-jsoncpp') + + if '+qt' in spec: + args.append('--qt-gui') + else: + args.append('--no-qt-gui') + + if '+doc' in spec: + args.append('--sphinx-html') + args.append('--sphinx-man') + + # Now for CMake arguments to pass after the initial bootstrap + args.append('--') else: - # Build and link to the Spack-installed third-party libraries - args.append('--system-libs') - - if spec.satisfies('@3.2:'): - # jsoncpp requires CMake to build - # use CMake-provided library to avoid circular dependency - args.append('--no-system-jsoncpp') - - if '+qt' in spec: - args.append('--qt-gui') - else: - args.append('--no-qt-gui') - - if '+doc' in spec: - args.append('--sphinx-html') - args.append('--sphinx-man') - - # Now for CMake arguments to pass after the initial bootstrap - args.append('--') + args.append('-DCMAKE_INSTALL_PREFIX=%s' % self.prefix) + if self.spec.satisfies('generator=Ninja'): + args.append('-GNinja') args.append('-DCMAKE_BUILD_TYPE={0}'.format( self.spec.variants['build_type'].value)) @@ -297,21 +318,61 @@ def bootstrap_args(self): return args + def winbootcmake(self, spec): + from spack import fetch_strategy, stage + urls = { + '3': ('https://cmake.org/files/v3.21/cmake-3.21.2-windows-x86_64.zip', "f21e72ede9d15070602b60b2c14dc779"), + '2': ('https://cmake.org/files/v2.8/cmake-2.8.4-win32-x86.zip', "a2525342e495518101381203bf4484c4") + } + if spec.satisfies('@3.0.2:'): + bootstrap_url = urls['3'] + else: + bootstrap_url = urls['2'] + remote = fetch_strategy.URLFetchStrategy(url=bootstrap_url[0], + checksum=bootstrap_url[1]) + bootstrap_stage_path = os.path.join(self.stage.path, "cmake-bootstraper") + with stage.Stage(remote, path=bootstrap_stage_path) as bootstrap_stage: + remote.stage = bootstrap_stage + remote.fetch() + remote.check() + remote.expand() + shutil.move(bootstrap_stage.source_path, self.stage.source_path) + + def cmake_bootstrap(self): + exe_prefix = self.stage.source_path + relative_cmake_exe = os.path.join('spack-src', 'bin', 'cmake.exe') + return Executable(os.path.join(exe_prefix, relative_cmake_exe)) + def bootstrap(self, spec, prefix): - bootstrap = Executable('./bootstrap') - bootstrap(*self.bootstrap_args()) + bootstrap_args = self.bootstrap_args() + if os.name == 'nt': + self.winbootcmake(spec) + bootstrap = self.cmake_bootstrap() + bootstrap_args.extend(['.']) + else: + bootstrap = Executable('./bootstrap') + bootstrap(*bootstrap_args) def build(self, spec, prefix): - make() + if self.generator == "Ninja": + ninja() + else: + make() @run_after('build') @on_package_attributes(run_tests=True) def build_test(self): # Some tests fail, takes forever - make('test') + if self.generator == "Ninja": + ninja('test') + else: + make('test') def install(self, spec, prefix): - make('install') + if self.generator == "Ninja": + ninja('install') + else: + make('install') if spec.satisfies('%fj'): for f in find(self.prefix, 'FindMPI.cmake', recursive=True): diff --git a/var/spack/repos/builtin/packages/nasm/msvc.mak.patch b/var/spack/repos/builtin/packages/nasm/msvc.mak.patch new file mode 100644 index 0000000000..cfc751c8cf --- /dev/null +++ b/var/spack/repos/builtin/packages/nasm/msvc.mak.patch @@ -0,0 +1,18 @@ +diff --git a/Mkfiles/msvc.mak b/Mkfiles/msvc1.mak +index 4f6121f..f0e6f4b 100644 +--- a/Mkfiles/msvc.mak ++++ b/Mkfiles/msvc1.mak +@@ -228,10 +228,10 @@ WARNFILES = asm\warnings.c include\warnings.h doc\warnings.src + + warnings: + $(RM_F) $(WARNFILES) +- $(MAKE) asm\warnings.time ++# $(MAKE) asm\warnings.time + +-asm\warnings.time: $(ALLOBJ:.@OBJEXT@=.c) +- : > asm\warnings.time ++# asm\warnings.time: $(ALLOBJ:.@OBJEXT@=.c) ++# : > asm\warnings.time + $(MAKE) $(WARNFILES) + + asm\warnings.c: asm\warnings.pl asm\warnings.time diff --git a/var/spack/repos/builtin/packages/nasm/package.py b/var/spack/repos/builtin/packages/nasm/package.py index 2f0c747a52..6bfa36a660 100644 --- a/var/spack/repos/builtin/packages/nasm/package.py +++ b/var/spack/repos/builtin/packages/nasm/package.py @@ -5,17 +5,20 @@ from spack import * +is_windows = str(spack.platforms.host()) == 'windows' -class Nasm(AutotoolsPackage): + +class Nasm(Package): """NASM (Netwide Assembler) is an 80x86 assembler designed for portability and modularity. It includes a disassembler as well.""" homepage = "https://www.nasm.us" - url = "https://www.nasm.us/pub/nasm/releasebuilds/2.14.02/nasm-2.14.02.tar.xz" + url = "https://www.nasm.us/pub/nasm/releasebuilds/2.14.02/nasm-2.14.02.tar.gz" list_url = "https://www.nasm.us/pub/nasm/releasebuilds" list_depth = 1 + a = '3caf6729c1073bf96629b57cee31eeb54f4f8129b01902c73428836550b30a3f' - version('2.15.05', sha256='3caf6729c1073bf96629b57cee31eeb54f4f8129b01902c73428836550b30a3f') + version('2.15.05', sha256='9182a118244b058651c576baa9d0366ee05983c4d4ae1d9ddd3236a9f2304997') version('2.14.02', sha256='e24ade3e928f7253aa8c14aa44726d1edf3f98643f87c9d72ec1df44b26be8f5') version('2.13.03', sha256='812ecfb0dcbc5bd409aaa8f61c7de94c5b8752a7b00c632883d15b2ed6452573') version('2.11.06', sha256='90f60d95a15b8a54bf34d87b9be53da89ee3d6213ea739fb2305846f4585868a') @@ -24,9 +27,13 @@ class Nasm(AutotoolsPackage): # https://bugzilla.nasm.us/show_bug.cgi?id=3392461 patch('https://src.fedoraproject.org/rpms/nasm/raw/0cc3eb244bd971df81a7f02bc12c5ec259e1a5d6/f/0001-Remove-invalid-pure_func-qualifiers.patch', level=1, sha256='ac9f315d204afa6b99ceefa1fe46d4eed2b8a23c7315d32d33c0f378d930e950', when='@2.13.03 %gcc@8:') + patch('msvc.mak.patch', when='platform=windows') + conflicts('%intel@:14', when='@2.14:', msg="Intel 14 has immature C11 support") + phases = ['configure', 'build', 'install'] + def patch(self): # Remove flags not recognized by the NVIDIA compiler if self.spec.satisfies('%nvhpc@:20.11'): @@ -34,3 +41,23 @@ def patch(self): 'CFLAGS="$pa_add_cflags__old_cflags"', 'configure') filter_file(r'CFLAGS="\$pa_add_flags__old_flags -Werror=.*"', 'CFLAGS="$pa_add_flags__old_flags"', 'configure') + + def configure(self, spec, prefix): + with working_dir(self.stage.source_path, create=True): + if not is_windows: + configure(['--prefix={0}'.format(self.prefix)]) + + def build(self, spec, prefix): + with working_dir(self.stage.source_path): + if is_windows: + touch('asm\\warnings.time') + nmake('/f', 'Mkfiles\\msvc.mak') + else: + make(['V=1']) + + def install(self, spec, prefix): + with working_dir(self.stage.source_path): + if is_windows: + pass + else: + make(['install']) diff --git a/var/spack/repos/builtin/packages/netlib-lapack/package.py b/var/spack/repos/builtin/packages/netlib-lapack/package.py index 5d527fedec..82af36426d 100644 --- a/var/spack/repos/builtin/packages/netlib-lapack/package.py +++ b/var/spack/repos/builtin/packages/netlib-lapack/package.py @@ -71,7 +71,6 @@ class NetlibLapack(CMakePackage): depends_on('blas', when='+external-blas') depends_on('netlib-xblas+fortran+plain_blas', when='+xblas') depends_on('python@2.7:', type='test') - depends_on('ninja@1.10.0:', when='platform=windows') # We need to run every phase twice in order to get static and shared # versions of the libraries. When ~shared, we run the default diff --git a/var/spack/repos/builtin/packages/openssl/package.py b/var/spack/repos/builtin/packages/openssl/package.py index f21b6b81a8..f57a65271a 100644 --- a/var/spack/repos/builtin/packages/openssl/package.py +++ b/var/spack/repos/builtin/packages/openssl/package.py @@ -5,6 +5,7 @@ import os import re +import sys import llnl.util.tty as tty @@ -96,6 +97,8 @@ class Openssl(Package): # Uses Fake Autotools, should subclass Package depends_on('perl@5.14.0:', type=('build', 'test')) depends_on('ca-certificates-mozilla', type=('build', 'run'), when='certs=mozilla') + conflicts('+dynamic', when=sys.platform != 'win32') + @classmethod def determine_version(cls, exe): output = Executable(exe)('version', output=str, error=str) @@ -151,7 +154,7 @@ def install(self, spec, prefix): 'CC=\"%s\"' % os.environ.get('SPACK_CC'), 'CXX=\"%s\"' % os.environ.get('SPACK_CXX'), '%s' % shared_flag, - 'VC-WIN64A') + 'VC-WIN64A', ignore_quotes=True) else: config = Executable('./config') config('--prefix=%s' % prefix, diff --git a/var/spack/repos/builtin/packages/perl/package.py b/var/spack/repos/builtin/packages/perl/package.py index 0d666b41d3..714c7e3a7b 100644 --- a/var/spack/repos/builtin/packages/perl/package.py +++ b/var/spack/repos/builtin/packages/perl/package.py @@ -11,15 +11,20 @@ # Author: Justin Too # Date: September 6, 2015 # + import os import re from contextlib import contextmanager from llnl.util.lang import match_predicate +from llnl.util.symlink import symlink from spack import * from spack.operating_systems.mac_os import macos_version +host = spack.platforms.host() +is_windows = str(host) == 'windows' + class Perl(Package): # Perl doesn't use Autotools, it should subclass Package """Perl 5 is a highly capable, feature-rich programming language with over @@ -64,15 +69,16 @@ class Perl(Package): # Perl doesn't use Autotools, it should subclass Package extendable = True - # Bind us below gdbm-1.20 due to API change: https://github.com/Perl/perl5/issues/18915 - depends_on('gdbm@:1.19') - # :5.28 needs gdbm@:1:14.1: https://rt-archive.perl.org/perl5/Ticket/Display.html?id=133295 - depends_on('gdbm@:1.14.1', when='@:5.28.0') - depends_on('berkeley-db') - depends_on('bzip2') - depends_on('zlib') - # :5.24.1 needs zlib@:1.2.8: https://rt.cpan.org/Public/Bug/Display.html?id=120134 - depends_on('zlib@:1.2.8', when='@5.20.3:5.24.1') + if not is_windows: + # Bind us below gdbm-1.20 due to API change: https://github.com/Perl/perl5/issues/18915 + depends_on('gdbm@:1.19') + # :5.28 needs gdbm@:1:14.1: https://rt-archive.perl.org/perl5/Ticket/Display.html?id=133295 + depends_on('gdbm@:1.14.1', when='@:5.28.0') + depends_on('berkeley-db') + depends_on('bzip2') + depends_on('zlib') + # :5.24.1 needs zlib@:1.2.8: https://rt.cpan.org/Public/Bug/Display.html?id=120134 + depends_on('zlib@:1.2.8', when='@5.20.3:5.24.1') # there has been a long fixed issue with 5.22.0 with regard to the ccflags # definition. It is well documented here: @@ -181,6 +187,23 @@ def do_stage(self, mirror_only=False): perm = os.stat(filename).st_mode os.chmod(filename, perm | 0o200) + @property + def nmake_arguments(self): + args = [] + if self.spec.satisfies('%msvc'): + args.append('CCTYPE=%s' % self.compiler.msvc_version) + else: + raise RuntimeError("Perl unsupported for non MSVC compilers on Windows") + args.append('INST_TOP="%s"' % self.prefix.replace('/', '\\')) + args.append("INST_ARCH=\\$(ARCHNAME)") + if self.spec.satisfies('~shared'): + args.append("ALL_STATIC=%s" % "define") + if self.spec.satisfies('~threads'): + args.extend(["USE_MULTI=undef", "USE_ITHREADS=undef", "USE_IMP_SYS=undef"]) + if not host.is_64bit(): + args.append("WIN64=undef") + return args + def configure_args(self): spec = self.spec prefix = self.prefix @@ -229,30 +252,69 @@ def configure_args(self): return config_args def configure(self, spec, prefix): + if is_windows: + return configure = Executable('./Configure') configure(*self.configure_args()) def build(self, spec, prefix): - make() + if is_windows: + pass + else: + make() @run_after('build') @on_package_attributes(run_tests=True) def build_test(self): - make('test') + if is_windows: + win32_dir = os.path.join(self.stage.source_path, "win32") + with working_dir(win32_dir): + nmake('test', ignore_quotes=True) + else: + make('test') def install(self, spec, prefix): - make('install') + if is_windows: + win32_dir = os.path.join(self.stage.source_path, "win32") + with working_dir(win32_dir): + nmake('install', *self.nmake_arguments, ignore_quotes=True) + else: + make('install') + + @run_after('install') + def symlink_windows(self): + if not is_windows: + return + win_install_path = os.path.join(self.prefix.bin, "MSWin32") + if host.is_64bit(): + win_install_path += "-x64" + else: + win_install_path += "-x86" + if self.spec.satisfies("+threads"): + win_install_path += "-multi-thread" + else: + win_install_path += "-perlio" + + for f in os.listdir(os.path.join(self.prefix.bin, win_install_path)): + lnk_path = os.path.join(self.prefix.bin, f) + src_path = os.path.join(win_install_path, f) + if not os.path.exists(lnk_path): + symlink(src_path, lnk_path) @run_after('install') def install_cpanm(self): spec = self.spec - + maker = make + win_prefix = '' + if is_windows: + maker = nmake + win_prefix = self.stage.source_path if '+cpanm' in spec: - with working_dir(join_path('cpanm', 'cpanm')): + with working_dir(join_path(win_prefix, 'cpanm', 'cpanm')): perl = spec['perl'].command perl('Makefile.PL') - make() - make('install') + maker() + maker('install') def _setup_dependent_env(self, env, dependent_spec, deptypes): """Set PATH and PERL5LIB to include the extension and @@ -295,6 +357,9 @@ def setup_dependent_package(self, module, dependent_spec): mkdirp(module.perl_lib_dir) def setup_build_environment(self, env): + if is_windows: + return + spec = self.spec if (spec.version <= Version('5.34.0') @@ -321,7 +386,8 @@ def filter_config_dot_pm(self): frustrates filter_file on some filesystems (NFSv4), so make them temporarily writable. """ - + if is_windows: + return kwargs = {'ignore_absent': True, 'backup': False, 'string': False} # Find the actual path to the installed Config.pm file. @@ -409,8 +475,11 @@ def command(self): Executable: the Perl command """ for ver in ('', self.spec.version): - path = os.path.join(self.prefix.bin, '{0}{1}'.format( - self.spec.name, ver)) + ext = '' + if is_windows: + ext = '.exe' + path = os.path.join(self.prefix.bin, '{0}{1}{2}'.format( + self.spec.name, ver, ext)) if os.path.exists(path): return Executable(path) else: diff --git a/var/spack/repos/builtin/packages/python/cpython-windows-externals.patch b/var/spack/repos/builtin/packages/python/cpython-windows-externals.patch new file mode 100644 index 0000000000..c3bcce983f --- /dev/null +++ b/var/spack/repos/builtin/packages/python/cpython-windows-externals.patch @@ -0,0 +1,28 @@ +diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat +index b5a44e3..52941c7 100644 +--- a/PCbuild/get_externals.bat ++++ b/PCbuild/get_externals.bat +@@ -76,7 +76,7 @@ for %%e in (%libraries%) do ( + echo.Fetching external binaries... + + set binaries= +-if NOT "%IncludeLibffi%"=="false" set binaries=%binaries% libffi ++if NOT "%IncludeLibffi%"=="false" set binaries=%binaries% libffi-3.3.0 + if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-1.1.1k-1 + if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-8.6.9.0 + if NOT "%IncludeSSLSrc%"=="false" set binaries=%binaries% nasm-2.11.06 +diff --git a/PCbuild/python.props b/PCbuild/python.props +index 419d5eb..c66fb07 100644 +--- a/PCbuild/python.props ++++ b/PCbuild/python.props +@@ -59,8 +59,8 @@ + $(ExternalsDir)sqlite-3.35.5.0\ + $(ExternalsDir)bzip2-1.0.6\ + $(ExternalsDir)xz-5.2.2\ +- $(ExternalsDir)libffi\ +- $(ExternalsDir)libffi\$(ArchName)\ ++ $(ExternalsDir)libffi-3.3.0\ ++ $(ExternalsDir)libffi-3.3.0\$(ArchName)\ + $(libffiOutDir)include + $(ExternalsDir)openssl-1.1.1k\ + $(ExternalsDir)openssl-bin-1.1.1k-1\$(ArchName)\ diff --git a/var/spack/repos/builtin/packages/python/package.py b/var/spack/repos/builtin/packages/python/package.py index 2ce0510819..cd67ac4dc3 100644 --- a/var/spack/repos/builtin/packages/python/package.py +++ b/var/spack/repos/builtin/packages/python/package.py @@ -4,10 +4,14 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) import glob +import inspect import json import os import re +import subprocess import sys +from distutils.dir_util import copy_tree +from shutil import copy import llnl.util.tty as tty from llnl.util.filesystem import get_filetype, path_contains_subdirectory @@ -18,8 +22,12 @@ from spack.util.environment import is_system_path from spack.util.prefix import Prefix +arch_map = {"AMD64": "x64", "x86": "Win32", + "IA64": "Win32", "EM64T": "Win32"} +is_windows = os.name == 'nt' -class Python(AutotoolsPackage): + +class Python(Package): """The Python programming language.""" homepage = "https://www.python.org/" @@ -29,11 +37,19 @@ class Python(AutotoolsPackage): maintainers = ['adamjstewart', 'skosukhin', 'scheibelp', 'varioustoxins'] + + phases = ['configure', 'build', 'install'] + + #: phase + install_targets = ['install'] + build_targets = [] + version('3.10.2', sha256='3c0ede893011319f9b0a56b44953a3d52c7abf9657c23fb4bc9ced93b86e9c97') version('3.10.1', sha256='b76117670e7c5064344b9c138e141a377e686b9063f3a8a620ff674fa8ec90d3') version('3.10.0', sha256='c4e0cbad57c90690cb813fb4663ef670b4d0f587d8171e2c42bd4c9245bd2758') version('3.9.10', sha256='1aa9c0702edbae8f6a2c95f70a49da8420aaa76b7889d3419c186bfc8c0e571e', preferred=True) version('3.9.9', sha256='2cc7b67c1f3f66c571acc42479cdf691d8ed6b47bee12c9b68430413a17a44ea') + version('3.9.9', sha256='2cc7b67c1f3f66c571acc42479cdf691d8ed6b47bee12c9b68430413a17a44ea') version('3.9.8', sha256='7447fb8bb270942d620dd24faa7814b1383b61fa99029a240025fd81c1db8283') version('3.9.7', sha256='a838d3f9360d157040142b715db34f0218e535333696a5569dc6f854604eb9d1') version('3.9.6', sha256='d0a35182e19e416fc8eae25a3dcd4d02d4997333e4ad1f2eee6010aadc3fe866') @@ -146,12 +162,12 @@ class Python(AutotoolsPackage): description='Enable expensive build-time optimizations, if available' ) # See https://legacy.python.org/dev/peps/pep-0394/ - variant('pythoncmd', default=True, + variant('pythoncmd', default=not is_windows, description="Symlink 'python3' executable to 'python' " "(not PEP 394 compliant)") # Optional Python modules - variant('readline', default=True, description='Build readline module') + variant('readline', default=not is_windows, description='Build readline module') variant('ssl', default=True, description='Build ssl module') variant('sqlite3', default=True, description='Build sqlite3 module') variant('dbm', default=True, description='Build dbm module') @@ -166,33 +182,34 @@ class Python(AutotoolsPackage): variant('tix', default=False, description='Build Tix module') variant('ensurepip', default=True, description='Build ensurepip module', when='@2.7.9:2,3.4:') - depends_on('pkgconfig@0.9.0:', type='build') - depends_on('gettext +libxml2', when='+libxml2') - depends_on('gettext ~libxml2', when='~libxml2') + if os.name != 'nt': + depends_on('pkgconfig@0.9.0:', type='build') + depends_on('gettext +libxml2', when='+libxml2') + depends_on('gettext ~libxml2', when='~libxml2') - # Optional dependencies - # See detect_modules() in setup.py for details - depends_on('readline', when='+readline') - depends_on('ncurses', when='+readline') - depends_on('openssl', when='+ssl') - # https://raw.githubusercontent.com/python/cpython/84471935ed2f62b8c5758fd544c7d37076fe0fa5/Misc/NEWS - # https://docs.python.org/3.5/whatsnew/changelog.html#python-3-5-4rc1 - depends_on('openssl@:1.0.2z', when='@:2.7.13,3.0.0:3.5.2+ssl') - depends_on('openssl@1.0.2:', when='@3.7:+ssl') # https://docs.python.org/3/whatsnew/3.7.html#build-changes - depends_on('openssl@1.1.1:', when='@3.10:+ssl') # https://docs.python.org/3.10/whatsnew/3.10.html#build-changes - depends_on('sqlite@3.0.8:', when='@:3.9+sqlite3') - depends_on('sqlite@3.7.15:', when='@3.10:+sqlite3') # https://docs.python.org/3.10/whatsnew/3.10.html#build-changes - depends_on('gdbm', when='+dbm') # alternatively ndbm or berkeley-db - depends_on('libnsl', when='+nis') - depends_on('zlib@1.1.3:', when='+zlib') - depends_on('bzip2', when='+bz2') - depends_on('xz', when='@3.3:+lzma') - depends_on('expat', when='+pyexpat') - depends_on('libffi', when='+ctypes') - depends_on('tk', when='+tkinter') - depends_on('tcl', when='+tkinter') - depends_on('uuid', when='+uuid') - depends_on('tix', when='+tix') + # Optional dependencies + # See detect_modules() in setup.py for details + depends_on('readline', when='+readline') + depends_on('ncurses', when='+readline') + depends_on('openssl', when='+ssl') + # https://raw.githubusercontent.com/python/cpython/84471935ed2f62b8c5758fd544c7d37076fe0fa5/Misc/NEWS + # https://docs.python.org/3.5/whatsnew/changelog.html#python-3-5-4rc1 + depends_on('openssl@:1.0.2z', when='@:2.7.13,3.0.0:3.5.2+ssl') + depends_on('openssl@1.0.2:', when='@3.7:+ssl') # https://docs.python.org/3/whatsnew/3.7.html#build-changes + depends_on('openssl@1.1.1:', when='@3.10:+ssl') # https://docs.python.org/3.10/whatsnew/3.10.html#build-changes + depends_on('sqlite@3.0.8:', when='@:3.9+sqlite3') + depends_on('sqlite@3.7.15:', when='@3.10:+sqlite3') # https://docs.python.org/3.10/whatsnew/3.10.html#build-changes + depends_on('gdbm', when='+dbm') # alternatively ndbm or berkeley-db + depends_on('libnsl', when='+nis') + depends_on('zlib@1.1.3:', when='+zlib') + depends_on('bzip2', when='+bz2') + depends_on('xz', when='@3.3:+lzma') + depends_on('expat', when='+pyexpat') + depends_on('libffi', when='+ctypes') + depends_on('tk', when='+tkinter') + depends_on('tcl', when='+tkinter') + depends_on('uuid', when='+uuid') + depends_on('tix', when='+tix') # Python needs to be patched to build extensions w/ mixed C/C++ code: # https://github.com/NixOS/nixpkgs/pull/19585/files @@ -211,6 +228,7 @@ class Python(AutotoolsPackage): patch('python-3.7.3-distutils-C++.patch', when='@3.7.3') patch('python-3.7.4+-distutils-C++.patch', when='@3.7.4:') patch('python-3.7.4+-distutils-C++-testsuite.patch', when='@3.7.4:') + patch('cpython-windows-externals.patch', when='@:3.9.6 platform=windows') patch('tkinter.patch', when='@:2.8,3.3:3.7 platform=darwin') # Patch the setup script to deny that tcl/x11 exists rather than allowing @@ -438,6 +456,70 @@ def flag_handler(self, name, flags): # allow flags to be passed through compiler wrapper return (flags, None, None) + @property + def configure_directory(self): + """Returns the directory where 'configure' resides. + :return: directory where to find configure + """ + return self.stage.source_path + + @property + def build_directory(self): + """Override to provide another place to build the package""" + return self.configure_directory + + @property + def plat_arch(self): + arch = platform.machine() + if arch in arch_map: + arch = arch_map[arch] + return arch + + @property + def win_build_params(self): + args = [] + args.append("-p %s" % self.plat_arch) + if self.spec.satisfies('+debug'): + args.append('-d') + if self.spec.satisfies('~ctypes'): + args.append('--no-ctypes') + if self.spec.satisfies('~ssl'): + args.append('--no-ssl') + if self.spec.satisfies('~tkinter'): + args.append('--no-tkinter') + return args + + def win_installer(self, prefix): + proj_root = self.stage.source_path + pcbuild_root = os.path.join(proj_root, "PCbuild") + build_root = os.path.join(pcbuild_root, platform.machine().lower()) + include_dir = os.path.join(proj_root, "Include") + copy_tree(include_dir, prefix.include) + doc_dir = os.path.join(proj_root, "Doc") + copy_tree(doc_dir, prefix.Doc) + tools_dir = os.path.join(proj_root, "Tools") + copy_tree(tools_dir, prefix.Tools) + lib_dir = os.path.join(proj_root, "Lib") + copy_tree(lib_dir, prefix.Lib) + pyconfig = os.path.join(proj_root, "PC", "pyconfig.h") + copy(pyconfig, prefix.include) + shared_libraries = [] + shared_libraries.extend(glob.glob("%s\\*.exe" % build_root)) + shared_libraries.extend(glob.glob("%s\\*.dll" % build_root)) + shared_libraries.extend(glob.glob("%s\\*.pyd" % build_root)) + os.makedirs(prefix.DLLs) + for lib in shared_libraries: + file_name = os.path.basename(lib) + if file_name.endswith(".exe") or\ + (file_name.endswith(".dll") and "python" in file_name)\ + or "vcruntime" in file_name: + copy(lib, prefix) + else: + copy(lib, prefix.DLLs) + static_libraries = glob.glob("%s\\*.lib") + for lib in static_libraries: + copy(lib, prefix.libs) + def configure_args(self): spec = self.spec config_args = [] @@ -549,6 +631,54 @@ def configure_args(self): return config_args + def configure(self, spec, prefix): + """Runs configure with the arguments specified in + :meth:`~spack.build_systems.autotools.AutotoolsPackage.configure_args` + and an appropriately set prefix. + """ + with working_dir(self.build_directory, create=True): + if is_windows: + pass + else: + options = getattr(self, 'configure_flag_args', []) + options += ['--prefix={0}'.format(prefix)] + options += self.configure_args() + configure(*options) + + def build(self, spec, prefix): + """Makes the build targets specified by + :py:attr:``~.AutotoolsPackage.build_targets`` + """ + # Windows builds use a batch script to drive + # configure and build in one step + with working_dir(self.build_directory): + if is_windows: + pcbuild_root = os.path.join(self.stage.source_path, "PCbuild") + builder_cmd = os.path.join(pcbuild_root, 'build.bat') + try: + subprocess.check_output( # novermin + " ".join([builder_cmd] + self.win_build_params), + stderr=subprocess.STDOUT + ) + except subprocess.CalledProcessError as e: + raise ProcessError("Process exited with status %d" % e.returncode, + long_message=e.output.decode('utf-8')) + else: + # See https://autotools.io/automake/silent.html + params = ['V=1'] + params += self.build_targets + inspect.getmodule(self).make(*params) + + def install(self, spec, prefix): + """Makes the install targets specified by + :py:attr:``~.AutotoolsPackage.install_targets`` + """ + with working_dir(self.build_directory): + if is_windows: + self.win_installer(prefix) + else: + inspect.getmodule(self).make(*self.install_targets) + @run_after('install') def filter_compilers(self): """Run after install to tell the configuration files and Makefiles @@ -557,7 +687,8 @@ def filter_compilers(self): If this isn't done, they'll have CC and CXX set to Spack's generic cc and c++. We want them to be bound to whatever compiler they were built with.""" - + if is_windows: + return kwargs = {'ignore_absent': True, 'backup': False, 'string': True} filenames = [ @@ -570,6 +701,8 @@ def filter_compilers(self): @run_after('install') def symlink(self): + if is_windows: + return spec = self.spec prefix = self.prefix @@ -759,6 +892,26 @@ def config_vars(self): Returns: dict: variable definitions """ + # Some values set by sysconfig may not always exist on Windows, so + # compute Windows alternatives + def repair_win_sysconf(conf): + if is_windows: + conf["LIBDIR"] = os.path.join(conf["LIBDEST"], "..", "libs") + conf["LIBPL"] = conf["LIBDIR"] + conf["PYTHONFRAMEWORKPREFIX"] = "" + conf["LDLIBRARY"] = "python" + conf["VERSION"] + ".dll" + conf["LIBRARY"] = "python" + conf["VERSION"] + ".lib" + conf["CC"] = "" + conf["CXX"] = "" + conf["LDSHARED"] = "" + conf["LDCXXSHARED"] = "" + + return conf + + # TODO: distutils is deprecated in Python 3.10 and will be removed in + # Python 3.12, find a different way to access this information. + # Also, calling the python executable disallows us from cross-compiling, + # so we want to try to avoid that if possible. cmd = """ import json from sysconfig import ( @@ -823,7 +976,7 @@ def config_vars(self): config.update(json.loads(self.command('-c', cmd, output=str))) except (ProcessError, RuntimeError): pass - self._config_vars[dag_hash] = config + self._config_vars[dag_hash] = repair_win_sysconf(config) return self._config_vars[dag_hash] def get_sysconfigdata_name(self): @@ -865,6 +1018,9 @@ def libs(self): # In Ubuntu 16.04.6 and python 2.7.12 from the system, lib could be # in LBPL # https://mail.python.org/pipermail/python-dev/2013-April/125733.html + # LIBPL does not exist in Windows, avoid uneccesary KeyError while allowing + # later failures. + # Return empty string rather than none so os.path doesn't complain libpl = self.config_vars['LIBPL'] # The system Python installation on macOS and Homebrew installations @@ -881,7 +1037,7 @@ def libs(self): if '+shared' in self.spec: ldlibrary = self.config_vars['LDLIBRARY'] - + win_bin_dir = self.config_vars['BINDIR'] if os.path.exists(os.path.join(libdir, ldlibrary)): return LibraryList(os.path.join(libdir, ldlibrary)) elif os.path.exists(os.path.join(libpl, ldlibrary)): @@ -891,6 +1047,9 @@ def libs(self): elif macos_developerdir and \ os.path.exists(os.path.join(macos_developerdir, ldlibrary)): return LibraryList(os.path.join(macos_developerdir, ldlibrary)) + elif is_windows and \ + os.path.exists(os.path.join(win_bin_dir, ldlibrary)): + return LibraryList(os.path.join(win_bin_dir, ldlibrary)) else: msg = 'Unable to locate {0} libraries in {1}' raise RuntimeError(msg.format(ldlibrary, libdir)) @@ -1075,7 +1234,7 @@ def setup_dependent_build_environment(self, env, dependent_spec): # fact that LDSHARED is set in the environment, therefore we export # the variable only if the new value is different from what we got # from the sysconfigdata file: - if config_link != new_link: + if config_link != new_link and not is_windows: env.set(link_var, new_link) def setup_dependent_run_environment(self, env, dependent_spec): @@ -1211,7 +1370,8 @@ def deactivate(self, ext_pkg, view, **args): )) def add_files_to_view(self, view, merge_map): - bin_dir = self.spec.prefix.bin + bin_dir = self.spec.prefix.bin if os.name != 'nt'\ + else self.spec.prefix for src, dst in merge_map.items(): if not path_contains_subdirectory(src, bin_dir): view.link(src, dst, spec=self.spec) @@ -1243,7 +1403,8 @@ def add_files_to_view(self, view, merge_map): view.link(new_link_target, dst, spec=self.spec) def remove_files_from_view(self, view, merge_map): - bin_dir = self.spec.prefix.bin + bin_dir = self.spec.prefix.bin if os.name != 'nt'\ + else self.spec.prefix for src, dst in merge_map.items(): if not path_contains_subdirectory(src, bin_dir): view.remove_file(src, dst) diff --git a/var/spack/repos/builtin/packages/ruby/package.py b/var/spack/repos/builtin/packages/ruby/package.py index 870a416283..b4ca1afaff 100644 --- a/var/spack/repos/builtin/packages/ruby/package.py +++ b/var/spack/repos/builtin/packages/ruby/package.py @@ -5,8 +5,12 @@ import re +from spack import * -class Ruby(AutotoolsPackage): +is_windows = str(spack.platforms.host()) == 'windows' + + +class Ruby(Package): """A dynamic, open source programming language with a focus on simplicity and productivity.""" @@ -27,23 +31,22 @@ class Ruby(AutotoolsPackage): version('2.5.3', sha256='9828d03852c37c20fa333a0264f2490f07338576734d910ee3fd538c9520846c') version('2.2.0', sha256='7671e394abfb5d262fbcd3b27a71bf78737c7e9347fa21c39e58b0bb9c4840fc') - variant('openssl', default=True, description="Enable OpenSSL support") - variant('readline', default=False, description="Enable Readline support") + if not is_windows: + variant('openssl', default=True, description="Enable OpenSSL support") + variant('readline', default=False, description="Enable Readline support") + depends_on('pkgconfig', type=('build')) + depends_on('libffi') + depends_on('libx11', when='@:2.3') + depends_on('tcl', when='@:2.3') + depends_on('tk', when='@:2.3') + depends_on('readline', when='+readline') + depends_on('zlib') + with when('+openssl'): + depends_on('openssl@:1') + depends_on('openssl@:1.0', when='@:2.3') extendable = True - - depends_on('pkgconfig', type=('build')) - depends_on('libffi') - depends_on('zlib') - depends_on('libx11', when='@:2.3') - depends_on('tcl', when='@:2.3') - depends_on('tk', when='@:2.3') - depends_on('readline', when='+readline') - - with when('+openssl'): - depends_on('openssl@:1') - depends_on('openssl@:1.0', when='@:2.3') - + phases = ['autoreconf', 'configure', 'build', 'install'] # Known build issues when Avira antivirus software is running: # https://github.com/rvm/rvm/issues/4313#issuecomment-374020379 # TODO: add check for this and warn user @@ -116,6 +119,32 @@ def setup_dependent_package(self, module, dependent_spec): module.gem = Executable(self.prefix.bin.gem) module.rake = Executable(self.prefix.bin.rake) + def configure(self, spec, prefix): + with working_dir(self.build_directory, create=True): + if is_windows: + Executable("win32\\configure.bat", "--prefix=%s" % self.prefix) + else: + options = getattr(self, 'configure_flag_args', []) + options += ['--prefix={0}'.format(prefix)] + options += self.configure_args() + configure(*options) + + def build(self, spec, prefix): + with working_dir(self.build_directory): + if is_windows: + nmake() + else: + params = ['V=1'] + params += self.build_targets + make(*params) + + def install(self, spec, prefix): + with working_dir(self.build_directory): + if is_windows: + nmake('install') + else: + make(*self.install_targets) + @run_after('install') def post_install(self): """ RubyGems updated their SSL certificates at some point, so diff --git a/var/spack/repos/builtin/packages/zlib/package.py b/var/spack/repos/builtin/packages/zlib/package.py index 9a445b8173..3b6b1fd18d 100644 --- a/var/spack/repos/builtin/packages/zlib/package.py +++ b/var/spack/repos/builtin/packages/zlib/package.py @@ -6,7 +6,11 @@ # Although zlib comes with a configure script, it does not use Autotools # The AutotoolsPackage causes zlib to fail to build with PGI -class Zlib(CMakePackage): +import glob +import os + + +class Zlib(Package): """A free, general-purpose, legally unencumbered lossless data-compression library. """ @@ -37,16 +41,29 @@ def libs(self): ['libz'], root=self.prefix, recursive=True, shared=shared ) - def cmake_args(self): - args = ['-DBUILD_SHARED_LIBS:BOOL=' + - ('ON' if self._building_shared else 'OFF')] - return args + def win_install(self): + build_dir = self.stage.source_path + install_tree = {} + install_tree["bin"] = glob.glob(os.path.join(build_dir, "*.dll")) + install_tree["lib"] = glob.glob(os.path.join(build_dir, "*.lib")) + compose_src_path = lambda x: os.path.join(build_dir, x) + install_tree["include"] = [compose_src_path("zlib.h"), + compose_src_path("zconf.h")] + install_tree["share"] = {"man": {"man3": [compose_src_path("zlib.3")]}} - @property - def build_directory(self): - return join_path(self.stage.source_path, - 'spack-build-shared' if self._building_shared - else 'spack-build-static') + def installtree(dst, tree): + for inst_dir in tree: + if type(tree[inst_dir]) is list: + install_dst = getattr(dst, inst_dir) + try: + os.makedirs(install_dst) + except OSError: + pass + for file in tree[inst_dir]: + copy(file, install_dst) + else: + installtree(getattr(dst, inst_dir), tree[inst_dir]) + installtree(self.prefix, install_tree) def setup_build_environment(self, env): if '+pic' in self.spec: @@ -54,27 +71,10 @@ def setup_build_environment(self, env): if '+optimize' in self.spec: env.append_flags('CFLAGS', '-O2') - # Build, install, and check both static and shared versions of the - # libraries when +shared - @when('+shared platform=windows') - def cmake(self, spec, prefix): - for self._building_shared in (False, True): - super(Zlib, self).cmake(spec, prefix) - - @when('+shared platform=windows') - def build(self, spec, prefix): - for self._building_shared in (False, True): - super(Zlib, self).build(spec, prefix) - - @when('+shared platform=windows') - def check(self): - for self._building_shared in (False, True): - super(Zlib, self).check() - def install(self, spec, prefix): - if 'platform=windows' in self.spec and '+shared' in self.spec: - for self._building_shared in (False, True): - super(Zlib, self).install(spec, prefix) + if 'platform=windows' in self.spec: + nmake('-f' 'win32\\Makefile.msc') + self.win_install() else: config_args = [] if '~shared' in spec: