From c52426ea7a17a60dd1f4e6d8aeaa4ead157badc7 Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Wed, 15 Sep 2021 07:44:16 +0200 Subject: [PATCH] Make clingo the default solver (#25502) Modifications: - [x] Change `defaults/config.yaml` - [x] Add a fix for bootstrapping patchelf from sources if `compilers.yaml` is empty - [x] Make `SPACK_TEST_SOLVER=clingo` the default for unit-tests - [x] Fix package failures in the e4s pipeline Caveats: 1. CentOS 6 still uses the original concretizer as it can't connect to the buildcache due to issues with `ssl` (bootstrapping from sources requires a C++14 capable compiler) 1. I had to update the image tag for GitlabCI in e699f14. 1. libtool v2.4.2 has been deprecated and other packages received some update --- .github/workflows/unit_tests.yaml | 7 ++++-- etc/spack/defaults/config.yaml | 7 +++--- lib/spack/spack/bootstrap.py | 10 ++++++++ lib/spack/spack/test/conftest.py | 2 +- .../gitlab/cloud_pipelines/.gitlab-ci.yml | 2 +- .../repos/builtin/packages/libtool/package.py | 25 ++++++++++++------- .../repos/builtin/packages/numactl/package.py | 4 +++ 7 files changed, 40 insertions(+), 17 deletions(-) diff --git a/.github/workflows/unit_tests.yaml b/.github/workflows/unit_tests.yaml index 9fdcf4aefa..6cc63b3649 100644 --- a/.github/workflows/unit_tests.yaml +++ b/.github/workflows/unit_tests.yaml @@ -225,23 +225,25 @@ jobs: if: ${{ needs.changes.outputs.with_coverage == 'true' }} env: HOME: /home/spack-test + SPACK_TEST_SOLVER: original run: | whoami && echo $HOME && cd $HOME git clone https://github.com/spack/spack.git && cd spack git fetch origin ${{ github.ref }}:test-branch git checkout test-branch - share/spack/qa/run-unit-tests + bin/spack unit-test -x - name: Run unit tests (only package tests) if: ${{ needs.changes.outputs.with_coverage == 'false' }} env: HOME: /home/spack-test ONLY_PACKAGES: true + SPACK_TEST_SOLVER: original run: | whoami && echo $HOME && cd $HOME git clone https://github.com/spack/spack.git && cd spack git fetch origin ${{ github.ref }}:test-branch git checkout test-branch - share/spack/qa/run-unit-tests + bin/spack unit-test -x -m "not maybeslow" -k "package_sanity" # Test RHEL8 UBI with platform Python. This job is run # only on PRs modifying core Spack @@ -268,6 +270,7 @@ jobs: shell: runuser -u spack-test -- bash {0} run: | source share/spack/setup-env.sh + spack -d solve zlib spack unit-test -k 'not cvs and not svn and not hg' -x --verbose # Test for the clingo based solver (using clingo-cffi) clingo-cffi: diff --git a/etc/spack/defaults/config.yaml b/etc/spack/defaults/config.yaml index 64a50cc805..88a19d3222 100644 --- a/etc/spack/defaults/config.yaml +++ b/etc/spack/defaults/config.yaml @@ -160,11 +160,10 @@ config: # sufficiently for many specs. # # 'clingo': Uses a logic solver under the hood to solve DAGs with full - # backtracking and optimization for user preferences. + # backtracking and optimization for user preferences. Spack will + # try to bootstrap the logic solver, if not already available. # - # 'clingo' currently requires the clingo ASP solver to be installed and - # built with python bindings. 'original' is built in. - concretizer: original + concretizer: clingo # How long to wait to lock the Spack installation database. This lock is used diff --git a/lib/spack/spack/bootstrap.py b/lib/spack/spack/bootstrap.py index efcb8a7e19..e95afd8adc 100644 --- a/lib/spack/spack/bootstrap.py +++ b/lib/spack/spack/bootstrap.py @@ -489,6 +489,16 @@ def _bootstrap_config_scopes(): @contextlib.contextmanager def ensure_bootstrap_configuration(): + # We may need to compile code from sources, so ensure we have compilers + # for the current platform before switching parts. + arch = spack.architecture.default_arch() + arch = spack.spec.ArchSpec(str(arch)) # The call below expects an ArchSpec object + if not spack.compilers.compilers_for_arch(arch): + compiler_cmd = spack.main.SpackCommand('compiler') + compiler_cmd( + 'find', output=os.devnull, error=os.devnull, fail_on_error=False + ) + bootstrap_store_path = store_path() with spack.environment.deactivate_environment(): with spack.architecture.use_platform(spack.architecture.real_platform()): diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py index cd6ce907e7..456624127a 100644 --- a/lib/spack/spack/test/conftest.py +++ b/lib/spack/spack/test/conftest.py @@ -585,7 +585,7 @@ def configuration_dir(tmpdir_factory, linux_os): tmpdir.ensure('user', dir=True) # Slightly modify config.yaml and compilers.yaml - solver = os.environ.get('SPACK_TEST_SOLVER', 'original') + solver = os.environ.get('SPACK_TEST_SOLVER', 'clingo') config_yaml = test_config.join('config.yaml') modules_root = tmpdir_factory.mktemp('share') tcl_root = modules_root.ensure('modules', dir=True) diff --git a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml index f2ccfe5e51..c9cfa4cb8a 100644 --- a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml +++ b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml @@ -1,7 +1,7 @@ stages: [ "generate", "build" ] default: - image: { "name": "ghcr.io/scottwittenburg/ecpe4s-ubuntu18.04-runner-x86_64:2020-09-01", "entrypoint": [""] } + image: { "name": "ghcr.io/scottwittenburg/ecpe4s-ubuntu18.04-runner-x86_64:2021-05-15", "entrypoint": [""] } ######################################## # Job templates diff --git a/var/spack/repos/builtin/packages/libtool/package.py b/var/spack/repos/builtin/packages/libtool/package.py index 2c2906f597..a85d09e633 100644 --- a/var/spack/repos/builtin/packages/libtool/package.py +++ b/var/spack/repos/builtin/packages/libtool/package.py @@ -15,18 +15,25 @@ class Libtool(AutotoolsPackage, GNUMirrorPackage): version('develop', git='https://git.savannah.gnu.org/git/libtool.git', branch='master', submodules=True) version('2.4.6', sha256='e3bd4d5d3d025a36c21dd6af7ea818a2afcd4dfc1ea5a17b39d7854bcd0c06e3') - version('2.4.2', sha256='b38de44862a987293cd3d8dfae1c409d514b6c4e794ebc93648febf9afc38918') + # Version released in 2011 + version('2.4.2', sha256='b38de44862a987293cd3d8dfae1c409d514b6c4e794ebc93648febf9afc38918', deprecated=True) depends_on('m4@1.4.6:', type='build') - depends_on('autoconf', type='build', when='@2.4.2,develop') - depends_on('automake', type='build', when='@2.4.2,develop') - depends_on('help2man', type='build', when='@2.4.2,develop') - depends_on('xz', type='build', when='@develop') - depends_on('texinfo', type='build', when='@develop') - # Fix parsing of compiler output when collecting predeps and postdeps - # https://lists.gnu.org/archive/html/bug-libtool/2016-03/msg00003.html - patch('flag_space.patch', when='@develop') + with when('@2.4.2'): + depends_on('autoconf', type='build') + depends_on('automake', type='build') + depends_on('help2man', type='build') + + with when('@develop'): + depends_on('autoconf', type='build') + depends_on('automake', type='build') + depends_on('help2man', type='build') + depends_on('xz', type='build') + depends_on('texinfo', type='build') + # Fix parsing of compiler output when collecting predeps and postdeps + # https://lists.gnu.org/archive/html/bug-libtool/2016-03/msg00003.html + patch('flag_space.patch') build_directory = 'spack-build' diff --git a/var/spack/repos/builtin/packages/numactl/package.py b/var/spack/repos/builtin/packages/numactl/package.py index 6aae477c83..52abf2353a 100644 --- a/var/spack/repos/builtin/packages/numactl/package.py +++ b/var/spack/repos/builtin/packages/numactl/package.py @@ -28,6 +28,10 @@ class Numactl(AutotoolsPackage): depends_on('libtool', type='build') depends_on('m4', type='build') + # Numactl has hardcoded minimum versions for libtool, + # libtool@develop returns UNKOWN as a version tag and fails + conflicts('libtool@develop') + def autoreconf(self, spec, prefix): bash = which('bash') bash('./autogen.sh')