Make clingo the default solver (#25502)

Modifications:
- [x] Change `defaults/config.yaml`
- [x] Add a fix for bootstrapping patchelf from sources if `compilers.yaml` is empty
- [x] Make `SPACK_TEST_SOLVER=clingo` the default for unit-tests
- [x] Fix package failures in the e4s pipeline

Caveats:
1. CentOS 6 still uses the original concretizer as it can't connect to the buildcache due to issues with `ssl` (bootstrapping from sources requires a C++14 capable compiler)
1. I had to update the image tag for GitlabCI in e699f14.  
1. libtool v2.4.2 has been deprecated and other packages received some update

.github/workflows/unit_tests.yaml

@@ -225,23 +225,25 @@ jobs:
       if: ${{ needs.changes.outputs.with_coverage == 'true' }}
       env:
           HOME: /home/spack-test
+          SPACK_TEST_SOLVER: original
       run: |
           whoami && echo $HOME && cd $HOME
           git clone https://github.com/spack/spack.git && cd spack
           git fetch origin ${{ github.ref }}:test-branch
           git checkout test-branch
-          share/spack/qa/run-unit-tests
+          bin/spack unit-test -x
     - name: Run unit tests (only package tests)
       if: ${{ needs.changes.outputs.with_coverage == 'false' }}
       env:
           HOME: /home/spack-test
           ONLY_PACKAGES: true
+          SPACK_TEST_SOLVER: original
       run: |
           whoami && echo $HOME && cd $HOME
           git clone https://github.com/spack/spack.git && cd spack
           git fetch origin ${{ github.ref }}:test-branch
           git checkout test-branch
-          share/spack/qa/run-unit-tests
+          bin/spack unit-test -x -m "not maybeslow" -k "package_sanity"
 
   # Test RHEL8 UBI with platform Python. This job is run
   # only on PRs modifying core Spack
@@ -268,6 +270,7 @@ jobs:
       shell: runuser -u spack-test -- bash {0}
       run: |
           source share/spack/setup-env.sh
+          spack -d solve zlib
           spack unit-test -k 'not cvs and not svn and not hg' -x --verbose
   # Test for the clingo based solver (using clingo-cffi)
   clingo-cffi:

etc/spack/defaults/config.yaml

@@ -160,11 +160,10 @@ config:
   #       sufficiently for many specs.
   #
   #   'clingo': Uses a logic solver under the hood to solve DAGs with full
-  #       backtracking and optimization for user preferences.
+  #       backtracking and optimization for user preferences. Spack will
+  #       try to bootstrap the logic solver, if not already available.
   #
-  # 'clingo' currently requires the clingo ASP solver to be installed and
-  # built with python bindings. 'original' is built in.
-  concretizer: original
+  concretizer: clingo
 
 
   # How long to wait to lock the Spack installation database. This lock is used

lib/spack/spack/bootstrap.py

@@ -489,6 +489,16 @@ def _bootstrap_config_scopes():
 
 @contextlib.contextmanager
 def ensure_bootstrap_configuration():
+    # We may need to compile code from sources, so ensure we have compilers
+    # for the current platform before switching parts.
+    arch = spack.architecture.default_arch()
+    arch = spack.spec.ArchSpec(str(arch))  # The call below expects an ArchSpec object
+    if not spack.compilers.compilers_for_arch(arch):
+        compiler_cmd = spack.main.SpackCommand('compiler')
+        compiler_cmd(
+            'find', output=os.devnull, error=os.devnull, fail_on_error=False
+        )
+
     bootstrap_store_path = store_path()
     with spack.environment.deactivate_environment():
         with spack.architecture.use_platform(spack.architecture.real_platform()):

lib/spack/spack/test/conftest.py

@@ -585,7 +585,7 @@ def configuration_dir(tmpdir_factory, linux_os):
     tmpdir.ensure('user', dir=True)
 
     # Slightly modify config.yaml and compilers.yaml
-    solver = os.environ.get('SPACK_TEST_SOLVER', 'original')
+    solver = os.environ.get('SPACK_TEST_SOLVER', 'clingo')
     config_yaml = test_config.join('config.yaml')
     modules_root = tmpdir_factory.mktemp('share')
     tcl_root = modules_root.ensure('modules', dir=True)

share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml

@@ -1,7 +1,7 @@
 stages: [ "generate", "build" ]
 
 default:
-  image: { "name": "ghcr.io/scottwittenburg/ecpe4s-ubuntu18.04-runner-x86_64:2020-09-01", "entrypoint": [""] }
+  image: { "name": "ghcr.io/scottwittenburg/ecpe4s-ubuntu18.04-runner-x86_64:2021-05-15", "entrypoint": [""] }
 
 ########################################
 # Job templates

var/spack/repos/builtin/packages/libtool/package.py

@@ -15,18 +15,25 @@ class Libtool(AutotoolsPackage, GNUMirrorPackage):
     version('develop', git='https://git.savannah.gnu.org/git/libtool.git',
             branch='master', submodules=True)
     version('2.4.6', sha256='e3bd4d5d3d025a36c21dd6af7ea818a2afcd4dfc1ea5a17b39d7854bcd0c06e3')
-    version('2.4.2', sha256='b38de44862a987293cd3d8dfae1c409d514b6c4e794ebc93648febf9afc38918')
+    # Version released in 2011
+    version('2.4.2', sha256='b38de44862a987293cd3d8dfae1c409d514b6c4e794ebc93648febf9afc38918', deprecated=True)
 
     depends_on('m4@1.4.6:', type='build')
-    depends_on('autoconf', type='build', when='@2.4.2,develop')
-    depends_on('automake', type='build', when='@2.4.2,develop')
-    depends_on('help2man', type='build', when='@2.4.2,develop')
-    depends_on('xz', type='build', when='@develop')
-    depends_on('texinfo', type='build', when='@develop')
 
-    # Fix parsing of compiler output when collecting predeps and postdeps
-    # https://lists.gnu.org/archive/html/bug-libtool/2016-03/msg00003.html
-    patch('flag_space.patch', when='@develop')
+    with when('@2.4.2'):
+        depends_on('autoconf', type='build')
+        depends_on('automake', type='build')
+        depends_on('help2man', type='build')
+
+    with when('@develop'):
+        depends_on('autoconf', type='build')
+        depends_on('automake', type='build')
+        depends_on('help2man', type='build')
+        depends_on('xz', type='build')
+        depends_on('texinfo', type='build')
+        # Fix parsing of compiler output when collecting predeps and postdeps
+        # https://lists.gnu.org/archive/html/bug-libtool/2016-03/msg00003.html
+        patch('flag_space.patch')
 
     build_directory = 'spack-build'
 

var/spack/repos/builtin/packages/numactl/package.py

@@ -28,6 +28,10 @@ class Numactl(AutotoolsPackage):
     depends_on('libtool',  type='build')
     depends_on('m4',       type='build')
 
+    # Numactl has hardcoded minimum versions for libtool,
+    # libtool@develop returns UNKOWN as a version tag and fails
+    conflicts('libtool@develop')
+
     def autoreconf(self, spec, prefix):
         bash = which('bash')
         bash('./autogen.sh')