Added tests on compiler's version detection (#10931)

* Split get_compiler_version into two functions:
  get_compiler_version_output runs the compiler with the relevant
  option to print the version; extract_version_from_output determines
  the version by examining this output. This makes it easier to test
  the customized version detection for each compiler. Users can
  customize this by overriding the following:
  * version_argument: this is the argument that tells the compiler to
    print its version. It assumes that the compiler will report its
	version if invoked with a single option (like "--version")
  * version_regex: the regular expression used to extract the version
    from the compiler argument. This assumes that a regular
	expression is sufficient to extract the version, and that the
	version can be extracted from a single capture group (Spack uses
	the first capture group)
  * default_version: allows you to completely override all version
    detection logic
  * get_compiler_version_output: if getting the compiler to report
    its version is more complex than invoking it with a single arg
  * extract_version_from_output: if it is difficult to define a regex
    that can be used to extract the version from the output
* Added tests for version detection of most compilers
* Removed redundant code from xl_r compiler class (by inheriting
  from xl compiler definition)
This commit is contained in:
Massimiliano Culpo 2019-04-02 21:52:47 +02:00 committed by Peter Scheibel
parent b08518dadd
commit a1255b61c0
12 changed files with 200 additions and 258 deletions

View file

@ -7,6 +7,7 @@
import re
import itertools
import llnl.util.lang
import llnl.util.tty as tty
import llnl.util.multiproc as mp
@ -16,7 +17,7 @@
from spack.util.executable import Executable, ProcessError
from spack.util.environment import get_path
__all__ = ['Compiler', 'get_compiler_version']
__all__ = ['Compiler']
def _verify_executables(*paths):
@ -25,24 +26,18 @@ def _verify_executables(*paths):
raise CompilerAccessError(path)
_version_cache = {}
@llnl.util.lang.memoized
def get_compiler_version_output(compiler_path, version_arg):
"""Invokes the compiler at a given path passing a single
version argument and returns the output.
def get_compiler_version(compiler_path, version_arg, regex='(.*)'):
key = (compiler_path, version_arg, regex)
if key not in _version_cache:
compiler = Executable(compiler_path)
output = compiler(version_arg, output=str, error=str)
match = re.search(regex, output)
_version_cache[key] = match.group(1) if match else 'unknown'
return _version_cache[key]
def dumpversion(compiler_path):
"""Simple default dumpversion method -- this is what gcc does."""
return get_compiler_version(compiler_path, '-dumpversion')
Args:
compiler_path (path): path of the compiler to be invoked
version_arg (str): the argument used to extract version information
"""
compiler = Executable(compiler_path)
output = compiler(version_arg, output=str, error=str)
return output
def tokenize_flags(flags_str):
@ -93,6 +88,12 @@ class Compiler(object):
# version suffix for gcc.
suffixes = [r'-.*']
#: Compiler argument that produces version information
version_argument = '-dumpversion'
#: Regex used to extract version from compiler's output
version_regex = '(.*)'
# Default flags used by a compiler to set an rpath
@property
def cc_rpath_arg(self):
@ -204,7 +205,15 @@ def cxx17_flag(self):
@classmethod
def default_version(cls, cc):
"""Override just this to override all compiler version functions."""
return dumpversion(cc)
output = get_compiler_version_output(cc, cls.version_argument)
return cls.extract_version_from_output(output)
@classmethod
@llnl.util.lang.memoized
def extract_version_from_output(cls, output):
"""Extracts the version from compiler's output."""
match = re.search(cls.version_regex, output)
return match.group(1) if match else 'unknown'
@classmethod
def cc_version(cls, cc):

View file

@ -3,10 +3,10 @@
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
from spack.compiler import Compiler, get_compiler_version
import spack.compiler
class Arm(Compiler):
class Arm(spack.compiler.Compiler):
# Subclasses use possible names of C compiler
cc_names = ['armclang']
@ -25,6 +25,18 @@ class Arm(Compiler):
'f77': 'arm/armflang',
'fc': 'arm/armflang'}
# The ``--version`` option seems to be the most consistent one for
# arm compilers. Output looks like this:
#
# $ arm<c/f>lang --version
# Arm C/C++/Fortran Compiler version 19.0 (build number 73) (based on LLVM 7.0.2) # NOQA
# Target: aarch64--linux-gnu
# Thread model: posix
# InstalledDir:
# /opt/arm/arm-hpc-compiler-19.0_Generic-AArch64_RHEL-7_aarch64-linux/bin
version_argument = '--version'
version_regex = r'Arm C\/C\+\+\/Fortran Compiler version ([^ )]+)'
@property
def openmp_flag(self):
return "-fopenmp"
@ -45,22 +57,6 @@ def cxx17_flag(self):
def pic_flag(self):
return "-fPIC"
@classmethod
def default_version(cls, comp):
"""The ``--version`` option seems to be the most consistent one
for arm compilers. Output looks like this::
$ arm<c/f>lang --version
Arm C/C++/Fortran Compiler version 19.0 (build number 73) (based on LLVM 7.0.2) # NOQA
Target: aarch64--linux-gnu
Thread model: posix
InstalledDir:
/opt/arm/arm-hpc-compiler-19.0_Generic-AArch64_RHEL-7_aarch64-linux/bin # NOQA
"""
return get_compiler_version(
comp, '--version',
r'Arm C\/C\+\+\/Fortran Compiler version ([^ )]+)')
@classmethod
def fc_version(cls, fc):
return cls.default_version(fc)

View file

@ -3,10 +3,10 @@
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
from spack.compiler import Compiler, get_compiler_version
import spack.compiler
class Cce(Compiler):
class Cce(spack.compiler.Compiler):
"""Cray compiler environment compiler."""
# Subclasses use possible names of C compiler
cc_names = ['cc']
@ -31,9 +31,8 @@ class Cce(Compiler):
'f77': 'cce/ftn',
'fc': 'cce/ftn'}
@classmethod
def default_version(cls, comp):
return get_compiler_version(comp, '-V', r'[Vv]ersion.*?(\d+(\.\d+)+)')
version_argument = '-V'
version_regex = r'[Vv]ersion.*?(\d+(\.\d+)+)'
@property
def openmp_flag(self):

View file

@ -8,10 +8,11 @@
import sys
from shutil import copytree, ignore_patterns
import llnl.util.lang
import llnl.util.tty as tty
import spack.paths
from spack.compiler import Compiler, _version_cache, UnsupportedCompilerFlag
from spack.compiler import Compiler, UnsupportedCompilerFlag
from spack.util.executable import Executable
from spack.version import ver
@ -161,6 +162,7 @@ def pic_flag(self):
return "-fPIC"
@classmethod
@llnl.util.lang.memoized
def default_version(cls, comp):
"""The ``--version`` option works for clang compilers.
On most platforms, output looks like this::
@ -175,22 +177,13 @@ def default_version(cls, comp):
Target: x86_64-apple-darwin15.2.0
Thread model: posix
"""
if comp not in _version_cache:
compiler = Executable(comp)
output = compiler('--version', output=str, error=str)
_version_cache[comp] = cls.detect_version_from_str(output)
return _version_cache[comp]
compiler = Executable(comp)
output = compiler('--version', output=str, error=str)
return cls.extract_version_from_output(output)
@classmethod
def detect_version_from_str(cls, output):
"""Returns the version that has been detected from the string
passed as input. If no detection is possible returns the
string 'unknown'.
Args:
output (str): string used to detect a compiler version
"""
@llnl.util.lang.memoized
def extract_version_from_output(cls, output):
ver = 'unknown'
match = re.search(
# Apple's LLVM compiler has its own versions, so suffix them.

View file

@ -3,9 +3,11 @@
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import re
import spack.compilers.clang
from spack.compiler import \
Compiler, get_compiler_version, UnsupportedCompilerFlag
from spack.compiler import Compiler, UnsupportedCompilerFlag
from spack.version import ver
@ -114,7 +116,10 @@ def default_version(cls, cc):
version = super(Gcc, cls).default_version(cc)
if ver(version) >= ver('7'):
version = get_compiler_version(cc, '-dumpfullversion')
output = spack.compiler.get_compiler_version_output(
cc, '-dumpfullversion'
)
version = cls.extract_version_from_output(output)
return version
@classmethod
@ -139,11 +144,14 @@ def fc_version(cls, fc):
7.2.0
"""
version = get_compiler_version(
fc, '-dumpversion',
r'(?:GNU Fortran \(GCC\) )?([\d.]+)')
output = spack.compiler.get_compiler_version_output(fc, '-dumpversion')
match = re.search(r'(?:GNU Fortran \(GCC\) )?([\d.]+)', output)
version = match.group(match.lastindex) if match else 'unknown'
if ver(version) >= ver('7'):
version = get_compiler_version(fc, '-dumpfullversion')
output = spack.compiler.get_compiler_version_output(
fc, '-dumpfullversion'
)
version = cls.extract_version_from_output(output)
return version
@classmethod

View file

@ -3,8 +3,7 @@
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
from spack.compiler import \
Compiler, get_compiler_version, UnsupportedCompilerFlag
from spack.compiler import Compiler, UnsupportedCompilerFlag
from spack.version import ver
@ -30,6 +29,9 @@ class Intel(Compiler):
PrgEnv = 'PrgEnv-intel'
PrgEnv_compiler = 'intel'
version_argument = '--version'
version_regex = r'\((?:IFORT|ICC)\) ([^ ]+)'
@property
def openmp_flag(self):
if self.version < ver('16.0'):
@ -67,22 +69,6 @@ def cxx14_flag(self):
def pic_flag(self):
return "-fPIC"
@classmethod
def default_version(cls, comp):
"""The ``--version`` option seems to be the most consistent one
for intel compilers. Output looks like this::
icpc (ICC) 12.1.5 20120612
Copyright (C) 1985-2012 Intel Corporation. All rights reserved.
or::
ifort (IFORT) 12.1.5 20120612
Copyright (C) 1985-2012 Intel Corporation. All rights reserved.
"""
return get_compiler_version(
comp, '--version', r'\((?:IFORT|ICC)\) ([^ ]+)')
@property
def stdcxx_libs(self):
return ('-cxxlib', )

View file

@ -3,10 +3,10 @@
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
from spack.compiler import Compiler, get_compiler_version
import spack.compiler
class Nag(Compiler):
class Nag(spack.compiler.Compiler):
# Subclasses use possible names of C compiler
cc_names = []
@ -27,6 +27,9 @@ class Nag(Compiler):
'f77': 'nag/nagfor',
'fc': 'nag/nagfor'}
version_argument = '-V'
version_regex = r'NAG Fortran Compiler Release ([0-9.]+)'
@property
def openmp_flag(self):
return "-openmp"
@ -51,14 +54,3 @@ def f77_rpath_arg(self):
@property
def fc_rpath_arg(self):
return '-Wl,-Wl,,-rpath,,'
@classmethod
def default_version(cls, comp):
"""The ``-V`` option works for nag compilers.
Output looks like this::
NAG Fortran Compiler Release 6.0(Hibiya) Build 1037
Product NPL6A60NA for x86-64 Linux
"""
return get_compiler_version(
comp, '-V', r'NAG Fortran Compiler Release ([0-9.]+)')

View file

@ -3,10 +3,10 @@
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
from spack.compiler import Compiler, get_compiler_version
import spack.compiler
class Pgi(Compiler):
class Pgi(spack.compiler.Compiler):
# Subclasses use possible names of C compiler
cc_names = ['pgcc']
@ -28,6 +28,9 @@ class Pgi(Compiler):
PrgEnv = 'PrgEnv-pgi'
PrgEnv_compiler = 'pgi'
version_argument = '-V'
version_regex = r'pg[^ ]* ([0-9.]+)-[0-9]+ [^ ]+ target on '
@property
def openmp_flag(self):
return "-mp"
@ -39,23 +42,3 @@ def cxx11_flag(self):
@property
def pic_flag(self):
return "-fpic"
@classmethod
def default_version(cls, comp):
"""The ``-V`` option works for all the PGI compilers.
Output looks like this::
pgcc 15.10-0 64-bit target on x86-64 Linux -tp sandybridge
The Portland Group - PGI Compilers and Tools
Copyright (c) 2015, NVIDIA CORPORATION. All rights reserved.
on x86-64, and::
pgcc 17.4-0 linuxpower target on Linuxpower
PGI Compilers and Tools
Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved.
on PowerPC.
"""
return get_compiler_version(
comp, '-V', r'pg[^ ]* ([0-9.]+)-[0-9]+ [^ ]+ target on ')

View file

@ -3,8 +3,7 @@
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
from spack.compiler import \
Compiler, get_compiler_version, UnsupportedCompilerFlag
from spack.compiler import Compiler, UnsupportedCompilerFlag
from spack.version import ver
@ -27,6 +26,9 @@ class Xl(Compiler):
'f77': 'xl/xlf',
'fc': 'xl/xlf90'}
version_argument = '-qversion'
version_regex = r'([0-9]?[0-9]\.[0-9])'
@property
def openmp_flag(self):
return "-qsmp=omp"
@ -53,58 +55,29 @@ def fflags(self):
# Its use has no negative side effects.
return "-qzerosize"
@classmethod
def default_version(cls, comp):
"""The '-qversion' is the standard option fo XL compilers.
Output looks like this::
IBM XL C/C++ for Linux, V11.1 (5724-X14)
Version: 11.01.0000.0000
or::
IBM XL Fortran for Linux, V13.1 (5724-X16)
Version: 13.01.0000.0000
or::
IBM XL C/C++ for AIX, V11.1 (5724-X13)
Version: 11.01.0000.0009
or::
IBM XL C/C++ Advanced Edition for Blue Gene/P, V9.0
Version: 09.00.0000.0017
"""
return get_compiler_version(
comp, '-qversion', r'([0-9]?[0-9]\.[0-9])')
@classmethod
def fc_version(cls, fc):
"""The fortran and C/C++ versions of the XL compiler are always
two units apart. By this we mean that the fortran release that
goes with XL C/C++ 11.1 is 13.1. Having such a difference in
version number is confusing spack quite a lot. Most notably
if you keep the versions as is the default xl compiler will
only have fortran and no C/C++. So we associate the Fortran
compiler with the version associated to the C/C++ compiler.
One last stumble. Version numbers over 10 have at least a .1
those under 10 a .0. There is no xlf 9.x or under currently
available. BG/P and BG/L can such a compiler mix and possibly
older version of AIX and linux on power.
"""
fver = get_compiler_version(fc, '-qversion', r'([0-9]?[0-9]\.[0-9])')
if fver >= 16:
"""Starting with version 16.1, the XL C and Fortran compilers
have the same version. So no need to downgrade the Fortran
compiler version to match that of the C compiler version.
"""
return str(fver)
cver = float(fver) - 2
if cver < 10:
cver = cver - 0.1
return str(cver)
# The fortran and C/C++ versions of the XL compiler are always
# two units apart. By this we mean that the fortran release that
# goes with XL C/C++ 11.1 is 13.1. Having such a difference in
# version number is confusing spack quite a lot. Most notably
# if you keep the versions as is the default xl compiler will
# only have fortran and no C/C++. So we associate the Fortran
# compiler with the version associated to the C/C++ compiler.
# One last stumble. Version numbers over 10 have at least a .1
# those under 10 a .0. There is no xlf 9.x or under currently
# available. BG/P and BG/L can such a compiler mix and possibly
# older version of AIX and linux on power.
fortran_version = cls.default_version(fc)
if fortran_version >= 16:
# Starting with version 16.1, the XL C and Fortran compilers
# have the same version. So no need to downgrade the Fortran
# compiler version to match that of the C compiler version.
return str(fortran_version)
c_version = float(fortran_version) - 2
if c_version < 10:
c_version = c_version - 0.1
return str(c_version)
@classmethod
def f77_version(cls, f77):

View file

@ -3,12 +3,10 @@
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
from spack.compiler import \
Compiler, get_compiler_version, UnsupportedCompilerFlag
from spack.version import ver
import spack.compilers.xl
class XlR(Compiler):
class XlR(spack.compilers.xl.Xl):
# Subclasses use possible names of C compiler
cc_names = ['xlc_r']
@ -26,86 +24,3 @@ class XlR(Compiler):
'cxx': 'xl_r/xlc++_r',
'f77': 'xl_r/xlf_r',
'fc': 'xl_r/xlf90_r'}
@property
def openmp_flag(self):
return "-qsmp=omp"
@property
def cxx11_flag(self):
if self.version < ver('13.1'):
raise UnsupportedCompilerFlag(self,
"the C++11 standard",
"cxx11_flag",
"< 13.1")
else:
return "-qlanglvl=extended0x"
@property
def pic_flag(self):
return("-qpic")
@property
def fflags(self):
# The -qzerosize flag is effective only for the Fortran 77
# compilers and allows the use of zero size objects.
# For Fortran 90 and beyond, it is set by default and has not impact.
# Its use has no negative side effects.
return "-qzerosize"
@classmethod
def default_version(cls, comp):
"""The '-qversion' is the standard option fo XL compilers.
Output looks like this::
IBM XL C/C++ for Linux, V11.1 (5724-X14)
Version: 11.01.0000.0000
or::
IBM XL Fortran for Linux, V13.1 (5724-X16)
Version: 13.01.0000.0000
or::
IBM XL C/C++ for AIX, V11.1 (5724-X13)
Version: 11.01.0000.0009
or::
IBM XL C/C++ Advanced Edition for Blue Gene/P, V9.0
Version: 09.00.0000.0017
"""
return get_compiler_version(
comp, '-qversion', r'([0-9]?[0-9]\.[0-9])')
@classmethod
def fc_version(cls, fc):
"""The fortran and C/C++ versions of the XL compiler are always
two units apart. By this we mean that the fortran release that
goes with XL C/C++ 11.1 is 13.1. Having such a difference in
version number is confusing spack quite a lot. Most notably
if you keep the versions as is the default xl compiler will
only have fortran and no C/C++. So we associate the Fortran
compiler with the version associated to the C/C++ compiler.
One last stumble. Version numbers over 10 have at least a .1
those under 10 a .0. There is no xlf 9.x or under currently
available. BG/P and BG/L can such a compiler mix and possibly
older version of AIX and linux on power.
"""
fver = get_compiler_version(fc, '-qversion', r'([0-9]?[0-9]\.[0-9])')
if fver >= 16:
"""Starting with version 16.1, the XL C and Fortran compilers
have the same version. So no need to downgrade the Fortran
compiler version to match that of the C compiler version.
"""
return str(fver)
cver = float(fver) - 2
if cver < 10:
cver = cver - 0.1
return str(cver)
@classmethod
def f77_version(cls, f77):
return cls.fc_version(f77)

View file

@ -15,7 +15,7 @@
import spack.util.pattern
from spack.version import Version
test_version = '4.5-spacktest'
test_version = '4.5.3'
@pytest.fixture()

View file

@ -11,7 +11,16 @@
import spack.spec
import spack.compiler
import spack.compilers as compilers
import spack.compilers.arm
import spack.compilers.clang
import spack.compilers.gcc
import spack.compilers.intel
import spack.compilers.nag
import spack.compilers.pgi
import spack.compilers.xl
import spack.compilers.xl_r
from spack.compiler import _get_versioned_tuple, Compiler
@ -249,5 +258,84 @@ def test_xl_r_flags():
'Thread model: posix\n', '3.1'),
])
def test_clang_version_detection(version_str, expected_version):
version = spack.compilers.clang.Clang.detect_version_from_str(version_str)
version = compilers.clang.Clang.extract_version_from_output(version_str)
assert version == expected_version
@pytest.mark.parametrize('version_str,expected_version', [
('Arm C/C++/Fortran Compiler version 19.0 (build number 73) (based on LLVM 7.0.2)\n' # NOQA
'Target: aarch64--linux-gnu\n'
'Thread model: posix\n'
'InstalledDir:\n'
'/opt/arm/arm-hpc-compiler-19.0_Generic-AArch64_RHEL-7_aarch64-linux/bin\n', # NOQA
'19.0')
])
def test_arm_version_detection(version_str, expected_version):
version = spack.compilers.arm.Arm.extract_version_from_output(version_str)
assert version == expected_version
@pytest.mark.parametrize('version_str,expected_version', [
# Output of -dumpversion changed to return only major from GCC 7
('4.4.7\n', '4.4.7'),
('7\n', '7')
])
def test_gcc_version_detection(version_str, expected_version):
version = spack.compilers.gcc.Gcc.extract_version_from_output(version_str)
assert version == expected_version
@pytest.mark.parametrize('version_str,expected_version', [
('icpc (ICC) 12.1.5 20120612\n'
'Copyright (C) 1985-2012 Intel Corporation. All rights reserved.\n',
'12.1.5'),
('ifort (IFORT) 12.1.5 20120612\n'
'Copyright (C) 1985-2012 Intel Corporation. All rights reserved.\n',
'12.1.5')
])
def test_intel_version_detection(version_str, expected_version):
version = compilers.intel.Intel.extract_version_from_output(version_str)
assert version == expected_version
@pytest.mark.parametrize('version_str,expected_version', [
('NAG Fortran Compiler Release 6.0(Hibiya) Build 1037\n'
'Product NPL6A60NA for x86-64 Linux\n', '6.0')
])
def test_nag_version_detection(version_str, expected_version):
version = spack.compilers.nag.Nag.extract_version_from_output(version_str)
assert version == expected_version
@pytest.mark.parametrize('version_str,expected_version', [
# Output on x86-64
('pgcc 15.10-0 64-bit target on x86-64 Linux -tp sandybridge\n'
'The Portland Group - PGI Compilers and Tools\n'
'Copyright (c) 2015, NVIDIA CORPORATION. All rights reserved.\n',
'15.10'),
# Output on PowerPC
('pgcc 17.4-0 linuxpower target on Linuxpower\n'
'PGI Compilers and Tools\n'
'Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved.\n', '17.4')
])
def test_pgi_version_detection(version_str, expected_version):
version = spack.compilers.pgi.Pgi.extract_version_from_output(version_str)
assert version == expected_version
@pytest.mark.parametrize('version_str,expected_version', [
('IBM XL C/C++ for Linux, V11.1 (5724-X14)\n'
'Version: 11.01.0000.0000\n', '11.1'),
('IBM XL Fortran for Linux, V13.1 (5724-X16)\n'
'Version: 13.01.0000.0000\n', '13.1'),
('IBM XL C/C++ for AIX, V11.1 (5724-X13)\n'
'Version: 11.01.0000.0009\n', '11.1'),
('IBM XL C/C++ Advanced Edition for Blue Gene/P, V9.0\n'
'Version: 09.00.0000.0017\n', '9.0')
])
def test_xl_version_detection(version_str, expected_version):
version = spack.compilers.xl.Xl.extract_version_from_output(version_str)
assert version == expected_version
version = spack.compilers.xl_r.XlR.extract_version_from_output(version_str)
assert version == expected_version