Implicit rpaths for NAG/GCC mixed toolchain (#14782)
* Implicit rpaths for NAG. * set up environment when checking for implicit rpaths
This commit is contained in:
Sergey Kosukhin
GitHub
parent
70c3b0ba09
commit
7aa9cb0f7a
6 changed files with 139 additions and 38 deletions
|
|
@ -3,6 +3,7 @@
|
|||
#
|
||||
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
|
||||
|
||||
import contextlib
|
||||
import os
|
||||
import platform
|
||||
import re
|
||||
|
|
@ -354,10 +355,11 @@ def _get_compiler_link_paths(self, paths):
|
|||
for flag_type in flags:
|
||||
for flag in self.flags.get(flag_type, []):
|
||||
compiler_exe.add_default_arg(flag)
|
||||
output = str(compiler_exe(self.verbose_flag, fin, '-o', fout,
|
||||
output=str, error=str)) # str for py2
|
||||
|
||||
return _parse_non_system_link_dirs(output)
|
||||
with self._compiler_environment():
|
||||
output = str(compiler_exe(
|
||||
self.verbose_flag, fin, '-o', fout,
|
||||
output=str, error=str)) # str for py2
|
||||
return _parse_non_system_link_dirs(output)
|
||||
except spack.util.executable.ProcessError as pe:
|
||||
tty.debug('ProcessError: Command exited with non-zero status: ' +
|
||||
pe.long_message)
|
||||
|
|
@ -468,32 +470,12 @@ def get_real_version(self):
|
|||
Use the runtime environment of the compiler (modules and environment
|
||||
modifications) to enable the compiler to run properly on any platform.
|
||||
"""
|
||||
# store environment to replace later
|
||||
backup_env = os.environ.copy()
|
||||
|
||||
# load modules and set env variables
|
||||
for module in self.modules:
|
||||
# On cray, mic-knl module cannot be loaded without cce module
|
||||
# See: https://github.com/spack/spack/issues/3153
|
||||
if os.environ.get("CRAY_CPU_TARGET") == 'mic-knl':
|
||||
spack.util.module_cmd.load_module('cce')
|
||||
spack.util.module_cmd.load_module(module)
|
||||
|
||||
# apply other compiler environment changes
|
||||
env = spack.util.environment.EnvironmentModifications()
|
||||
env.extend(spack.schema.environment.parse(self.environment))
|
||||
env.apply_modifications()
|
||||
|
||||
cc = spack.util.executable.Executable(self.cc)
|
||||
output = cc(self.version_argument,
|
||||
output=str, error=str,
|
||||
ignore_errors=tuple(self.ignore_version_errors))
|
||||
|
||||
# Restore environment
|
||||
os.environ.clear()
|
||||
os.environ.update(backup_env)
|
||||
|
||||
return self.extract_version_from_output(output)
|
||||
with self._compiler_environment():
|
||||
output = cc(self.version_argument,
|
||||
output=str, error=str,
|
||||
ignore_errors=tuple(self.ignore_version_errors))
|
||||
return self.extract_version_from_output(output)
|
||||
|
||||
#
|
||||
# Compiler classes have methods for querying the version of
|
||||
|
|
@ -562,6 +544,30 @@ def __str__(self):
|
|||
self.cc, self.cxx, self.f77, self.fc, self.modules,
|
||||
str(self.operating_system)))))
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _compiler_environment(self):
|
||||
# store environment to replace later
|
||||
backup_env = os.environ.copy()
|
||||
|
||||
# load modules and set env variables
|
||||
for module in self.modules:
|
||||
# On cray, mic-knl module cannot be loaded without cce module
|
||||
# See: https://github.com/spack/spack/issues/3153
|
||||
if os.environ.get("CRAY_CPU_TARGET") == 'mic-knl':
|
||||
spack.util.module_cmd.load_module('cce')
|
||||
spack.util.module_cmd.load_module(module)
|
||||
|
||||
# apply other compiler environment changes
|
||||
env = spack.util.environment.EnvironmentModifications()
|
||||
env.extend(spack.schema.environment.parse(self.environment))
|
||||
env.apply_modifications()
|
||||
|
||||
yield
|
||||
|
||||
# Restore environment
|
||||
os.environ.clear()
|
||||
os.environ.update(backup_env)
|
||||
|
||||
|
||||
class CompilerAccessError(spack.error.SpackError):
|
||||
|
||||
|
|
|
|||
|
|
@ -30,6 +30,36 @@ class Nag(spack.compiler.Compiler):
|
|||
version_argument = '-V'
|
||||
version_regex = r'NAG Fortran Compiler Release ([0-9.]+)'
|
||||
|
||||
@property
|
||||
def verbose_flag(self):
|
||||
# NAG does not support a flag that would enable verbose output and
|
||||
# compilation/linking at the same time (with either '-#' or '-dryrun'
|
||||
# the compiler only prints the commands but does not run them).
|
||||
# Therefore, the only thing we can do is to pass the '-v' argument to
|
||||
# the underlying GCC. In order to get verbose output from the latter
|
||||
# at both compile and linking stages, we need to call NAG with two
|
||||
# additional flags: '-Wc,-v' and '-Wl,-v'. However, we return only
|
||||
# '-Wl,-v' for the following reasons:
|
||||
# 1) the interface of this method does not support multiple flags in
|
||||
# the return value and, at least currently, verbose output at the
|
||||
# linking stage has a higher priority for us;
|
||||
# 2) NAG is usually mixed with GCC compiler, which also accepts
|
||||
# '-Wl,-v' and produces meaningful result with it: '-v' is passed
|
||||
# to the linker and the latter produces verbose output for the
|
||||
# linking stage ('-Wc,-v', however, would break the compilation
|
||||
# with a message from GCC that the flag is not recognized).
|
||||
#
|
||||
# This way, we at least enable the implicit rpath detection, which is
|
||||
# based on compilation of a C file (see method
|
||||
# spack.compiler._get_compiler_link_paths): in the case of a mixed
|
||||
# NAG/GCC toolchain, the flag will be passed to g++ (e.g.
|
||||
# 'g++ -Wl,-v ./main.c'), otherwise, the flag will be passed to nagfor
|
||||
# (e.g. 'nagfor -Wl,-v ./main.c' - note that nagfor recognizes '.c'
|
||||
# extension and treats the file accordingly). The list of detected
|
||||
# rpaths will contain only GCC-related directories and rpaths to
|
||||
# NAG-related directories are injected by nagfor anyway.
|
||||
return "-Wl,-v"
|
||||
|
||||
@property
|
||||
def openmp_flag(self):
|
||||
return "-openmp"
|
||||
|
|
|
|||
|
|
@ -157,13 +157,14 @@ def test_compiler_flags_from_config_are_grouped():
|
|||
class MockCompiler(Compiler):
|
||||
def __init__(self):
|
||||
super(MockCompiler, self).__init__(
|
||||
"badcompiler@1.0.0",
|
||||
default_compiler_entry['operating_system'],
|
||||
None,
|
||||
[default_compiler_entry['paths']['cc'],
|
||||
default_compiler_entry['paths']['cxx'],
|
||||
default_compiler_entry['paths']['fc'],
|
||||
default_compiler_entry['paths']['f77']])
|
||||
cspec="badcompiler@1.0.0",
|
||||
operating_system=default_compiler_entry['operating_system'],
|
||||
target=None,
|
||||
paths=[default_compiler_entry['paths']['cc'],
|
||||
default_compiler_entry['paths']['cxx'],
|
||||
default_compiler_entry['paths']['fc'],
|
||||
default_compiler_entry['paths']['f77']],
|
||||
environment={})
|
||||
|
||||
_get_compiler_link_paths = Compiler._get_compiler_link_paths
|
||||
|
||||
|
|
@ -267,6 +268,32 @@ def test_get_compiler_link_paths_no_verbose_flag():
|
|||
assert dirs == []
|
||||
|
||||
|
||||
def test_get_compiler_link_paths_load_env(working_env, monkeypatch, tmpdir):
|
||||
gcc = str(tmpdir.join('gcc'))
|
||||
with open(gcc, 'w') as f:
|
||||
f.write("""#!/bin/bash
|
||||
if [[ $ENV_SET == "1" && $MODULE_LOADED == "1" ]]; then
|
||||
echo '""" + no_flag_output + """'
|
||||
fi
|
||||
""")
|
||||
fs.set_executable(gcc)
|
||||
|
||||
# Set module load to turn compiler on
|
||||
def module(*args):
|
||||
if args[0] == 'show':
|
||||
return ''
|
||||
elif args[0] == 'load':
|
||||
os.environ['MODULE_LOADED'] = "1"
|
||||
monkeypatch.setattr(spack.util.module_cmd, 'module', module)
|
||||
|
||||
compiler = MockCompiler()
|
||||
compiler.environment = {'set': {'ENV_SET': '1'}}
|
||||
compiler.modules = ['turn_on']
|
||||
|
||||
dirs = compiler._get_compiler_link_paths([gcc])
|
||||
assert dirs == no_flag_dirs
|
||||
|
||||
|
||||
# Get the desired flag from the specified compiler spec.
|
||||
def flag_value(flag, spec):
|
||||
compiler = None
|
||||
|
|
|
|||
|
|
@ -0,0 +1,3 @@
|
|||
collect2 version 6.5.0
|
||||
/usr/bin/ld -plugin /scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/libexec/gcc/x86_64-pc-linux-gnu/6.5.0/liblto_plugin.so -plugin-opt=/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/libexec/gcc/x86_64-pc-linux-gnu/6.5.0/lto-wrapper -plugin-opt=-fresolution=/tmp/ccbFmewQ.res -plugin-opt=-pass-through=-lgcc_s -plugin-opt=-pass-through=-lgcc -plugin-opt=-pass-through=-lc -plugin-opt=-pass-through=-lgcc_s -plugin-opt=-pass-through=-lgcc -rpath /scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib:/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib64 --eh-frame-hdr -m elf_x86_64 -dynamic-linker /lib64/ld-linux-x86-64.so.2 -o output /usr/lib/x86_64-linux-gnu/crt1.o /usr/lib/x86_64-linux-gnu/crti.o /scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/6.5.0/crtbegin.o -L/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/6.5.0 -L/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/6.5.0/../../../../lib64 -L/lib/x86_64-linux-gnu -L/lib/../lib64 -L/usr/lib/x86_64-linux-gnu -L/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/6.5.0/../../.. -v /tmp/ccxz6i1I.o -lstdc++ -lm -lgcc_s -lgcc -lc -lgcc_s -lgcc /scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/6.5.0/crtend.o /usr/lib/x86_64-linux-gnu/crtn.o
|
||||
GNU ld (GNU Binutils for Debian) 2.28
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
NAG Fortran Compiler Release 6.2(Chiyoda) Build 6223
|
||||
Reading specs from /scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/6.5.0/specs
|
||||
COLLECT_GCC=/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/bin/gcc
|
||||
COLLECT_LTO_WRAPPER=/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/libexec/gcc/x86_64-pc-linux-gnu/6.5.0/lto-wrapper
|
||||
Target: x86_64-pc-linux-gnu
|
||||
Configured with: /tmp/m300488/spack-stage/spack-stage-gcc-6.5.0-4sdjgrsboy3lowtq3t7pmp7rx3ogkqtz/spack-src/configure --prefix=/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs --with-pkgversion='Spack GCC' --with-bugurl=https://github.com/spack/spack/issues --disable-multilib --enable-languages=c,c++,fortran --disable-nls --with-mpfr=/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/mpfr-3.1.6-w63rspk --with-gmp=/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gmp-6.1.2-et64cuj --with-system-zlib --with-mpc=/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/mpc-1.1.0-en66k4t --with-isl=/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/isl-0.18-62v4uyg
|
||||
Thread model: posix
|
||||
gcc version 6.5.0 (Spack GCC)
|
||||
COMPILER_PATH=/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/libexec/gcc/x86_64-pc-linux-gnu/6.5.0/:/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/libexec/gcc/x86_64-pc-linux-gnu/6.5.0/:/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/libexec/gcc/x86_64-pc-linux-gnu/:/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/6.5.0/:/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/
|
||||
LIBRARY_PATH=/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/6.5.0/:/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/6.5.0/../../../../lib64/:/lib/x86_64-linux-gnu/:/lib/../lib64/:/usr/lib/x86_64-linux-gnu/:/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/6.5.0/../../../:/lib/:/usr/lib/
|
||||
COLLECT_GCC_OPTIONS='-m64' '-o' 'output' '-v' '-mtune=generic' '-march=x86-64'
|
||||
/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/libexec/gcc/x86_64-pc-linux-gnu/6.5.0/collect2 -plugin /scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/libexec/gcc/x86_64-pc-linux-gnu/6.5.0/liblto_plugin.so -plugin-opt=/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/libexec/gcc/x86_64-pc-linux-gnu/6.5.0/lto-wrapper -plugin-opt=-fresolution=/tmp/ccBpU203.res -plugin-opt=-pass-through=-lgcc -plugin-opt=-pass-through=-lgcc_s -plugin-opt=-pass-through=-lc -plugin-opt=-pass-through=-lgcc -plugin-opt=-pass-through=-lgcc_s -rpath /scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib:/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib64 --eh-frame-hdr -m elf_x86_64 -dynamic-linker /lib64/ld-linux-x86-64.so.2 -o output /usr/lib/x86_64-linux-gnu/crt1.o /usr/lib/x86_64-linux-gnu/crti.o /scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/6.5.0/crtbegin.o -L/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/6.5.0 -L/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/6.5.0/../../../../lib64 -L/lib/x86_64-linux-gnu -L/lib/../lib64 -L/usr/lib/x86_64-linux-gnu -L/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/6.5.0/../../.. /sw/stretch-x64/nag/nag-6.2/lib/NAG_Fortran/f62init.o /sw/stretch-x64/nag/nag-6.2/lib/NAG_Fortran/quickfit.o /tmp/main.000786.o -rpath /sw/stretch-x64/nag/nag-6.2/lib/NAG_Fortran /sw/stretch-x64/nag/nag-6.2/lib/NAG_Fortran/libf62rts.so /sw/stretch-x64/nag/nag-6.2/lib/NAG_Fortran/libf62rts.a -lm -lgcc --as-needed -lgcc_s --no-as-needed -lc -lgcc --as-needed -lgcc_s --no-as-needed /scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/6.5.0/crtend.o /usr/lib/x86_64-linux-gnu/crtn.o
|
||||
COLLECT_GCC_OPTIONS='-m64' '-o' 'output' '-v' '-mtune=generic' '-march=x86-64'
|
||||
|
|
@ -32,8 +32,8 @@ def check_link_paths(filename, paths):
|
|||
|
||||
def test_icc16_link_paths():
|
||||
check_link_paths('icc-16.0.3.txt', [
|
||||
'/usr/tce/packages/intel/intel-16.0.3/compilers_and_libraries_2016.3.210/linux/compiler/lib/intel64_lin', # noqa
|
||||
'/usr/tce/packages/gcc/gcc-4.9.3/lib64/gcc/x86_64-unknown-linux-gnu/4.9.3', # noqa
|
||||
'/usr/tce/packages/intel/intel-16.0.3/compilers_and_libraries_2016.3.210/linux/compiler/lib/intel64_lin', # noqa
|
||||
'/usr/tce/packages/gcc/gcc-4.9.3/lib64/gcc/x86_64-unknown-linux-gnu/4.9.3', # noqa
|
||||
'/usr/tce/packages/gcc/gcc-4.9.3/lib64'])
|
||||
|
||||
|
||||
|
|
@ -82,6 +82,28 @@ def test_clang_apple_ld_link_paths():
|
|||
'/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.13.sdk/usr/lib']) # noqa
|
||||
|
||||
|
||||
def test_nag_mixed_gcc_gnu_ld_link_paths():
|
||||
# This is a test of a mixed NAG/GCC toolchain, i.e. 'cxx' is set to g++ and
|
||||
# is used for the rpath detection. The reference compiler output is a
|
||||
# result of
|
||||
# '/path/to/gcc/bin/g++ -Wl,-v ./main.c'.
|
||||
check_link_paths('collect2-6.3.0-gnu-ld.txt', [
|
||||
'/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/6.5.0', # noqa
|
||||
'/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib64', # noqa
|
||||
'/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib']) # noqa
|
||||
|
||||
|
||||
def test_nag_link_paths():
|
||||
# This is a test of a NAG-only toolchain, i.e. 'cc' and 'cxx' are empty,
|
||||
# and therefore 'fc' is used for the rpath detection). The reference
|
||||
# compiler output is a result of
|
||||
# 'nagfor -Wc=/path/to/gcc/bin/gcc -Wl,-v ./main.c'.
|
||||
check_link_paths('nag-6.2-gcc-6.5.0.txt', [
|
||||
'/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/6.5.0', # noqa
|
||||
'/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib64', # noqa
|
||||
'/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib']) # noqa
|
||||
|
||||
|
||||
def test_obscure_parsing_rules():
|
||||
check_link_paths('obscure-parsing-rules.txt', [
|
||||
'/first/path',
|
||||
|
|
|
|||
Loading…
Reference in a new issue