hlrs-software-stack/spack
7
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Extended set of environment modification commands. (#8996)

Browse source
This commit is contained in:
Sergey Kosukhin 2018-09-05 19:56:45 +02:00 committed by becker33
parent 501669faba
commit f9617b2ad8
5 changed files with 217 additions and 88 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
lib/spack/docs/basic_usage.rst
View file

@ -684,11 +684,12 @@ Compiler environment variables and additional RPATHs
In the exceptional case a compiler requires setting special environment In the exceptional case a compiler requires setting special environment
variables, like an explicit library load path. These can bet set in an variables, like an explicit library load path. These can bet set in an
extra section in the compiler configuration. The user can also specify extra section in the compiler configuration (the supported environment
additional ``RPATHs`` that the compiler will add to all executables modification commands are: ``set``, ``unset``, ``append-path``, and
generated by that compiler. This is useful for forcing certain compilers ``prepend-path``). The user can also specify additional ``RPATHs`` that the
to RPATH their own runtime libraries, so that executables will run compiler will add to all executables generated by that compiler. This is
without the need to set ``LD_LIBRARY_PATH``. useful for forcing certain compilers to RPATH their own runtime libraries, so
that executables will run without the need to set ``LD_LIBRARY_PATH``.
.. code-block:: yaml .. code-block:: yaml
@ -701,12 +702,29 @@ without the need to set ``LD_LIBRARY_PATH``.
f77: /opt/gcc/bin/gfortran f77: /opt/gcc/bin/gfortran
fc: /opt/gcc/bin/gfortran fc: /opt/gcc/bin/gfortran
environment: environment:
unset:
BAD_VARIABLE: # The colon is required but the value must be empty
set: set:
LD_LIBRARY_PATH : /opt/gcc/lib GOOD_VARIABLE_NUM: 1
GOOD_VARIABLE_STR: good
prepend-path:
PATH: /path/to/binutils
append-path:
LD_LIBRARY_PATH: /opt/gcc/lib
extra_rpaths: extra_rpaths:
- /path/to/some/compiler/runtime/directory - /path/to/some/compiler/runtime/directory
- /path/to/some/other/compiler/runtime/directory - /path/to/some/other/compiler/runtime/directory
.. note::
The section `environment` is interpreted as an ordered dictionary, which
means two things. First, environment modification are applied in the order
they are specified in the configuration file. Second, you cannot express
environment modifications that require mixing different commands, i.e. you
cannot `set` one variable, than `prepend-path` to another one, and than
again `set` a third one.
^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^^^^^
Architecture specifiers Architecture specifiers
^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^^^^^

12
lib/spack/env/cc vendored
View file

@ -202,18 +202,6 @@ if [[ -z $command ]]; then
die "ERROR: Compiler '$SPACK_COMPILER_SPEC' does not support compiling $language programs." die "ERROR: Compiler '$SPACK_COMPILER_SPEC' does not support compiling $language programs."
fi fi
#
# Set paths as defined in the 'environment' section of the compiler config
# names are stored in SPACK_ENV_TO_SET
# values are stored in SPACK_ENV_SET_<varname>
#
IFS=':' read -ra env_set_varnames <<< "$SPACK_ENV_TO_SET"
for varname in "${env_set_varnames[@]}"; do
spack_varname="SPACK_ENV_SET_$varname"
export "$varname"="${!spack_varname}"
unset "$spack_varname"
done
# #
# Filter '.' and Spack environment directories out of PATH so that # Filter '.' and Spack environment directories out of PATH so that
# this script doesn't just call itself # this script doesn't just call itself

22
lib/spack/spack/build_environment.py
View file

@ -303,14 +303,20 @@ def set_build_environment_variables(pkg, env, dirty):
# the given compiler # the given compiler
compiler = pkg.compiler compiler = pkg.compiler
environment = compiler.environment environment = compiler.environment
if 'set' in environment:
env_to_set = environment['set'] for command, variable in iteritems(environment):
for key, value in iteritems(env_to_set): if command == 'set':
env.set('SPACK_ENV_SET_%s' % key, value) for name, value in iteritems(variable):
env.set('%s' % key, value) env.set(name, value)
# Let shell know which variables to set elif command == 'unset':
env_variables = ":".join(env_to_set.keys()) for name, _ in iteritems(variable):
env.set('SPACK_ENV_TO_SET', env_variables) env.unset(name)
elif command == 'prepend-path':
for name, value in iteritems(variable):
env.prepend_path(name, value)
elif command == 'append-path':
for name, value in iteritems(variable):
env.append_path(name, value)
if compiler.extra_rpaths: if compiler.extra_rpaths:
extra_rpaths = ':'.join(compiler.extra_rpaths) extra_rpaths = ':'.join(compiler.extra_rpaths)

158
lib/spack/spack/schema/compilers.py
View file

@ -34,75 +34,109 @@
'title': 'Spack compiler configuration file schema', 'title': 'Spack compiler configuration file schema',
'type': 'object', 'type': 'object',
'additionalProperties': False, 'additionalProperties': False,
'patternProperties': { 'properties': {
'compilers': { 'compilers': {
'type': 'array', 'type': 'array',
'items': { 'items': {
'compiler': { 'type': 'object',
'type': 'object', 'additionalProperties': False,
'additionalProperties': False, 'properties': {
'required': [ 'compiler': {
'paths', 'spec', 'modules', 'operating_system'], 'type': 'object',
'properties': { 'additionalProperties': False,
'paths': { 'required': [
'type': 'object', 'paths', 'spec', 'modules', 'operating_system'],
'required': ['cc', 'cxx', 'f77', 'fc'], 'properties': {
'additionalProperties': False, 'paths': {
'properties': { 'type': 'object',
'cc': {'anyOf': [{'type': 'string'}, 'required': ['cc', 'cxx', 'f77', 'fc'],
{'type': 'null'}]}, 'additionalProperties': False,
'cxx': {'anyOf': [{'type': 'string'}, 'properties': {
{'type': 'null'}]}, 'cc': {'anyOf': [{'type': 'string'},
'f77': {'anyOf': [{'type': 'string'},
{'type': 'null'}]},
'fc': {'anyOf': [{'type': 'string'},
{'type': 'null'}]}}},
'flags': {
'type': 'object',
'additionalProperties': False,
'properties': {
'cflags': {'anyOf': [{'type': 'string'},
{'type': 'null'}]},
'cxxflags': {'anyOf': [{'type': 'string'},
{'type': 'null'}]},
'fflags': {'anyOf': [{'type': 'string'},
{'type': 'null'}]},
'cppflags': {'anyOf': [{'type': 'string'},
{'type': 'null'}]},
'ldflags': {'anyOf': [{'type': 'string'},
{'type': 'null'}]}, {'type': 'null'}]},
'ldlibs': {'anyOf': [{'type': 'string'}, 'cxx': {'anyOf': [{'type': 'string'},
{'type': 'null'}]}}}, {'type': 'null'}]},
'spec': {'type': 'string'}, 'f77': {'anyOf': [{'type': 'string'},
'operating_system': {'type': 'string'}, {'type': 'null'}]},
'alias': {'anyOf': [{'type': 'string'}, 'fc': {'anyOf': [{'type': 'string'},
{'type': 'null'}]}, {'type': 'null'}]}}},
'modules': {'anyOf': [{'type': 'string'}, 'flags': {
{'type': 'null'}, 'type': 'object',
{'type': 'array'}]}, 'additionalProperties': False,
'environment': { 'properties': {
'type': 'object', 'cflags': {'anyOf': [{'type': 'string'},
'default': {}, {'type': 'null'}]},
'additionalProperties': False, 'cxxflags': {'anyOf': [{'type': 'string'},
'properties': { {'type': 'null'}]},
'set': { 'fflags': {'anyOf': [{'type': 'string'},
'type': 'object', {'type': 'null'}]},
'patternProperties': { 'cppflags': {'anyOf': [{'type': 'string'},
r'\w[\w-]*': { # variable name {'type': 'null'}]},
'type': 'string' 'ldflags': {'anyOf': [{'type': 'string'},
{'type': 'null'}]},
'ldlibs': {'anyOf': [{'type': 'string'},
{'type': 'null'}]}}},
'spec': {'type': 'string'},
'operating_system': {'type': 'string'},
'target': {'type': 'string'},
'alias': {'anyOf': [{'type': 'string'},
{'type': 'null'}]},
'modules': {'anyOf': [{'type': 'string'},
{'type': 'null'},
{'type': 'array'}]},
'environment': {
'type': 'object',
'default': {},
'additionalProperties': False,
'properties': {
'set': {
'type': 'object',
'patternProperties': {
# Variable name
r'\w[\w-]*': {
'anyOf': [{'type': 'string'},
{'type': 'number'}]
}
}
},
'unset': {
'type': 'object',
'patternProperties': {
# Variable name
r'\w[\w-]*': {'type': 'null'}
}
},
'prepend-path': {
'type': 'object',
'patternProperties': {
# Variable name
r'\w[\w-]*': {
'anyOf': [{'type': 'string'},
{'type': 'number'}]
}
}
},
'append-path': {
'type': 'object',
'patternProperties': {
# Variable name
r'\w[\w-]*': {
'anyOf': [{'type': 'string'},
{'type': 'number'}]
}
} }
} }
} }
},
'extra_rpaths': {
'type': 'array',
'default': [],
'items': {'type': 'string'}
} }
},
'extra_rpaths': {
'type': 'array',
'default': [],
'items': {'type': 'string'}
} }
}, }
}, }
}, }
}, }
}, }
} }

83
lib/spack/spack/test/build_environment.py
View file

@ -30,6 +30,7 @@
from spack.paths import build_env_path from spack.paths import build_env_path
from spack.build_environment import dso_suffix, _static_to_shared_library from spack.build_environment import dso_suffix, _static_to_shared_library
from spack.util.executable import Executable from spack.util.executable import Executable
from spack.util.spack_yaml import syaml_dict, syaml_str
@pytest.fixture @pytest.fixture
@ -126,3 +127,85 @@ def _set_wrong_cc(x):
assert os.environ['CC'] != 'NOT_THIS_PLEASE' assert os.environ['CC'] != 'NOT_THIS_PLEASE'
assert os.environ['ANOTHER_VAR'] == 'THIS_IS_SET' assert os.environ['ANOTHER_VAR'] == 'THIS_IS_SET'
@pytest.mark.usefixtures('config', 'mock_packages')
def test_compiler_config_modifications(monkeypatch):
s = spack.spec.Spec('cmake')
s.concretize()
pkg = s.package
os.environ['SOME_VAR_STR'] = ''
os.environ['SOME_VAR_NUM'] = '0'
os.environ['PATH_LIST'] = '/path/third:/path/forth'
os.environ['EMPTY_PATH_LIST'] = ''
os.environ.pop('NEW_PATH_LIST', None)
env_mod = syaml_dict()
set_cmd = syaml_dict()
env_mod[syaml_str('set')] = set_cmd
set_cmd[syaml_str('SOME_VAR_STR')] = syaml_str('SOME_STR')
set_cmd[syaml_str('SOME_VAR_NUM')] = 1
monkeypatch.setattr(pkg.compiler, 'environment', env_mod)
spack.build_environment.setup_package(pkg, False)
assert os.environ['SOME_VAR_STR'] == 'SOME_STR'
assert os.environ['SOME_VAR_NUM'] == str(1)
env_mod = syaml_dict()
unset_cmd = syaml_dict()
env_mod[syaml_str('unset')] = unset_cmd
unset_cmd[syaml_str('SOME_VAR_STR')] = None
monkeypatch.setattr(pkg.compiler, 'environment', env_mod)
assert 'SOME_VAR_STR' in os.environ
spack.build_environment.setup_package(pkg, False)
assert 'SOME_VAR_STR' not in os.environ
env_mod = syaml_dict()
set_cmd = syaml_dict()
env_mod[syaml_str('set')] = set_cmd
append_cmd = syaml_dict()
env_mod[syaml_str('append-path')] = append_cmd
unset_cmd = syaml_dict()
env_mod[syaml_str('unset')] = unset_cmd
prepend_cmd = syaml_dict()
env_mod[syaml_str('prepend-path')] = prepend_cmd
set_cmd[syaml_str('EMPTY_PATH_LIST')] = syaml_str('/path/middle')
append_cmd[syaml_str('PATH_LIST')] = syaml_str('/path/last')
append_cmd[syaml_str('EMPTY_PATH_LIST')] = syaml_str('/path/last')
append_cmd[syaml_str('NEW_PATH_LIST')] = syaml_str('/path/last')
unset_cmd[syaml_str('SOME_VAR_NUM')] = None
prepend_cmd[syaml_str('PATH_LIST')] = syaml_str('/path/first:/path/second')
prepend_cmd[syaml_str('EMPTY_PATH_LIST')] = syaml_str('/path/first')
prepend_cmd[syaml_str('NEW_PATH_LIST')] = syaml_str('/path/first')
prepend_cmd[syaml_str('SOME_VAR_NUM')] = syaml_str('/8')
assert 'SOME_VAR_NUM' in os.environ
monkeypatch.setattr(pkg.compiler, 'environment', env_mod)
spack.build_environment.setup_package(pkg, False)
# Check that the order of modifications is respected and the
# variable was unset before it was prepended.
assert os.environ['SOME_VAR_NUM'] == '/8'
expected = '/path/first:/path/second:/path/third:/path/forth:/path/last'
assert os.environ['PATH_LIST'] == expected
expected = '/path/first:/path/middle:/path/last'
assert os.environ['EMPTY_PATH_LIST'] == expected
expected = '/path/first:/path/last'
assert os.environ['NEW_PATH_LIST'] == expected
os.environ.pop('SOME_VAR_STR', None)
os.environ.pop('SOME_VAR_NUM', None)
os.environ.pop('PATH_LIST', None)
os.environ.pop('EMPTY_PATH_LIST', None)
os.environ.pop('NEW_PATH_LIST', None)