Allow compiler wrapper to modify environment (#2275)

* Allow compiler wrapper to modify environment

This adds the ability to set environment variables in the compiler
wrappers. These are specified as part of the compilers.yaml config.
The user may also specify RPATHs in compilers.yaml that should be
added.

* Minor doc tweak

lib/spack/docs/basic_usage.rst

@@ -672,6 +672,35 @@ in GNU Autotools. If all flags are set, the order is
 ``$cppflags $cflags|$cxxflags $ldflags <command> $ldlibs`` for C and C++ and
 ``$fflags $cppflags $ldflags <command> $ldlibs`` for Fortran.
 
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Compiler environment variables and additional RPATHs
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+In the exceptional case a compiler requires setting special environment
+variables, like an explicit library load path. These can bet set in an
+extra section in the compiler configuration.  The user can also specify
+additional ``RPATHs`` that the compiler will add to all executables
+generated by that compiler.  This is useful for forcing certain compilers
+to RPATH their own runtime libraries, so that executables will run
+without the need to set ``LD_LIBRARY_PATH``.
+
+.. code-block:: yaml
+
+  compilers:
+    - compiler:
+        spec: gcc@4.9.3
+        paths:
+          cc: /opt/gcc/bin/gcc
+          c++: /opt/gcc/bin/g++
+          f77: /opt/gcc/bin/gfortran
+          fc: /opt/gcc/bin/gfortran
+        environment:
+          set:
+            LD_LIBRARY_PATH : /opt/gcc/lib
+        extra_rpaths:
+        - /path/to/some/compiler/runtime/directory
+        - /path/to/some/other/compiler/runtime/directory
+
 ^^^^^^^^^^^^^^^^^^^^^^^
 Architecture specifiers
 ^^^^^^^^^^^^^^^^^^^^^^^

lib/spack/env/cc

@@ -174,6 +174,18 @@ if [[ -z $command ]]; then
     die "ERROR: Compiler '$SPACK_COMPILER_SPEC' does not support compiling $language programs."
 fi
 
+#
+# Set paths as defined in the 'environment' section of the compiler config
+#   names are stored in SPACK_ENV_TO_SET
+#   values are stored in SPACK_ENV_SET_<varname>
+#
+IFS=':' read -ra env_set_varnames <<< "$SPACK_ENV_TO_SET"
+for varname in "${env_set_varnames[@]}"; do
+    spack_varname="SPACK_ENV_SET_$varname"
+    export $varname=${!spack_varname}
+    unset $spack_varname
+done
+
 #
 # Filter '.' and Spack environment directories out of PATH so that
 # this script doesn't just call itself
@@ -311,6 +323,16 @@ elif [[ $mode == ld ]]; then
     $add_rpaths && args=("-rpath" "$SPACK_PREFIX/lib"   "${args[@]}")
 fi
 
+# Set extra RPATHs
+IFS=':' read -ra extra_rpaths <<< "$SPACK_COMPILER_EXTRA_RPATHS"
+for extra_rpath in "${extra_rpaths[@]}"; do
+    if [[ $mode == ccld ]]; then
+        $add_rpaths && args=("$rpath$extra_rpath" "${args[@]}")
+    elif [[ $mode == ld ]]; then
+        $add_rpaths && args=("-rpath" "$extra_rpath" "${args[@]}")
+    fi
+done
+
 # Add SPACK_LDLIBS to args
 case "$mode" in
     ld|ccld)

lib/spack/spack/build_environment.py

@@ -302,6 +302,23 @@ def set_build_environment_variables(pkg, env, dirty=False):
             if '/macports/' in p:
                 env.remove_path('PATH', p)
 
+    # Set environment variables if specified for
+    # the given compiler
+    compiler = pkg.compiler
+    environment = compiler.environment
+    if 'set' in environment:
+        env_to_set = environment['set']
+        for key, value in env_to_set.iteritems():
+            env.set('SPACK_ENV_SET_%s' % key, value)
+            env.set('%s' % key, value)
+        # Let shell know which variables to set
+        env_variables = ":".join(env_to_set.keys())
+        env.set('SPACK_ENV_TO_SET', env_variables)
+
+    if compiler.extra_rpaths:
+        extra_rpaths = ':'.join(compiler.extra_rpaths)
+        env.set('SPACK_COMPILER_EXTRA_RPATHS', extra_rpaths)
+
     # Add bin directories from dependencies to the PATH for the build.
     bin_dirs = reversed(filter(os.path.isdir, [
         '%s/bin' % d.prefix for d in pkg.spec.dependencies(deptype='build')]))

lib/spack/spack/cmd/compiler.py

@@ -140,6 +140,16 @@ def compiler_info(args):
                 print "\tflags:"
                 for flag, flag_value in c.flags.iteritems():
                     print "\t\t%s = %s" % (flag, flag_value)
+            if len(c.environment) != 0:
+                if len(c.environment['set']) != 0:
+                    print "\tenvironment:"
+                    print "\t    set:"
+                    for key, value in c.environment['set'].iteritems():
+                        print "\t        %s = %s" % (key, value)
+            if c.extra_rpaths:
+                print "\tExtra rpaths:"
+                for extra_rpath in c.extra_rpaths:
+                    print "\t\t%s" % extra_rpath
             print "\tmodules  = %s" % c.modules
             print "\toperating system  = %s" % c.operating_system
 

lib/spack/spack/compiler.py

@@ -113,7 +113,8 @@ def fc_rpath_arg(self):
     PrgEnv_compiler = None
 
     def __init__(self, cspec, operating_system,
-                 paths, modules=[], alias=None, **kwargs):
+                 paths, modules=[], alias=None, environment=None,
+                 extra_rpaths=None, **kwargs):
         self.operating_system = operating_system
         self.spec = cspec
         self.modules = modules
@@ -135,6 +136,9 @@ def check(exe):
             else:
                 self.fc  = check(paths[3])
 
+        self.environment = environment
+        self.extra_rpaths = extra_rpaths or []
+
         # Unfortunately have to make sure these params are accepted
         # in the same order they are returned by sorted(flags)
         # in compilers/__init__.py

lib/spack/spack/compilers/__init__.py

@@ -40,7 +40,8 @@
 
 _imported_compilers_module = 'spack.compilers'
 _path_instance_vars = ['cc', 'cxx', 'f77', 'fc']
-_other_instance_vars = ['modules', 'operating_system']
+_other_instance_vars = ['modules', 'operating_system', 'environment',
+                        'extra_rpaths']
 _cache_config_file = []
 
 
@@ -61,6 +62,8 @@ def _to_dict(compiler):
     d['flags'] = dict((fname, fvals) for fname, fvals in compiler.flags)
     d['operating_system'] = str(compiler.operating_system)
     d['modules'] = compiler.modules if compiler.modules else []
+    d['environment'] = compiler.environment if compiler.environment else {}
+    d['extra_rpaths'] = compiler.extra_rpaths if compiler.extra_rpaths else []
 
     if compiler.alias:
         d['alias'] = compiler.alias
@@ -238,11 +241,13 @@ def get_compilers(cspec):
                     items['operating_system'], platform)
 
             alias = items.get('alias', None)
-
             compiler_flags = items.get('flags', {})
+            environment = items.get('environment', {})
+            extra_rpaths = items.get('extra_rpaths', [])
 
             compilers.append(
-                cls(cspec, os, compiler_paths, mods, alias, **compiler_flags))
+                cls(cspec, os, compiler_paths, mods, alias, environment,
+                    extra_rpaths, **compiler_flags))
 
         return compilers
 

lib/spack/spack/schema/compilers.py

@@ -79,7 +79,27 @@
                                             {'type': 'null'}]},
                         'modules': {'anyOf': [{'type': 'string'},
                                               {'type': 'null'},
-                                              {'type': 'array'}]}
+                                              {'type': 'array'}]},
+                        'environment': {
+                            'type': 'object',
+                            'default': {},
+                            'additionalProperties': False,
+                            'properties': {
+                                'set': {
+                                    'type': 'object',
+                                    'patternProperties': {
+                                        r'\w[\w-]*': {  # variable name
+                                            'type': 'string'
+                                        }
+                                    }
+                                }
+                            }
+                        },
+                        'extra_rpaths': {
+                            'type': 'array',
+                            'default': [],
+                            'items': {'type': 'string'}
+                        }
                     },
                 },
             },