Merge branch 'features/spackathon' of https://github.com/NERSC/spack into features/spackathon

Merge Gregory Becker's commits with my own architecture commits

lib/spack/spack/build_environment.py

@@ -86,6 +86,28 @@ def __call__(self, *args, **kwargs):
         return super(MakeExecutable, self).__call__(*args, **kwargs)
 
 
+def load_module(mod):
+    """Takes a module name and removes modules until it is possible to
+    load that module. It then loads the provided module. Depends on the 
+    modulecmd implementation of modules used in cray and lmod.
+    """
+    #Create an executable of the module command that will output python code
+    modulecmd = which('modulecmd')
+    modulecmd.add_default_arg('python')
+    
+    # Read the module and remove any conflicting modules
+    # We do this without checking that they are already installed
+    # for ease of programming because unloading a module that is not
+    # loaded does nothing.
+    text = modulecmd('show', mod, return_oe=True).split()
+    for i, word in enumerate(text):
+        if word == 'conflict':
+            exec(compile(modulecmd('unload', text[i+1], return_oe=True), '<string>', 'exec'))    
+    # Load the module now that there are no conflicts
+    load = modulecmd('load', mod, return_oe=True)
+    exec(compile(load, '<string>', 'exec'))
+
+
 def set_compiler_environment_variables(pkg):
     assert(pkg.spec.concrete)
     compiler = pkg.compiler
@@ -108,11 +130,9 @@ def set_compiler_environment_variables(pkg):
 
     os.environ['SPACK_COMPILER_SPEC']  = str(pkg.spec.compiler)
 
-    if compiler.PrgEnv:
+    if compiler.modules:
-        os.environ['SPACK_CRAYPE']       = compiler.PrgEnv
+        for mod in compiler.modules:
-        os.environ['SPACK_COMP_MODULE']  = compiler.module
+            load_module(mod)
-
-
 
 
 def set_build_environment_variables(pkg):
@@ -163,8 +183,10 @@ def set_build_environment_variables(pkg):
             pcdir = join_path(p, libdir, 'pkgconfig')
             if os.path.isdir(pcdir):
                 pkg_config_dirs.append(pcdir)
-    path_set("PKG_CONFIG_PATH", pkg_config_dirs)
+    path_put_first("PKG_CONFIG_PATH", pkg_config_dirs)
 
+    if pkg.spec.architecture.compiler_strategy.lower() == 'module':
+        load_module(pkg.spec.architecture.module_name)
 
 def set_module_variables_for_package(pkg):
     """Populate the module scope of install() with some useful functions.
@@ -239,8 +261,8 @@ def get_rpaths(pkg):
 
 def setup_package(pkg):
     """Execute all environment setup routines."""
-    set_compiler_environment_variables(pkg)
     set_build_environment_variables(pkg)
+    set_compiler_environment_variables(pkg)
     set_module_variables_for_package(pkg)
 
     # Allow dependencies to set up environment as well.

lib/spack/spack/concretize.py

@@ -122,6 +122,34 @@ def concretize_architecture(self, spec):
         return True   # changed
 
 
+    def new_concretize_architecture(self, spec):
+        """If the spec already has an architecture and it is a an architecture type,
+        return. Otherwise, if it has an architecture that is a string type, generate an
+        architecture based on that type. If it has no architecture and the root of the
+        DAG has an architecture, then use that. Otherwise, take the system's default
+        architecture.
+        """
+        if spec.architecture is not None:
+            if isinstance(spec.architecture,spack.architecture.Target):
+                return False
+            else:
+                arch = spack.architecture.sys_type()
+                spec.architecture = arch.target(spec.architecture)
+                return True #changed
+
+        if spec.root.architecture:
+            if isinstance(spec.root.architecture,spack.architecture.Target):
+                spec.architecture = spec.root.architecture
+            else:
+                arch = spack.architecture.sys_type()
+                spec.architecture = arch.target(spec.root.architecture)
+        else:
+            arch = spack.architecture.sys_type()
+            spec.architecture = arch.target('default')
+        
+        return True #changed
+
+
     def concretize_variants(self, spec):
         """If the spec already has variants filled in, return.  Otherwise, add
            the default variants from the package specification.

lib/spack/spack/util/executable.py

@@ -56,7 +56,11 @@ def command(self):
 
     def __call__(self, *args, **kwargs):
         """Run the executable with subprocess.check_output, return output."""
-        return_output = kwargs.get("return_output", False)
+        # Return oe returns a combined stream, setting both output and error 
+        # without setting return oe returns them concatenated by a double line break
+        return_oe    = kwargs.get("return_oe", False)
+        return_output = True if return_oe else kwargs.get("return_output", False)
+        return_error  = True if return_oe else kwargs.get("return_error", False)
         fail_on_error = kwargs.get("fail_on_error", True)
         ignore_errors = kwargs.get("ignore_errors", ())
 
@@ -95,8 +99,8 @@ def streamify(arg, mode):
             proc = subprocess.Popen(
                 cmd,
                 stdin=input,
-                stderr=error,
+                stdout=subprocess.PIPE if return_output else output,
-                stdout=subprocess.PIPE if return_output else output)
+                stderr=subprocess.STDOUT if return_oe else (subprocess.PIPE if return_error else error))
             out, err = proc.communicate()
             self.returncode = proc.returncode
 
@@ -104,8 +108,15 @@ def streamify(arg, mode):
             if fail_on_error and rc != 0 and (rc not in ignore_errors):
                 raise ProcessError("Command exited with status %d:"
                                    % proc.returncode, cmd_line)
-            if return_output:
+            # Return out or error if specified. Return combined stream if requested,
-                return out
+            # otherwise return them concatenated by double line break if both requested.
+            if return_output or return_error:
+                if return_oe or not return_error:
+                    return out
+                elif return_output:
+                    return out+'\n\n'+err
+                else:
+                    return err
 
         except OSError, e:
             raise ProcessError(