diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py
index af3329ed56..9db4d4dd73 100644
--- a/lib/spack/spack/build_environment.py
+++ b/lib/spack/spack/build_environment.py
@@ -167,12 +167,6 @@ def set_compiler_environment_variables(pkg, env):
 
     env.set('SPACK_COMPILER_SPEC', str(pkg.spec.compiler))
 
-    for mod in compiler.modules:
-        # Fixes issue https://github.com/LLNL/spack/issues/3153
-        if os.environ.get("CRAY_CPU_TARGET") == "mic-knl":
-            load_module("cce")
-        load_module(mod)
-
     compiler.setup_custom_environment(pkg, env)
 
     return env
@@ -312,9 +306,6 @@ def set_build_environment_variables(pkg, env, dirty):
             if os.path.isdir(pcdir):
                 env.prepend_path('PKG_CONFIG_PATH', pcdir)
 
-    if pkg.architecture.target.module_name:
-        load_module(pkg.architecture.target.module_name)
-
     return env
 
 
@@ -484,7 +475,7 @@ def setup_package(pkg, dirty):
     set_compiler_environment_variables(pkg, spack_env)
     set_build_environment_variables(pkg, spack_env, dirty)
     pkg.architecture.platform.setup_platform_environment(pkg, spack_env)
-    load_external_modules(pkg)
+
     # traverse in postorder so package can use vars from its dependencies
     spec = pkg.spec
     for dspec in pkg.spec.traverse(order='post', root=False, deptype='build'):
@@ -511,6 +502,21 @@ def setup_package(pkg, dirty):
     validate(spack_env, tty.warn)
     spack_env.apply_modifications()
 
+    # All module loads that otherwise would belong in previous functions
+    # have to occur after the spack_env object has its modifications applied.
+    # Otherwise the environment modifications could undo module changes, such
+    # as unsetting LD_LIBRARY_PATH after a module changes it.
+    for mod in pkg.compiler.modules:
+        # Fixes issue https://github.com/LLNL/spack/issues/3153
+        if os.environ.get("CRAY_CPU_TARGET") == "mic-knl":
+            load_module("cce")
+        load_module(mod)
+
+    if pkg.architecture.target.module_name:
+        load_module(pkg.architecture.target.module_name)
+
+    load_external_modules(pkg)
+
 
 def fork(pkg, function, dirty):
     """Fork a child process to do part of a spack build.