Allow relative paths in config files (relative to file dirname) (#21996)

This allows users to use relative paths for mirrors and repos and other things that may be part of a Spack environment.  There are two ways to do it.

1. Relative to the file

    ```yaml
    spack:
      repos:
      - local_dir/my_repository
    ```

    Which will refer to a repository like this in the directory where `spack.yaml` lives:

    ```
    env/
      spack.yaml  <-- the config file above
      local_dir/
        my_repository/  <-- this repository
          repo.yaml
          packages/
    ```

2. Relative to the environment

    ```yaml
    spack:
      repos:
      - $env/local_dir/my_repository
    ```

Both of these would refer to the same directory, but they differ for included files.  For example, if you had this layout:

```
env/
    spack.yaml
    repository/
    includes/
        repos.yaml
        repository/
```

And this `spack.yaml`:

```yaml
spack:
    include: includes/repos.yaml
```

Then, these two `repos.yaml` files are functionally different:

```yaml
repos:
    - $env/repository    # refers to env/repository/ above

repos:
    - repository    # refers to env/includes/repository/ above
```
    
The $env variable will not be evaluated if there is no active environment. This generally means that it should not be used outside of an environment's spack.yaml file. However, if other aspects of your workflow guarantee that there is always an active environment, it may be used in other config scopes.

lib/spack/spack/test/config.py

@@ -16,6 +16,7 @@
 import spack.paths
 import spack.config
 import spack.main
+import spack.environment
 import spack.schema.compilers
 import spack.schema.config
 import spack.schema.env
@@ -267,7 +268,12 @@ def test_write_list_in_memory(mock_low_high_config):
     assert config == repos_high['repos'] + repos_low['repos']
 
 
-def test_substitute_config_variables(mock_low_high_config):
+class MockEnv(object):
+    def __init__(self, path):
+        self.path = path
+
+
+def test_substitute_config_variables(mock_low_high_config, monkeypatch):
     prefix = spack.paths.prefix.lstrip('/')
 
     assert os.path.join(
@@ -298,6 +304,33 @@ def test_substitute_config_variables(mock_low_high_config):
         '/foo/bar/baz', prefix, 'foo/bar/baz'
     ) != spack_path.canonicalize_path('/foo/bar/baz/${spack/foo/bar/baz/')
 
+    # $env replacement is a no-op when no environment is active
+    assert spack_path.canonicalize_path(
+        '/foo/bar/baz/$env'
+    ) == '/foo/bar/baz/$env'
+
+    # Fake an active environment and $env is replaced properly
+    fake_env_path = '/quux/quuux'
+    monkeypatch.setattr(spack.environment, 'get_env',
+                        lambda x, y: MockEnv(fake_env_path))
+    assert spack_path.canonicalize_path(
+        '$env/foo/bar/baz'
+    ) == os.path.join(fake_env_path, 'foo/bar/baz')
+
+    # relative paths without source information are relative to cwd
+    assert spack_path.canonicalize_path(
+        'foo/bar/baz'
+    ) == os.path.abspath('foo/bar/baz')
+
+    # relative paths with source information are relative to the file
+    spack.config.set(
+        'config:module_roots', {'lmod': 'foo/bar/baz'}, scope='low')
+    spack.config.config.clear_caches()
+    path = spack.config.get('config:module_roots:lmod')
+    assert spack_path.canonicalize_path(path) == os.path.normpath(
+        os.path.join(mock_low_high_config.scopes['low'].path,
+                     'foo/bar/baz'))
+
 
 packages_merge_low = {
     'packages': {

lib/spack/spack/util/path.py

@@ -17,7 +17,7 @@
 from llnl.util.lang import memoized
 
 import spack.paths
-
+import spack.util.spack_yaml as syaml
 
 __all__ = [
     'substitute_config_variables',
@@ -72,12 +72,22 @@ def substitute_config_variables(path):
     - $spack     The Spack instance's prefix
     - $user      The current user's username
     - $tempdir   Default temporary directory returned by tempfile.gettempdir()
+    - $env       The active Spack environment.
 
     These are substituted case-insensitively into the path, and users can
-    use either ``$var`` or ``${var}`` syntax for the variables.
-
+    use either ``$var`` or ``${var}`` syntax for the variables. $env is only
+    replaced if there is an active environment, and should only be used in
+    environment yaml files.
     """
-    # Look up replacements for re.sub in the replacements dict.
+    import spack.environment as ev  # break circular
+    env = ev.get_env({}, '')
+    if env:
+        replacements.update({'env': env.path})
+    else:
+        # If a previous invocation added env, remove it
+        replacements.pop('env', None)
+
+    # Look up replacements
     def repl(match):
         m = match.group(0).strip('${}')
         return replacements.get(m.lower(), match.group(0))
@@ -132,7 +142,19 @@ def add_padding(path, length):
 
 def canonicalize_path(path):
     """Same as substitute_path_variables, but also take absolute path."""
-    path = substitute_path_variables(path)
-    path = os.path.abspath(path)
+    # Get file in which path was written in case we need to make it absolute
+    # relative to that path.
+    filename = None
+    if isinstance(path, syaml.syaml_str):
+        filename = os.path.dirname(path._start_mark.name)
+        assert path._start_mark.name == path._end_mark.name
 
-    return path
+    path = substitute_path_variables(path)
+    if not os.path.isabs(path):
+        if filename:
+            path = os.path.join(filename, path)
+        else:
+            path = os.path.abspath(path)
+            tty.debug("Using current working directory as base for abspath")
+
+    return os.path.normpath(path)