Git fetching: add option to remove submodules (#14370)

Add an optional 'submodules_delete' field to Git versions in Spack
packages that allows them to remove specific submodules.

For example: the nervanagpu submodule has become unavailable for the
PyTorch project (see issue 19457 at
https://github.com/pytorch/pytorch/issues/). Removing this submodule
allows 0.4.1 to build.

lib/spack/docs/packaging_guide.rst

@@ -929,6 +929,9 @@ Git fetching supports the following parameters to ``version``:
 * ``tag``: Name of a tag to fetch.
 * ``commit``: SHA hash (or prefix) of a commit to fetch.
 * ``submodules``: Also fetch submodules recursively when checking out this repository.
+* ``submodules_delete``: A list of submodules to forcibly delete from the repository
+  after fetching. Useful if a version in the repository has submodules that
+  have disappeared/are no longer accessible.
 * ``get_full_repo``: Ensure the full git history is checked out with all remote
   branch information. Normally (``get_full_repo=False``, the default), the git
   option ``--depth 1`` will be used if the version of git and the specified

lib/spack/spack/fetch_strategy.py

@@ -714,7 +714,8 @@ class GitFetchStrategy(VCSFetchStrategy):
     Repositories are cloned into the standard stage source path directory.
     """
     url_attr = 'git'
-    optional_attrs = ['tag', 'branch', 'commit', 'submodules', 'get_full_repo']
+    optional_attrs = ['tag', 'branch', 'commit', 'submodules',
+                      'get_full_repo', 'submodules_delete']
 
     def __init__(self, **kwargs):
         # Discards the keywords in kwargs that may conflict with the next call
@@ -725,6 +726,7 @@ def __init__(self, **kwargs):
 
         self._git = None
         self.submodules = kwargs.get('submodules', False)
+        self.submodules_delete = kwargs.get('submodules_delete', False)
         self.get_full_repo = kwargs.get('get_full_repo', False)
 
     @property
@@ -858,6 +860,14 @@ def fetch(self):
                     git(*pull_args, ignore_errors=1)
                     git(*co_args)
 
+        if self.submodules_delete:
+            with working_dir(self.stage.source_path):
+                for submodule_to_delete in self.submodules_delete:
+                    args = ['rm', submodule_to_delete]
+                    if not spack.config.get('config:debug'):
+                        args.insert(1, '--quiet')
+                    git(*args)
+
         # Init submodules if the user asked for them.
         if self.submodules:
             with working_dir(self.stage.source_path):

lib/spack/spack/test/conftest.py

@@ -744,11 +744,31 @@ def mock_archive(request, tmpdir_factory):
 
 @pytest.fixture(scope='session')
 def mock_git_repository(tmpdir_factory):
-    """Creates a very simple git repository with two branches and
+    """Creates a simple git repository with two branches,
-    two commits.
+    two commits and two submodules. Each submodule has one commit.
     """
     git = spack.util.executable.which('git', required=True)
 
+    suburls = []
+    for submodule_count in range(2):
+        tmpdir = tmpdir_factory.mktemp('mock-git-repo-submodule-dir-{0}'
+                                       .format(submodule_count))
+        tmpdir.ensure(spack.stage._source_path_subdir, dir=True)
+        repodir = tmpdir.join(spack.stage._source_path_subdir)
+        suburls.append((submodule_count, 'file://' + str(repodir)))
+
+        # Initialize the repository
+        with repodir.as_cwd():
+            git('init')
+            git('config', 'user.name', 'Spack')
+            git('config', 'user.email', 'spack@spack.io')
+
+            # r0 is just the first commit
+            submodule_file = 'r0_file_{0}'.format(submodule_count)
+            repodir.ensure(submodule_file)
+            git('add', submodule_file)
+            git('commit', '-m', 'mock-git-repo r0 {0}'.format(submodule_count))
+
     tmpdir = tmpdir_factory.mktemp('mock-git-repo-dir')
     tmpdir.ensure(spack.stage._source_path_subdir, dir=True)
     repodir = tmpdir.join(spack.stage._source_path_subdir)
@@ -759,6 +779,9 @@ def mock_git_repository(tmpdir_factory):
         git('config', 'user.name', 'Spack')
         git('config', 'user.email', 'spack@spack.io')
         url = 'file://' + str(repodir)
+        for number, suburl in suburls:
+            git('submodule', 'add', suburl,
+                'third_party/submodule{0}'.format(number))
 
         # r0 is just the first commit
         r0_file = 'r0_file'

lib/spack/spack/test/git_fetch.py

@@ -19,7 +19,6 @@
 from spack.fetch_strategy import GitFetchStrategy
 from spack.util.executable import which
 
-
 pytestmark = pytest.mark.skipif(
     not which('git'), reason='requires git to be installed')
 
@@ -217,3 +216,59 @@ def test_get_full_repo(get_full_repo, git_version, mock_git_repository,
         else:
             assert(nbranches == 2)
             assert(ncommits == 1)
+
+
+@pytest.mark.disable_clean_stage_check
+@pytest.mark.parametrize("submodules", [True, False])
+def test_gitsubmodule(submodules, mock_git_repository, config,
+                      mutable_mock_repo):
+    """
+    Test GitFetchStrategy behavior with submodules
+    """
+    type_of_test = 'tag-branch'
+    t = mock_git_repository.checks[type_of_test]
+
+    # Construct the package under test
+    spec = Spec('git-test')
+    spec.concretize()
+    pkg = spack.repo.get(spec)
+    args = copy.copy(t.args)
+    args['submodules'] = submodules
+    pkg.versions[ver('git')] = args
+    pkg.do_stage()
+    with working_dir(pkg.stage.source_path):
+        for submodule_count in range(2):
+            file_path = os.path.join(pkg.stage.source_path,
+                                     'third_party/submodule{0}/r0_file_{0}'
+                                     .format(submodule_count))
+            if submodules:
+                assert os.path.isfile(file_path)
+            else:
+                assert not os.path.isfile(file_path)
+
+
+@pytest.mark.disable_clean_stage_check
+def test_gitsubmodules_delete(mock_git_repository, config, mutable_mock_repo):
+    """
+    Test GitFetchStrategy behavior with submodules_delete
+    """
+    type_of_test = 'tag-branch'
+    t = mock_git_repository.checks[type_of_test]
+
+    # Construct the package under test
+    spec = Spec('git-test')
+    spec.concretize()
+    pkg = spack.repo.get(spec)
+    args = copy.copy(t.args)
+    args['submodules'] = True
+    args['submodules_delete'] = ['third_party/submodule0',
+                                 'third_party/submodule1']
+    pkg.versions[ver('git')] = args
+    pkg.do_stage()
+    with working_dir(pkg.stage.source_path):
+        file_path = os.path.join(pkg.stage.source_path,
+                                 'third_party/submodule0')
+        assert not os.path.isdir(file_path)
+        file_path = os.path.join(pkg.stage.source_path,
+                                 'third_party/submodule1')
+        assert not os.path.isdir(file_path)

var/spack/repos/builtin/packages/py-torch/package.py

@@ -58,7 +58,8 @@ class PyTorch(PythonPackage, CudaPackage):
     version('1.1.0', tag='v1.1.0', submodules=True)
     version('1.0.1', tag='v1.0.1', submodules=True)
     version('1.0.0', tag='v1.0.0', submodules=True)
-    version('0.4.1', tag='v0.4.1', submodules=True)
+    version('0.4.1', tag='v0.4.1', submodules=True,
+            submodules_delete=['third_party/nervanagpu'])
     version('0.4.0', tag='v0.4.0', submodules=True)
     version('0.3.1', tag='v0.3.1', submodules=True)