Testing: ensure that all packages can be pickled (#19890)

As of #18205, all packages must be pickle-able to be installed by
Spack.

This adds a test to check that each package can be pickled. If any
package fails to pickle, the test keeps going and collects the names
of all failed packages; it then takes the first one that failed and
attempts to re-pickle it, generating the full stack trace for the
failed pickle attempt.

lib/spack/spack/test/package_sanity.py

@@ -18,6 +18,9 @@
 # do sanity checks only on packagess modified by a PR
 import spack.cmd.flake8 as flake8
 import spack.util.crypto as crypto
+import pickle
+
+import llnl.util.tty as tty
 
 
 def check_repo():
@@ -32,6 +35,27 @@ def test_get_all_packages():
     check_repo()
 
 
+def test_packages_are_pickleable():
+    failed_to_pickle = list()
+    for name in spack.repo.all_package_names():
+        pkg = spack.repo.get(name)
+        try:
+            pickle.dumps(pkg)
+        except Exception:
+            # If there are any failures, keep track of all packages that aren't
+            # pickle-able and re-run the pickling later on to recreate the
+            # error
+            failed_to_pickle.append(name)
+
+    if failed_to_pickle:
+        tty.msg('The following packages failed to pickle: ' +
+                ', '.join(failed_to_pickle))
+
+        for name in failed_to_pickle:
+            pkg = spack.repo.get(name)
+            pickle.dumps(pkg)
+
+
 def test_get_all_mock_packages():
     """Get the mock packages once each too."""
     db = spack.repo.RepoPath(spack.paths.mock_packages_path)