OpenCV and OpenBLAS: add external find support (#30240)

Added support for finding the OpenCV package via the find external
command. Included support for identifying variants based on available
shared libraries.

Added support to finding the OpenBLAS package via the find external
command.

Enabled packages to show that they can be discovered via the find
external command in the info message.

Updated the OpenCV and OpenBLAS packages to use the extensible search
mechanism for library extensions on multiple OS platforms.

Corrected how find externals works on Darwin for OpenCV and OpenBLAS
to accommodate that the version numbers are placed before the file
extension instead of after it, as on Linux.

Co-authored-by: Adam J. Stewart <ajstewart426@gmail.com>
Co-authored-by: Massimiliano Culpo <massimiliano.culpo@gmail.com>

lib/spack/llnl/util/filesystem.py

@@ -64,6 +64,7 @@
     'is_exe',
     'join_path',
     'last_modification_time_recursive',
+    'library_extensions',
     'mkdirp',
     'partition_path',
     'prefixes',
@@ -109,12 +110,15 @@ def path_contains_subdirectory(path, root):
     return norm_path.startswith(norm_root)
 
 
+#: This generates the library filenames that may appear on any OS.
+library_extensions = ['a', 'la', 'so', 'tbd', 'dylib']
+
+
 def possible_library_filenames(library_names):
     """Given a collection of library names like 'libfoo', generate the set of
-    library filenames that may be found on the system (e.g. libfoo.so). This
+    library filenames that may be found on the system (e.g. libfoo.so).
-    generates the library filenames that may appear on any OS.
     """
-    lib_extensions = ['a', 'la', 'so', 'tbd', 'dylib']
+    lib_extensions = library_extensions
     return set(
         '.'.join((lib, extension)) for lib, extension in
         itertools.product(library_names, lib_extensions))

lib/spack/spack/cmd/info.py

@@ -184,8 +184,9 @@ def print_detectable(pkg):
     color.cprint('')
     color.cprint(section_title('Externally Detectable: '))
 
-    # If the package has an 'executables' field, it can detect an installation
+    # If the package has an 'executables' of 'libraries' field, it
-    if hasattr(pkg, 'executables'):
+    # can detect an installation
+    if hasattr(pkg, 'executables') or hasattr(pkg, 'libraries'):
         find_attributes = []
         if hasattr(pkg, 'determine_version'):
             find_attributes.append('version')

lib/spack/spack/detection/path.py

@@ -74,7 +74,8 @@ def executables_in_path(path_hints=None):
 
 
 def libraries_in_ld_library_path(path_hints=None):
-    """Get the paths of all libraries available from LD_LIBRARY_PATH.
+    """Get the paths of all libraries available from LD_LIBRARY_PATH,
+    LIBRARY_PATH, DYLD_LIBRARY_PATH, and DYLD_FALLBACK_LIBRARY_PATH.
 
     For convenience, this is constructed as a dictionary where the keys are
     the library paths and the values are the names of the libraries
@@ -85,9 +86,15 @@ def libraries_in_ld_library_path(path_hints=None):
 
     Args:
         path_hints (list): list of paths to be searched. If None the list will be
-            constructed based on the LD_LIBRARY_PATH environment variable.
+            constructed based on the set of LD_LIBRARY_PATH, LIBRARY_PATH,
+            DYLD_LIBRARY_PATH, and DYLD_FALLBACK_LIBRARY_PATH environment
+            variables.
     """
-    path_hints = path_hints or spack.util.environment.get_path('LD_LIBRARY_PATH')
+    path_hints = path_hints or \
+        spack.util.environment.get_path('LIBRARY_PATH') + \
+        spack.util.environment.get_path('LD_LIBRARY_PATH') + \
+        spack.util.environment.get_path('DYLD_LIBRARY_PATH') + \
+        spack.util.environment.get_path('DYLD_FALLBACK_LIBRARY_PATH')
     search_paths = llnl.util.filesystem.search_paths_for_libraries(*path_hints)
 
     path_to_lib = {}

var/spack/repos/builtin/packages/openblas/package.py

@@ -6,6 +6,8 @@
 import os
 import re
 
+from llnl.util.filesystem import library_extensions
+
 from spack import *
 from spack.package_test import compare_output_file, compile_c_and_execute
 
@@ -17,6 +19,8 @@ class Openblas(MakefilePackage):
     url      = 'https://github.com/xianyi/OpenBLAS/archive/v0.2.19.tar.gz'
     git      = 'https://github.com/xianyi/OpenBLAS.git'
 
+    libraries = ['libopenblas']
+
     version('develop', branch='develop')
     version('0.3.20', sha256='8495c9affc536253648e942908e88e097f2ec7753ede55aca52e5dead3029e3c')
     version('0.3.19', sha256='947f51bfe50c2a0749304fbe373e00e7637600b0a47b78a51382aeb30ca08562')
@@ -145,6 +149,16 @@ class Openblas(MakefilePackage):
 
     depends_on('perl', type='build')
 
+    @classmethod
+    def determine_version(cls, lib):
+        ver = None
+        for ext in library_extensions:
+            match = re.search(r'lib(\S*?)-r(\d+\.\d+\.\d+)\.%s' %
+                              ext, lib)
+            if match:
+                ver = match.group(2)
+        return ver
+
     @property
     def parallel(self):
         # unclear whether setting `-j N` externally was supported before 0.3

var/spack/repos/builtin/packages/opencv/package.py

@@ -3,6 +3,10 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
+import re
+
+from llnl.util.filesystem import library_extensions
+
 
 class Opencv(CMakePackage, CudaPackage):
     """OpenCV (Open Source Computer Vision Library) is an open source computer
@@ -233,10 +237,16 @@ class Opencv(CMakePackage, CudaPackage):
         "js_bindings_generator",
     ]
 
+    # Define the list of libraries objects that may be used
+    # to find an external installation and its variants
+    libraries = []
+
     # module variants
     for mod in modules:
         # At least one of these modules must be enabled to build OpenCV
         variant(mod, default=False, description="Include opencv_{0} module".format(mod))
+        lib = 'libopencv_' + mod
+        libraries.append(lib)
 
     # module conflicts and dependencies
     with when("+calib3d"):
@@ -873,6 +883,48 @@ class Opencv(CMakePackage, CudaPackage):
     conflicts("+win32ui", when="platform=linux", msg="Windows only")
     conflicts("+win32ui", when="platform=cray", msg="Windows only")
 
+    @classmethod
+    def determine_version(cls, lib):
+        ver = None
+        for ext in library_extensions:
+            pattern = None
+            if ext == 'dylib':
+                # Darwin switches the order of the version compared to Linux
+                pattern = re.compile(r'lib(\S*?)_(\S*)\.(\d+\.\d+\.\d+)\.%s' %
+                                     ext)
+            else:
+                pattern = re.compile(r'lib(\S*?)_(\S*)\.%s\.(\d+\.\d+\.\d+)' %
+                                     ext)
+            match = pattern.search(lib)
+            if match:
+                ver = match.group(3)
+        return ver
+
+    @classmethod
+    def determine_variants(cls, libs, version_str):
+        variants = []
+        remaining_modules = set(Opencv.modules)
+        for lib in libs:
+            for ext in library_extensions:
+                pattern = None
+                if ext == 'dylib':
+                    # Darwin switches the order of the version compared to Linux
+                    pattern = re.compile(r'lib(\S*?)_(\S*)\.(\d+\.\d+\.\d+)\.%s' %
+                                         ext)
+                else:
+                    pattern = re.compile(r'lib(\S*?)_(\S*)\.%s\.(\d+\.\d+\.\d+)' %
+                                         ext)
+                match = pattern.search(lib)
+                if match and not match.group(2) == 'core':
+                    variants.append('+' + match.group(2))
+                    remaining_modules.remove(match.group(2))
+
+        # If libraries are not found, mark those variants as disabled
+        for mod in remaining_modules:
+            variants.append('~' + mod)
+
+        return ' '.join(variants)
+
     def cmake_args(self):
         spec = self.spec
         args = [