From 7626ec4579c15371d73ac8485adbb9675ca4b7c3 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Fri, 29 Jun 2018 18:10:36 -0400 Subject: [PATCH] refactor: move spack.util.multiproc to llnl.util.multiproc - multiproc doesn't depend on Spack - llnl.util.lock test uses it, but shouldn't use parts of Spack. --- lib/spack/{spack => llnl}/util/multiproc.py | 0 lib/spack/spack/architecture.py | 12 ++++++------ lib/spack/spack/compiler.py | 4 ++-- lib/spack/spack/operating_systems/cnl.py | 4 ++-- lib/spack/spack/test/llnl/util/lock.py | 4 ++-- 5 files changed, 12 insertions(+), 12 deletions(-) rename lib/spack/{spack => llnl}/util/multiproc.py (100%) diff --git a/lib/spack/spack/util/multiproc.py b/lib/spack/llnl/util/multiproc.py similarity index 100% rename from lib/spack/spack/util/multiproc.py rename to lib/spack/llnl/util/multiproc.py diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index f7b1946c46..4ef61d569f 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -79,14 +79,14 @@ import inspect import platform as py_platform -from llnl.util.lang import memoized, list_modules, key_ordering +import llnl.util.multiproc as mp import llnl.util.tty as tty +from llnl.util.lang import memoized, list_modules, key_ordering import spack.paths import spack.error as serr from spack.util.naming import mod_to_class from spack.util.environment import get_path -from spack.util.multiproc import parmap from spack.util.spack_yaml import syaml_dict @@ -280,9 +280,9 @@ def find_compilers(self, *paths): # NOTE: we import spack.compilers here to avoid init order cycles import spack.compilers types = spack.compilers.all_compiler_types() - compiler_lists = parmap(lambda cmp_cls: - self.find_compiler(cmp_cls, *filtered_path), - types) + compiler_lists = mp.parmap( + lambda cmp_cls: self.find_compiler(cmp_cls, *filtered_path), + types) # ensure all the version calls we made are cached in the parent # process, as well. This speeds up Spack a lot. @@ -300,7 +300,7 @@ def find_compiler(self, cmp_cls, *path): prefixes, suffixes, and versions. e.g., gcc-mp-4.7 would be grouped with g++-mp-4.7 and gfortran-mp-4.7. """ - dicts = parmap( + dicts = mp.parmap( lambda t: cmp_cls._find_matches_in_path(*t), [(cmp_cls.cc_names, cmp_cls.cc_version) + tuple(path), (cmp_cls.cxx_names, cmp_cls.cxx_version) + tuple(path), diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index b219eee09d..afb18ddcac 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -27,11 +27,11 @@ import itertools import llnl.util.tty as tty +import llnl.util.multiproc as mp import spack.error import spack.spec import spack.architecture -from spack.util.multiproc import parmap from spack.util.executable import Executable, ProcessError from spack.util.environment import get_path @@ -280,7 +280,7 @@ def _find_matches_in_path(cls, compiler_names, detect_version, *path): key = (full_path,) + match.groups() + (detect_version,) checks.append(key) - successful = [k for k in parmap(_get_versioned_tuple, checks) + successful = [k for k in mp.parmap(_get_versioned_tuple, checks) if k is not None] # The 'successful' list is ordered like the input paths. diff --git a/lib/spack/spack/operating_systems/cnl.py b/lib/spack/spack/operating_systems/cnl.py index d5e8f46077..9be7fe641a 100644 --- a/lib/spack/spack/operating_systems/cnl.py +++ b/lib/spack/spack/operating_systems/cnl.py @@ -25,9 +25,9 @@ import re import llnl.util.tty as tty +import llnl.util.multiproc as mp from spack.architecture import OperatingSystem -from spack.util.multiproc import parmap from spack.util.module_cmd import get_module_cmd @@ -60,7 +60,7 @@ def find_compilers(self, *paths): import spack.compilers types = spack.compilers.all_compiler_types() - compiler_lists = parmap( + compiler_lists = mp.parmap( lambda cmp_cls: self.find_compiler(cmp_cls, *paths), types) # ensure all the version calls we made are cached in the parent diff --git a/lib/spack/spack/test/llnl/util/lock.py b/lib/spack/spack/test/llnl/util/lock.py index 0060bc9273..9b32b9fd15 100644 --- a/lib/spack/spack/test/llnl/util/lock.py +++ b/lib/spack/spack/test/llnl/util/lock.py @@ -72,10 +72,10 @@ import pytest +import llnl.util.multiproc as mp from llnl.util.filesystem import touch, group_ids import spack.util.lock -from spack.util.multiproc import Barrier from spack.util.lock import Lock, WriteTransaction, ReadTransaction, LockError @@ -205,7 +205,7 @@ def lock_path(lock_dir): def local_multiproc_test(*functions): """Order some processes using simple barrier synchronization.""" - b = Barrier(len(functions), timeout=barrier_timeout) + b = mp.Barrier(len(functions), timeout=barrier_timeout) procs = [Process(target=f, args=(b,)) for f in functions] for p in procs: