From bb63327da0483d163f4d7c479054848ae46cc431 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 7 Oct 2013 18:54:58 -0700 Subject: [PATCH] Refactor utils into separate modules. --- lib/spack/spack/arch.py | 6 +- lib/spack/spack/cmd/test.py | 2 +- lib/spack/spack/compilers/__init__.py | 3 +- lib/spack/spack/globals.py | 10 +- lib/spack/spack/package.py | 2 +- lib/spack/spack/packages/__init__.py | 2 +- lib/spack/spack/url.py | 6 +- lib/spack/spack/util/__init__.py | 209 -------------------------- lib/spack/spack/util/compression.py | 15 ++ lib/spack/spack/util/environment.py | 30 ++++ lib/spack/spack/util/executable.py | 55 +++++++ lib/spack/spack/util/filesystem.py | 71 +++++++++ lib/spack/spack/util/lang.py | 32 ++++ lib/spack/spack/validate.py | 3 +- 14 files changed, 221 insertions(+), 225 deletions(-) create mode 100644 lib/spack/spack/util/compression.py create mode 100644 lib/spack/spack/util/environment.py create mode 100644 lib/spack/spack/util/executable.py create mode 100644 lib/spack/spack/util/filesystem.py create mode 100644 lib/spack/spack/util/lang.py diff --git a/lib/spack/spack/arch.py b/lib/spack/spack/arch.py index 74d2f35c41..1c5d5a2c5f 100644 --- a/lib/spack/spack/arch.py +++ b/lib/spack/spack/arch.py @@ -2,9 +2,9 @@ import platform as py_platform import spack -import error as serr -from version import Version -from util import memoized +import spack.error as serr +from spack.version import Version +from spack.util.lang import memoized class InvalidSysTypeError(serr.SpackError): diff --git a/lib/spack/spack/cmd/test.py b/lib/spack/spack/cmd/test.py index a246cdf7b6..c6ac064001 100644 --- a/lib/spack/spack/cmd/test.py +++ b/lib/spack/spack/cmd/test.py @@ -1,7 +1,7 @@ import spack import spack.packages as packages import spack.test -from spack.util import list_modules +from spack.util.lang import list_modules from spack.colify import colify from pprint import pprint diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 3ed2db8423..24a99a1bae 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -4,8 +4,7 @@ import spack import spack.compilers.gcc -from spack.util import list_modules, memoized - +from spack.util.lang import memoized, list_modules @memoized def supported_compilers(): diff --git a/lib/spack/spack/globals.py b/lib/spack/spack/globals.py index 3dc7310a9d..a9fa25c784 100644 --- a/lib/spack/spack/globals.py +++ b/lib/spack/spack/globals.py @@ -1,8 +1,10 @@ import os -from version import Version -from util import * -import arch -from directory_layout import DefaultDirectoryLayout + +import spack.arch as arch +from spack.version import Version +from spack.util.filesystem import * +from spack.util.executable import * +from spack.directory_layout import DefaultDirectoryLayout # This lives in $prefix/lib/spac/spack/__file__ prefix = ancestor(__file__, 4) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 0611d005db..7e11c92d2d 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -24,11 +24,11 @@ import validate import url - from spec import Compiler from version import * from multi_function import platform from stage import Stage +from spack.util.lang import memoized, list_modules class Package(object): diff --git a/lib/spack/spack/packages/__init__.py b/lib/spack/spack/packages/__init__.py index 01788984ea..b9987aa040 100644 --- a/lib/spack/spack/packages/__init__.py +++ b/lib/spack/spack/packages/__init__.py @@ -8,7 +8,7 @@ import spack import spack.error import spack.spec -from spack.util import * +from spack.util.filesystem import new_path import spack.arch as arch # Valid package names can contain '-' but can't start with it. diff --git a/lib/spack/spack/url.py b/lib/spack/spack/url.py index 1d1fccdfe7..59d05203b6 100644 --- a/lib/spack/spack/url.py +++ b/lib/spack/spack/url.py @@ -24,7 +24,7 @@ import re import spack.error -import spack.util +import spack.util.filesystem as fs from spack.version import Version # @@ -61,9 +61,9 @@ def parse_version_string_with_indices(path): if os.path.isdir(path): stem = os.path.basename(path) elif re.search(r'((?:sourceforge.net|sf.net)/.*)/download$', path): - stem = spack.util.stem(os.path.dirname(path)) + stem = fs.stem(os.path.dirname(path)) else: - stem = spack.util.stem(path) + stem = fs.stem(path) version_types = [ # GitHub tarballs, e.g. v1.2.3 diff --git a/lib/spack/spack/util/__init__.py b/lib/spack/spack/util/__init__.py index 2781262b5a..e69de29bb2 100644 --- a/lib/spack/spack/util/__init__.py +++ b/lib/spack/spack/util/__init__.py @@ -1,209 +0,0 @@ -import os -import re -import errno -import shutil -import subprocess -import multiprocessing -from itertools import product -import functools -from contextlib import closing, contextmanager - -import tty - -# Supported archvie extensions. -PRE_EXTS = ["tar"] -EXTS = ["gz", "bz2", "xz", "Z", "zip", "tgz"] - -# Add EXTS last so that .tar.gz is matched *before* tar.gz -ALLOWED_ARCHIVE_TYPES = [".".join(l) for l in product(PRE_EXTS, EXTS)] + EXTS - - -def memoized(obj): - """Decorator that caches the results of a function, storing them - in an attribute of that function.""" - cache = obj.cache = {} - @functools.wraps(obj) - def memoizer(*args, **kwargs): - if args not in cache: - cache[args] = obj(*args, **kwargs) - return cache[args] - return memoizer - - -def install(src, dest): - tty.info("Installing %s to %s" % (src, dest)) - shutil.copy(src, dest) - - -def list_modules(directory): - """Lists all of the modules, excluding __init__.py, in - a particular directory.""" - for name in os.listdir(directory): - if name == '__init__.py': - continue - - path = new_path(directory, name) - if os.path.isdir(path): - init_py = new_path(path, '__init__.py') - if os.path.isfile(init_py): - yield name - - elif name.endswith('.py'): - yield re.sub('.py$', '', name) - - -@contextmanager -def working_dir(dirname): - orig_dir = os.getcwd() - os.chdir(dirname) - yield - os.chdir(orig_dir) - - -def mkdirp(*paths): - for path in paths: - if not os.path.exists(path): - os.makedirs(path) - elif not os.path.isdir(path): - raise OSError(errno.EEXIST, "File alredy exists", path) - - -def env_flag(name): - if name in os.environ: - return os.environ[name].lower() == "true" - return False - - -def path_set(var_name, directories): - path_str = ":".join(str(dir) for dir in directories) - os.environ[var_name] = path_str - - -def path_put_first(var_name, directories): - """Puts the provided directories first in the path, adding them - if they're not already there. - """ - path = os.environ.get(var_name, "").split(':') - - for dir in directories: - if dir in path: - path.remove(dir) - - new_path = tuple(directories) + tuple(path) - path_set(var_name, new_path) - - -def pop_keys(dictionary, *keys): - for key in keys: - if key in dictionary: - dictionary.pop(key) - - -def remove_items(item_list, *items): - for item in items: - if item in item_list: - item_list.remove(item) - - -def has_whitespace(string): - return re.search(r'\s', string) - - -def new_path(prefix, *args): - path=str(prefix) - for elt in args: - path = os.path.join(path, str(elt)) - - if has_whitespace(path): - tty.die("Invalid path: '%s'. Use a path without whitespace." % path) - - return path - - -def ancestor(dir, n=1): - """Get the nth ancestor of a directory.""" - parent = os.path.abspath(dir) - for i in range(n): - parent = os.path.dirname(parent) - return parent - - -class Executable(object): - """Class representing a program that can be run on the command line.""" - def __init__(self, name): - self.exe = name.split(' ') - - def add_default_arg(self, arg): - self.exe.append(arg) - - def __call__(self, *args, **kwargs): - """Run the executable with subprocess.check_output, return output.""" - return_output = kwargs.get("return_output", False) - fail_on_error = kwargs.get("fail_on_error", True) - - quoted_args = [arg for arg in args if re.search(r'^"|^\'|"$|\'$', arg)] - if quoted_args: - tty.warn("Quotes in package command arguments can confuse shell scripts like configure.", - "The following arguments may cause problems when executed:", - str("\n".join([" "+arg for arg in quoted_args])), - "Quotes aren't needed because spack doesn't use a shell. Consider removing them") - - cmd = self.exe + list(args) - tty.verbose(" ".join(cmd)) - - if return_output: - return subprocess.check_output(cmd) - elif fail_on_error: - return subprocess.check_call(cmd) - else: - return subprocess.call(cmd) - - def __repr__(self): - return "" % self.exe - - -def which(name, **kwargs): - """Finds an executable in the path like command-line which.""" - path = kwargs.get('path', os.environ.get('PATH', '').split(os.pathsep)) - required = kwargs.get('required', False) - - if not path: - path = [] - - for dir in path: - exe = os.path.join(dir, name) - if os.access(exe, os.X_OK): - return Executable(exe) - - if required: - tty.die("spack requires %s. Make sure it is in your path." % name) - return None - - -def stem(path): - """Get the part of a path that does not include its compressed - type extension.""" - for type in ALLOWED_ARCHIVE_TYPES: - suffix = r'\.%s$' % type - if re.search(suffix, path): - return re.sub(suffix, "", path) - return path - - -def decompressor_for(path): - """Get the appropriate decompressor for a path.""" - tar = which('tar', required=True) - tar.add_default_arg('-xf') - return tar - - -def md5(filename, block_size=2**20): - import hashlib - md5 = hashlib.md5() - with closing(open(filename)) as file: - while True: - data = file.read(block_size) - if not data: - break - md5.update(data) - return md5.hexdigest() diff --git a/lib/spack/spack/util/compression.py b/lib/spack/spack/util/compression.py new file mode 100644 index 0000000000..b0dc0241e3 --- /dev/null +++ b/lib/spack/spack/util/compression.py @@ -0,0 +1,15 @@ +from itertools import product + +# Supported archvie extensions. +PRE_EXTS = ["tar"] +EXTS = ["gz", "bz2", "xz", "Z", "zip", "tgz"] + +# Add EXTS last so that .tar.gz is matched *before* tar.gz +ALLOWED_ARCHIVE_TYPES = [".".join(l) for l in product(PRE_EXTS, EXTS)] + EXTS + + +def decompressor_for(path): + """Get the appropriate decompressor for a path.""" + tar = which('tar', required=True) + tar.add_default_arg('-xf') + return tar diff --git a/lib/spack/spack/util/environment.py b/lib/spack/spack/util/environment.py new file mode 100644 index 0000000000..4673b55ed3 --- /dev/null +++ b/lib/spack/spack/util/environment.py @@ -0,0 +1,30 @@ + +def env_flag(name): + if name in os.environ: + return os.environ[name].lower() == "true" + return False + + +def path_set(var_name, directories): + path_str = ":".join(str(dir) for dir in directories) + os.environ[var_name] = path_str + + +def path_put_first(var_name, directories): + """Puts the provided directories first in the path, adding them + if they're not already there. + """ + path = os.environ.get(var_name, "").split(':') + + for dir in directories: + if dir in path: + path.remove(dir) + + new_path = tuple(directories) + tuple(path) + path_set(var_name, new_path) + + +def pop_keys(dictionary, *keys): + for key in keys: + if key in dictionary: + dictionary.pop(key) diff --git a/lib/spack/spack/util/executable.py b/lib/spack/spack/util/executable.py new file mode 100644 index 0000000000..99b52ea299 --- /dev/null +++ b/lib/spack/spack/util/executable.py @@ -0,0 +1,55 @@ +import os +import subprocess +import spack.tty as tty + + +class Executable(object): + """Class representing a program that can be run on the command line.""" + def __init__(self, name): + self.exe = name.split(' ') + + def add_default_arg(self, arg): + self.exe.append(arg) + + def __call__(self, *args, **kwargs): + """Run the executable with subprocess.check_output, return output.""" + return_output = kwargs.get("return_output", False) + fail_on_error = kwargs.get("fail_on_error", True) + + quoted_args = [arg for arg in args if re.search(r'^"|^\'|"$|\'$', arg)] + if quoted_args: + tty.warn("Quotes in package command arguments can confuse shell scripts like configure.", + "The following arguments may cause problems when executed:", + str("\n".join([" "+arg for arg in quoted_args])), + "Quotes aren't needed because spack doesn't use a shell. Consider removing them") + + cmd = self.exe + list(args) + tty.verbose(" ".join(cmd)) + + if return_output: + return subprocess.check_output(cmd) + elif fail_on_error: + return subprocess.check_call(cmd) + else: + return subprocess.call(cmd) + + def __repr__(self): + return "" % self.exe + + +def which(name, **kwargs): + """Finds an executable in the path like command-line which.""" + path = kwargs.get('path', os.environ.get('PATH', '').split(os.pathsep)) + required = kwargs.get('required', False) + + if not path: + path = [] + + for dir in path: + exe = os.path.join(dir, name) + if os.access(exe, os.X_OK): + return Executable(exe) + + if required: + tty.die("spack requires %s. Make sure it is in your path." % name) + return None diff --git a/lib/spack/spack/util/filesystem.py b/lib/spack/spack/util/filesystem.py new file mode 100644 index 0000000000..e051dc2b6f --- /dev/null +++ b/lib/spack/spack/util/filesystem.py @@ -0,0 +1,71 @@ +import os +import re +import shutil +import errno +from contextlib import contextmanager, closing + +import spack.tty as tty +from spack.util.compression import ALLOWED_ARCHIVE_TYPES + +def install(src, dest): + """Manually install a file to a particular location.""" + tty.info("Installing %s to %s" % (src, dest)) + shutil.copy(src, dest) + + +@contextmanager +def working_dir(dirname): + orig_dir = os.getcwd() + os.chdir(dirname) + yield + os.chdir(orig_dir) + + +def mkdirp(*paths): + for path in paths: + if not os.path.exists(path): + os.makedirs(path) + elif not os.path.isdir(path): + raise OSError(errno.EEXIST, "File alredy exists", path) + + +def new_path(prefix, *args): + path=str(prefix) + for elt in args: + path = os.path.join(path, str(elt)) + + if re.search(r'\s', path): + tty.die("Invalid path: '%s'. Use a path without whitespace." % path) + + return path + + +def ancestor(dir, n=1): + """Get the nth ancestor of a directory.""" + parent = os.path.abspath(dir) + for i in range(n): + parent = os.path.dirname(parent) + return parent + + +def stem(path): + """Get the part of a path that does not include its compressed + type extension.""" + for type in ALLOWED_ARCHIVE_TYPES: + suffix = r'\.%s$' % type + if re.search(suffix, path): + return re.sub(suffix, "", path) + return path + + +def md5(filename, block_size=2**20): + """Computes the md5 hash of a file.""" + import hashlib + md5 = hashlib.md5() + with closing(open(filename)) as file: + while True: + data = file.read(block_size) + if not data: + break + md5.update(data) + return md5.hexdigest() diff --git a/lib/spack/spack/util/lang.py b/lib/spack/spack/util/lang.py new file mode 100644 index 0000000000..0d9b7e32bb --- /dev/null +++ b/lib/spack/spack/util/lang.py @@ -0,0 +1,32 @@ +import os +import re +import functools +from spack.util.filesystem import new_path + +def memoized(obj): + """Decorator that caches the results of a function, storing them + in an attribute of that function.""" + cache = obj.cache = {} + @functools.wraps(obj) + def memoizer(*args, **kwargs): + if args not in cache: + cache[args] = obj(*args, **kwargs) + return cache[args] + return memoizer + + +def list_modules(directory): + """Lists all of the modules, excluding __init__.py, in + a particular directory.""" + for name in os.listdir(directory): + if name == '__init__.py': + continue + + path = new_path(directory, name) + if os.path.isdir(path): + init_py = new_path(path, '__init__.py') + if os.path.isfile(init_py): + yield name + + elif name.endswith('.py'): + yield re.sub('.py$', '', name) diff --git a/lib/spack/spack/validate.py b/lib/spack/spack/validate.py index c64fea7574..ceeca052b5 100644 --- a/lib/spack/spack/validate.py +++ b/lib/spack/spack/validate.py @@ -1,7 +1,8 @@ import tty -from util import ALLOWED_ARCHIVE_TYPES from urlparse import urlparse +from spack.util.compression import ALLOWED_ARCHIVE_TYPES + ALLOWED_SCHEMES = ["http", "https", "ftp"] def url(url_string):