Refactor utils into separate modules.

lib/spack/spack/arch.py

@@ -2,9 +2,9 @@
 import platform as py_platform
 
 import spack
-import error as serr
-from version import Version
-from util import memoized
+import spack.error as serr
+from spack.version import Version
+from spack.util.lang import memoized
 
 
 class InvalidSysTypeError(serr.SpackError):

lib/spack/spack/cmd/test.py

@@ -1,7 +1,7 @@
 import spack
 import spack.packages as packages
 import spack.test
-from spack.util import list_modules
+from spack.util.lang import list_modules
 from spack.colify import colify
 from pprint import pprint
 

lib/spack/spack/compilers/__init__.py

@@ -4,8 +4,7 @@
 
 import spack
 import spack.compilers.gcc
-from spack.util import list_modules, memoized
-
+from spack.util.lang import memoized, list_modules
 
 @memoized
 def supported_compilers():

lib/spack/spack/globals.py

@@ -1,8 +1,10 @@
 import os
-from version import Version
-from util import *
-import arch
-from directory_layout import DefaultDirectoryLayout
+
+import spack.arch as arch
+from spack.version import Version
+from spack.util.filesystem import *
+from spack.util.executable import *
+from spack.directory_layout import DefaultDirectoryLayout
 
 # This lives in $prefix/lib/spac/spack/__file__
 prefix = ancestor(__file__, 4)

lib/spack/spack/package.py

@@ -24,11 +24,11 @@
 import validate
 import url
 
-
 from spec import Compiler
 from version import *
 from multi_function import platform
 from stage import Stage
+from spack.util.lang import memoized, list_modules
 
 
 class Package(object):

lib/spack/spack/packages/__init__.py

@@ -8,7 +8,7 @@
 import spack
 import spack.error
 import spack.spec
-from spack.util import *
+from spack.util.filesystem import new_path
 import spack.arch as arch
 
 # Valid package names can contain '-' but can't start with it.

lib/spack/spack/url.py

@@ -24,7 +24,7 @@
 import re
 
 import spack.error
-import spack.util
+import spack.util.filesystem as fs
 from spack.version import Version
 
 #
@@ -61,9 +61,9 @@ def parse_version_string_with_indices(path):
     if os.path.isdir(path):
         stem = os.path.basename(path)
     elif re.search(r'((?:sourceforge.net|sf.net)/.*)/download$', path):
-        stem = spack.util.stem(os.path.dirname(path))
+        stem = fs.stem(os.path.dirname(path))
     else:
-        stem = spack.util.stem(path)
+        stem = fs.stem(path)
 
     version_types = [
         # GitHub tarballs, e.g. v1.2.3

lib/spack/spack/util/__init__.py

@@ -1,209 +0,0 @@
-import os
-import re
-import errno
-import shutil
-import subprocess
-import multiprocessing
-from itertools import product
-import functools
-from contextlib import closing, contextmanager
-
-import tty
-
-# Supported archvie extensions.
-PRE_EXTS = ["tar"]
-EXTS     = ["gz", "bz2", "xz", "Z", "zip", "tgz"]
-
-# Add EXTS last so that .tar.gz is matched *before* tar.gz
-ALLOWED_ARCHIVE_TYPES = [".".join(l) for l in product(PRE_EXTS, EXTS)] + EXTS
-
-
-def memoized(obj):
-    """Decorator that caches the results of a function, storing them
-       in an attribute of that function."""
-    cache = obj.cache = {}
-    @functools.wraps(obj)
-    def memoizer(*args, **kwargs):
-        if args not in cache:
-            cache[args] = obj(*args, **kwargs)
-        return cache[args]
-    return memoizer
-
-
-def install(src, dest):
-    tty.info("Installing %s to %s" % (src, dest))
-    shutil.copy(src, dest)
-
-
-def list_modules(directory):
-    """Lists all of the modules, excluding __init__.py, in
-       a particular directory."""
-    for name in os.listdir(directory):
-        if name == '__init__.py':
-            continue
-
-        path = new_path(directory, name)
-        if os.path.isdir(path):
-            init_py = new_path(path, '__init__.py')
-            if os.path.isfile(init_py):
-                yield name
-
-        elif name.endswith('.py'):
-            yield re.sub('.py$', '', name)
-
-
-@contextmanager
-def working_dir(dirname):
-    orig_dir = os.getcwd()
-    os.chdir(dirname)
-    yield
-    os.chdir(orig_dir)
-
-
-def mkdirp(*paths):
-    for path in paths:
-        if not os.path.exists(path):
-            os.makedirs(path)
-        elif not os.path.isdir(path):
-            raise OSError(errno.EEXIST, "File alredy exists", path)
-
-
-def env_flag(name):
-    if name in os.environ:
-        return os.environ[name].lower() == "true"
-    return False
-
-
-def path_set(var_name, directories):
-    path_str = ":".join(str(dir) for dir in directories)
-    os.environ[var_name] = path_str
-
-
-def path_put_first(var_name, directories):
-    """Puts the provided directories first in the path, adding them
-       if they're not already there.
-    """
-    path = os.environ.get(var_name, "").split(':')
-
-    for dir in directories:
-        if dir in path:
-            path.remove(dir)
-
-    new_path = tuple(directories) + tuple(path)
-    path_set(var_name, new_path)
-
-
-def pop_keys(dictionary, *keys):
-    for key in keys:
-        if key in dictionary:
-            dictionary.pop(key)
-
-
-def remove_items(item_list, *items):
-    for item in items:
-        if item in item_list:
-            item_list.remove(item)
-
-
-def has_whitespace(string):
-    return re.search(r'\s', string)
-
-
-def new_path(prefix, *args):
-    path=str(prefix)
-    for elt in args:
-        path = os.path.join(path, str(elt))
-
-    if has_whitespace(path):
-        tty.die("Invalid path: '%s'.  Use a path without whitespace." % path)
-
-    return path
-
-
-def ancestor(dir, n=1):
-    """Get the nth ancestor of a directory."""
-    parent = os.path.abspath(dir)
-    for i in range(n):
-        parent = os.path.dirname(parent)
-    return parent
-
-
-class Executable(object):
-    """Class representing a program that can be run on the command line."""
-    def __init__(self, name):
-        self.exe = name.split(' ')
-
-    def add_default_arg(self, arg):
-        self.exe.append(arg)
-
-    def __call__(self, *args, **kwargs):
-        """Run the executable with subprocess.check_output, return output."""
-        return_output = kwargs.get("return_output", False)
-        fail_on_error = kwargs.get("fail_on_error", True)
-
-        quoted_args = [arg for arg in args if re.search(r'^"|^\'|"$|\'$', arg)]
-        if quoted_args:
-            tty.warn("Quotes in package command arguments can confuse shell scripts like configure.",
-                     "The following arguments may cause problems when executed:",
-                     str("\n".join(["    "+arg for arg in quoted_args])),
-                     "Quotes aren't needed because spack doesn't use a shell.  Consider removing them")
-
-        cmd = self.exe + list(args)
-        tty.verbose(" ".join(cmd))
-
-        if return_output:
-            return subprocess.check_output(cmd)
-        elif fail_on_error:
-            return subprocess.check_call(cmd)
-        else:
-            return subprocess.call(cmd)
-
-    def __repr__(self):
-        return "<exe: %s>" % self.exe
-
-
-def which(name, **kwargs):
-    """Finds an executable in the path like command-line which."""
-    path     = kwargs.get('path', os.environ.get('PATH', '').split(os.pathsep))
-    required = kwargs.get('required', False)
-
-    if not path:
-        path = []
-
-    for dir in path:
-        exe = os.path.join(dir, name)
-        if os.access(exe, os.X_OK):
-            return Executable(exe)
-
-    if required:
-        tty.die("spack requires %s.  Make sure it is in your path." % name)
-    return None
-
-
-def stem(path):
-    """Get the part of a path that does not include its compressed
-       type extension."""
-    for type in ALLOWED_ARCHIVE_TYPES:
-        suffix = r'\.%s$' % type
-        if re.search(suffix, path):
-            return re.sub(suffix, "", path)
-    return path
-
-
-def decompressor_for(path):
-    """Get the appropriate decompressor for a path."""
-    tar = which('tar', required=True)
-    tar.add_default_arg('-xf')
-    return tar
-
-
-def md5(filename, block_size=2**20):
-    import hashlib
-    md5 = hashlib.md5()
-    with closing(open(filename)) as file:
-        while True:
-            data = file.read(block_size)
-            if not data:
-                break
-            md5.update(data)
-        return md5.hexdigest()

lib/spack/spack/util/compression.py

@@ -0,0 +1,15 @@
+from itertools import product
+
+# Supported archvie extensions.
+PRE_EXTS = ["tar"]
+EXTS     = ["gz", "bz2", "xz", "Z", "zip", "tgz"]
+
+# Add EXTS last so that .tar.gz is matched *before* tar.gz
+ALLOWED_ARCHIVE_TYPES = [".".join(l) for l in product(PRE_EXTS, EXTS)] + EXTS
+
+
+def decompressor_for(path):
+    """Get the appropriate decompressor for a path."""
+    tar = which('tar', required=True)
+    tar.add_default_arg('-xf')
+    return tar

lib/spack/spack/util/environment.py

@@ -0,0 +1,30 @@
+
+def env_flag(name):
+    if name in os.environ:
+        return os.environ[name].lower() == "true"
+    return False
+
+
+def path_set(var_name, directories):
+    path_str = ":".join(str(dir) for dir in directories)
+    os.environ[var_name] = path_str
+
+
+def path_put_first(var_name, directories):
+    """Puts the provided directories first in the path, adding them
+       if they're not already there.
+    """
+    path = os.environ.get(var_name, "").split(':')
+
+    for dir in directories:
+        if dir in path:
+            path.remove(dir)
+
+    new_path = tuple(directories) + tuple(path)
+    path_set(var_name, new_path)
+
+
+def pop_keys(dictionary, *keys):
+    for key in keys:
+        if key in dictionary:
+            dictionary.pop(key)

lib/spack/spack/util/executable.py

@@ -0,0 +1,55 @@
+import os
+import subprocess
+import spack.tty as tty
+
+
+class Executable(object):
+    """Class representing a program that can be run on the command line."""
+    def __init__(self, name):
+        self.exe = name.split(' ')
+
+    def add_default_arg(self, arg):
+        self.exe.append(arg)
+
+    def __call__(self, *args, **kwargs):
+        """Run the executable with subprocess.check_output, return output."""
+        return_output = kwargs.get("return_output", False)
+        fail_on_error = kwargs.get("fail_on_error", True)
+
+        quoted_args = [arg for arg in args if re.search(r'^"|^\'|"$|\'$', arg)]
+        if quoted_args:
+            tty.warn("Quotes in package command arguments can confuse shell scripts like configure.",
+                     "The following arguments may cause problems when executed:",
+                     str("\n".join(["    "+arg for arg in quoted_args])),
+                     "Quotes aren't needed because spack doesn't use a shell.  Consider removing them")
+
+        cmd = self.exe + list(args)
+        tty.verbose(" ".join(cmd))
+
+        if return_output:
+            return subprocess.check_output(cmd)
+        elif fail_on_error:
+            return subprocess.check_call(cmd)
+        else:
+            return subprocess.call(cmd)
+
+    def __repr__(self):
+        return "<exe: %s>" % self.exe
+
+
+def which(name, **kwargs):
+    """Finds an executable in the path like command-line which."""
+    path     = kwargs.get('path', os.environ.get('PATH', '').split(os.pathsep))
+    required = kwargs.get('required', False)
+
+    if not path:
+        path = []
+
+    for dir in path:
+        exe = os.path.join(dir, name)
+        if os.access(exe, os.X_OK):
+            return Executable(exe)
+
+    if required:
+        tty.die("spack requires %s.  Make sure it is in your path." % name)
+    return None

lib/spack/spack/util/filesystem.py

@@ -0,0 +1,71 @@
+import os
+import re
+import shutil
+import errno
+from contextlib import contextmanager, closing
+
+import spack.tty as tty
+from spack.util.compression import ALLOWED_ARCHIVE_TYPES
+
+def install(src, dest):
+    """Manually install a file to a particular location."""
+    tty.info("Installing %s to %s" % (src, dest))
+    shutil.copy(src, dest)
+
+
+@contextmanager
+def working_dir(dirname):
+    orig_dir = os.getcwd()
+    os.chdir(dirname)
+    yield
+    os.chdir(orig_dir)
+
+
+def mkdirp(*paths):
+    for path in paths:
+        if not os.path.exists(path):
+            os.makedirs(path)
+        elif not os.path.isdir(path):
+            raise OSError(errno.EEXIST, "File alredy exists", path)
+
+
+def new_path(prefix, *args):
+    path=str(prefix)
+    for elt in args:
+        path = os.path.join(path, str(elt))
+
+    if re.search(r'\s', path):
+        tty.die("Invalid path: '%s'.  Use a path without whitespace." % path)
+
+    return path
+
+
+def ancestor(dir, n=1):
+    """Get the nth ancestor of a directory."""
+    parent = os.path.abspath(dir)
+    for i in range(n):
+        parent = os.path.dirname(parent)
+    return parent
+
+
+def stem(path):
+    """Get the part of a path that does not include its compressed
+       type extension."""
+    for type in ALLOWED_ARCHIVE_TYPES:
+        suffix = r'\.%s$' % type
+        if re.search(suffix, path):
+            return re.sub(suffix, "", path)
+    return path
+
+
+def md5(filename, block_size=2**20):
+    """Computes the md5 hash of a file."""
+    import hashlib
+    md5 = hashlib.md5()
+    with closing(open(filename)) as file:
+        while True:
+            data = file.read(block_size)
+            if not data:
+                break
+            md5.update(data)
+        return md5.hexdigest()

lib/spack/spack/util/lang.py

@@ -0,0 +1,32 @@
+import os
+import re
+import functools
+from spack.util.filesystem import new_path
+
+def memoized(obj):
+    """Decorator that caches the results of a function, storing them
+       in an attribute of that function."""
+    cache = obj.cache = {}
+    @functools.wraps(obj)
+    def memoizer(*args, **kwargs):
+        if args not in cache:
+            cache[args] = obj(*args, **kwargs)
+        return cache[args]
+    return memoizer
+
+
+def list_modules(directory):
+    """Lists all of the modules, excluding __init__.py, in
+       a particular directory."""
+    for name in os.listdir(directory):
+        if name == '__init__.py':
+            continue
+
+        path = new_path(directory, name)
+        if os.path.isdir(path):
+            init_py = new_path(path, '__init__.py')
+            if os.path.isfile(init_py):
+                yield name
+
+        elif name.endswith('.py'):
+            yield re.sub('.py$', '', name)

lib/spack/spack/validate.py

@@ -1,7 +1,8 @@
 import tty
-from util import ALLOWED_ARCHIVE_TYPES
 from urlparse import urlparse
 
+from spack.util.compression import ALLOWED_ARCHIVE_TYPES
+
 ALLOWED_SCHEMES    = ["http", "https", "ftp"]
 
 def url(url_string):