This adds support for multi-platform methods.

You can now do this:
class MyPackage(Package):
    def install(self):
        ...default install...

    @platform('bgqos_0')
    def install(self):
        ...specialized install for bgq...

This works on functions other than install, as well (as long as they're in a Package)
This commit is contained in:
Todd Gamblin 2013-04-04 09:52:15 -07:00
parent e1551de976
commit 422d291b11
14 changed files with 334 additions and 82 deletions

View file

@ -15,6 +15,7 @@ sys.path.insert(0, SPACK_LIB_PATH)
# clean up the scope and start using spack package instead.
del SPACK_FILE, SPACK_PREFIX, SPACK_LIB_PATH
import spack
import spack.tty as tty
# Command parsing
parser = argparse.ArgumentParser(
@ -43,4 +44,7 @@ spack.debug = args.debug
# Try to load the particular command asked for and run it
command = spack.cmd.get_command(args.command)
command(parser, args)
try:
command(parser, args)
except KeyboardInterrupt:
tty.die("Got a keyboard interrupt from the user.")

View file

@ -1,5 +1,6 @@
from globals import *
from utils import *
from exception import *
from error import *
from package import Package, depends_on
from multi_function import platform

View file

@ -1,34 +1,67 @@
import os
import platform
import platform as py_platform
import spack
import error as serr
from version import Version
from utils import memoized
instances = {}
macos_versions = [
('10.8', 'mountain_lion'),
('10.7', 'lion'),
('10.6', 'snow_leopard'),
('10.5', 'leopard')]
class InvalidSysTypeError(serr.SpackError):
def __init__(self, sys_type):
super(InvalidSysTypeError, self).__init__(
"Invalid sys_type value for Spack: " + sys_type)
class SysType(object):
def __init__(self, arch_string):
self.arch_string = arch_string
class NoSysTypeError(serr.SpackError):
def __init__(self):
super(NoSysTypeError, self).__init__(
"Could not determine sys_type for this machine.")
def __repr__(self):
return self.arch_string
def __str__(self):
return self.__repr__()
def get_sys_type_from_spack_globals():
"""Return the SYS_TYPE from spack globals, or None if it isn't set."""
if not hasattr(spack, "sys_type"):
return None
elif hasattr(spack.sys_type, "__call__"):
return spack.sys_type()
else:
return spack.sys_type
def get_sys_type_from_environment():
"""Return $SYS_TYPE or None if it's not defined."""
return os.environ.get('SYS_TYPE')
def get_mac_sys_type():
"""Return a Mac OS SYS_TYPE or None if this isn't a mac."""
mac_ver = py_platform.mac_ver()[0]
if not mac_ver:
return None
return "macosx_{}_{}".format(
Version(mac_ver).up_to(2), py_platform.machine())
@memoized
def sys_type():
stype = os.environ.get('SYS_TYPE')
if stype:
return SysType(stype)
elif platform.mac_ver()[0]:
version = Version(platform.mac_ver()[0])
for mac_ver, name in macos_versions:
if version >= Version(mac_ver):
return SysType(name)
"""Returns a SysType for the current machine."""
methods = [get_sys_type_from_spack_globals,
get_sys_type_from_environment,
get_mac_sys_type]
# search for a method that doesn't return None
sys_type = None
for method in methods:
sys_type = method()
if sys_type: break
# Couldn't determine the sys_type for this machine.
if sys_type == None:
raise NoSysTypeError()
if not type(sys_type) == str:
raise InvalidSysTypeError(sys_type)
return sys_type

View file

@ -0,0 +1,11 @@
import spack
import spack.arch as arch
description = "Print the spack sys_type for this machine"
def sys_type(parser, args):
configured_sys_type = arch.get_sys_type_from_spack_globals()
if not configured_sys_type:
configured_sys_type = "autodetect"
print "Configured sys_type: %s" % configured_sys_type
print "Autodetected default sys_type: %s" % arch.sys_type()

13
lib/spack/spack/error.py Normal file
View file

@ -0,0 +1,13 @@
class SpackError(Exception):
"""This is the superclass for all Spack errors.
Subclasses can be found in the modules they have to do with.
"""
def __init__(self, message):
super(SpackError, self).__init__(message)
class UnsupportedPlatformError(SpackError):
"""Raised by packages when a platform is not supported"""
def __init__(self, message):
super(UnsupportedPlatformError, self).__init__(message)

View file

@ -1,39 +0,0 @@
class SpackException(Exception):
def __init__(self, message):
self.message = message
class FailedDownloadException(SpackException):
def __init__(self, url):
super(FailedDownloadException, self).__init__("Failed to fetch file from URL: " + url)
self.url = url
class InvalidPackageNameException(SpackException):
def __init__(self, name):
super(InvalidPackageNameException, self).__init__("Invalid package name: " + name)
self.name = name
class CommandFailedException(SpackException):
def __init__(self, command):
super(CommandFailedException, self).__init__("Failed to execute command: " + command)
self.command = command
class VersionParseException(SpackException):
def __init__(self, msg, spec):
super(VersionParseException, self).__init__(msg)
self.spec = spec
class UndetectableVersionException(VersionParseException):
def __init__(self, spec):
super(UndetectableVersionException, self).__init__("Couldn't detect version in: " + spec, spec)
class UndetectableNameException(VersionParseException):
def __init__(self, spec):
super(UndetectableNameException, self).__init__("Couldn't parse package name in: " + spec)

View file

@ -1,6 +1,7 @@
import os
from version import Version
from utils import *
import arch
# This lives in $prefix/lib/spac/spack/__file__
prefix = ancestor(__file__, 4)
@ -20,7 +21,7 @@
install_path = new_path(prefix, "opt")
# Version information
spack_version = Version("0.1")
spack_version = Version("0.2")
# User's editor from the environment
editor = Executable(os.environ.get("EDITOR", ""))
@ -39,6 +40,21 @@
# location per the python implementation of tempfile.mkdtemp().
tmp_dirs = ['/nfs/tmp2', '/var/tmp', '/tmp']
#
# SYS_TYPE to use for the spack installation.
# Value of this determines what platform spack thinks it is by
# default. You can assign three types of values:
# 1. None
# Spack will try to determine the sys_type automatically.
#
# 2. A string
# Spack will assume that the sys_type is hardcoded to the value.
#
# 3. A function that returns a string:
# Spack will use this function to determine the sys_type.
#
sys_type = None
# Important environment variables
SPACK_NO_PARALLEL_MAKE = 'SPACK_NO_PARALLEL_MAKE'
SPACK_LIB = 'SPACK_LIB'

View file

@ -0,0 +1,146 @@
"""This module contains utilities for using multi-functions in spack.
You can think of multi-functions like overloaded functions -- they're
functions with the same name, and we need to select a version of the
function based on some criteria. e.g., for overloaded functions, you
would select a version of the function to call based on the types of
its arguments.
For spack, we might want to select a version of the function based on
the platform we want to build a package for, or based on the versions
of the dependencies of the package.
"""
import sys
import functools
import arch
import spack.error as serr
class NoSuchVersionError(serr.SpackError):
"""Raised when we can't find a version of a function for a platform."""
def __init__(self, fun_name, sys_type):
super(NoSuchVersionError, self).__init__(
"No version of %s found for %s!" % (fun_name, sys_type))
class PlatformMultiFunction(object):
"""This is a callable type for storing a collection of versions
of an instance method. The platform decorator (see docs below)
creates PlatformMultiFunctions and registers function versions
with them.
To register a function, you can do something like this:
pmf = PlatformMultiFunction()
pmf.regsiter("chaos_5_x86_64_ib", some_function)
When the pmf is actually called, it selects a version of
the function to call based on the sys_type of the object
it is called on.
See the docs for the platform decorator for more details.
"""
def __init__(self, default=None):
self.function_map = {}
self.default = default
if default:
self.__name__ = default.__name__
def register(self, platform, function):
"""Register a version of a function for a particular sys_type."""
self.function_map[platform] = function
if not hasattr(self, '__name__'):
self.__name__ = function.__name__
else:
assert(self.__name__ == function.__name__)
def __get__(self, obj, objtype):
"""This makes __call__ support instance methods."""
return functools.partial(self.__call__, obj)
def __call__(self, package_self, *args, **kwargs):
"""Try to find a function that matches package_self.sys_type.
If none is found, call the default function that this was
initialized with. If there is no default, raise an error.
"""
sys_type = package_self.sys_type
function = self.function_map.get(sys_type, self.default)
if function:
function(package_self, *args, **kwargs)
else:
raise NoSuchVersionError(self.__name__, sys_type)
def __str__(self):
return "<%s, %s>" % (self.default, self.function_map)
class platform(object):
"""This annotation lets packages declare platform-specific versions
of functions like install(). For example:
class SomePackage(Package):
...
def install(self, prefix):
# Do default install
@platform('chaos_5_x86_64_ib')
def install(self, prefix):
# This will be executed instead of the default install if
# the package's sys_type() is chaos_5_x86_64_ib.
@platform('bgqos_0")
def install(self, prefix):
# This will be executed if the package's sys_type is bgqos_0
This allows each package to have a default version of install() AND
specialized versions for particular platforms. The version that is
called depends on the sys_type of SomePackage.
Note that this works for functions other than install, as well. So,
if you only have part of the install that is platform specific, you
could do this:
class SomePackage(Package):
...
def setup(self):
# do nothing in the default case
pass
@platform('chaos_5_x86_64_ib')
def setup(self):
# do something for x86_64
def install(self, prefix):
# Do common install stuff
self.setup()
# Do more common install stuff
If there is no specialized version for the package's sys_type, the
default (un-decorated) version will be called. If there is no default
version and no specialized version, the call raises a
NoSuchVersionError.
Note that the default version of install() must *always* come first.
Otherwise it will override all of the platform-specific versions.
There's not much we can do to get around this because of the way
decorators work.
"""
class platform(object):
def __init__(self, sys_type):
self.sys_type = sys_type
def __call__(self, fun):
# Record the sys_type as an attribute on this function
fun.sys_type = self.sys_type
# Get the first definition of the function in the calling scope
calling_frame = sys._getframe(1).f_locals
original_fun = calling_frame.get(fun.__name__)
# Create a multifunction out of the original function if it
# isn't one already.
if not type(original_fun) == PlatformMultiFunction:
original_fun = PlatformMultiFunction(original_fun)
original_fun.register(self.sys_type, fun)
return original_fun

View file

@ -14,7 +14,7 @@
import os
import re
import subprocess
import platform
import platform as py_platform
import shutil
from spack import *
@ -24,6 +24,8 @@
import validate
import version
import arch
from multi_function import platform
from stage import Stage
@ -226,7 +228,7 @@ class SomePackage(Package):
clean() (some of them do this), and others to provide custom behavior.
"""
def __init__(self, arch=arch.sys_type()):
def __init__(self, sys_type=arch.sys_type()):
attr.required(self, 'homepage')
attr.required(self, 'url')
attr.required(self, 'md5')
@ -235,7 +237,7 @@ def __init__(self, arch=arch.sys_type()):
attr.setdefault(self, 'parallel', True)
# Architecture for this package.
self.arch = arch
self.sys_type = sys_type
# Name of package is the name of its module (the file that contains it)
self.name = inspect.getmodulename(self.module.__file__)
@ -266,7 +268,7 @@ def __init__(self, arch=arch.sys_type()):
self.dirty = False
# stage used to build this package.
Self.stage = Stage(self.stage_name, self.url)
self.stage = Stage(self.stage_name, self.url)
def add_commands_to_module(self):
@ -289,7 +291,7 @@ def add_commands_to_module(self):
# standard CMake arguments
m.std_cmake_args = ['-DCMAKE_INSTALL_PREFIX=%s' % self.prefix,
'-DCMAKE_BUILD_TYPE=None']
if platform.mac_ver()[0]:
if py_platform.mac_ver()[0]:
m.std_cmake_args.append('-DCMAKE_FIND_FRAMEWORK=LAST')
# Emulate some shell commands for convenience
@ -361,11 +363,13 @@ def all_dependents(self):
def stage_name(self):
return "%s-%s" % (self.name, self.version)
#
# Below properties determine the path where this package is installed.
#
@property
def platform_path(self):
"""Directory for binaries for the current platform."""
return new_path(install_path, self.arch)
return new_path(install_path, self.sys_type)
@property
@ -388,6 +392,9 @@ def prefix(self):
def remove_prefix(self):
"""Removes the prefix for a package along with any empty parent directories."""
if self.dirty:
return
if os.path.exists(self.prefix):
shutil.rmtree(self.prefix, True)
@ -448,10 +455,15 @@ def do_install(self):
self.install(self.prefix)
if not os.path.isdir(self.prefix):
tty.die("Install failed for %s. No install dir created." % self.name)
except subprocess.CalledProcessError, e:
if not self.dirty:
self.remove_prefix()
self.remove_prefix()
tty.die("Install failed for %s" % self.name, e.message)
except KeyboardInterrupt, e:
self.remove_prefix()
raise
except Exception, e:
if not self.dirty:
self.remove_prefix()
@ -576,7 +588,7 @@ def __str__(self):
def depends_on(*args, **kwargs):
"""Adds a depends_on local variable in the locals of
"""Adds a dependencies local variable in the locals of
the calling class, based on args.
"""
# This gets the calling frame so we can pop variables into it

View file

@ -10,6 +10,7 @@
import spack.arch as arch
import spack.version as version
import spack.attr as attr
import spack.error as serr
# Valid package names
valid_package = r'^[a-zA-Z0-9_-]*$'
@ -19,6 +20,13 @@
instances = {}
class InvalidPackageNameError(serr.SpackError):
"""Raised when we encounter a bad package name."""
def __init__(self, name):
super(InvalidPackageNameError, self).__init__(
"Invalid package name: " + name)
self.name = name
def valid_name(pkg):
return re.match(valid_package, pkg) and not re.search(invalid_package, pkg)
@ -26,7 +34,7 @@ def valid_name(pkg):
def validate_name(pkg):
if not valid_name(pkg):
raise spack.InvalidPackageNameException(pkg)
raise spack.InvalidPackageNameError(pkg)
def filename_for(pkg):
@ -36,7 +44,7 @@ def filename_for(pkg):
def installed_packages(**kwargs):
"""Returns a dict from SysType to lists of Package objects."""
"""Returns a dict from systype strings to lists of Package objects."""
list_installed = kwargs.get('installed', False)
pkgs = {}
@ -44,7 +52,7 @@ def installed_packages(**kwargs):
return pkgs
for sys_type in os.listdir(spack.install_path):
sys_type = arch.SysType(sys_type)
sys_type = sys_type
sys_path = new_path(spack.install_path, sys_type)
pkgs[sys_type] = [get(pkg) for pkg in os.listdir(sys_path)
if os.path.isdir(new_path(sys_path, pkg))]

View file

@ -11,6 +11,7 @@ class Libdwarf(Package):
depends_on("libelf")
def clean(self):
for dir in dwarf_dirs:
with working_dir(dir):
@ -19,6 +20,7 @@ def clean(self):
def install(self, prefix):
# dwarf build does not set arguments for ar properly
make.add_default_arg('ARFLAGS=rcs')
# Dwarf doesn't provide an install, so we have to do it.
@ -43,3 +45,9 @@ def install(self, prefix):
install('dwarfdump', bin)
install('dwarfdump.conf', lib)
install('dwarfdump.1', man1)
@platform('macosx_10.8_x86_64')
def install(self, prefix):
raise UnsupportedPlatformError(
"libdwarf doesn't currently build on Mac OS X.")

View file

@ -5,8 +5,16 @@
import getpass
import spack
import spack.error as serr
import tty
class FailedDownloadError(serr.SpackError):
"""Raised wen a download fails."""
def __init__(self, url):
super(FailedDownloadError, self).__init__(
"Failed to fetch file from URL: " + url)
self.url = url
class Stage(object):
"""A Stage object manaages a directory where an archive is downloaded,
@ -161,7 +169,7 @@ def fetch(self):
"your internet gateway issue and install again.")
if not self.archive_file:
raise FailedDownloadException(url)
raise FailedDownloadError(url)
return self.archive_file

View file

@ -26,7 +26,7 @@ def memoized(obj):
def memoizer(*args, **kwargs):
if args not in cache:
cache[args] = obj(*args, **kwargs)
return cache[args]
return cache[args]
return memoizer

View file

@ -2,7 +2,29 @@
import re
import utils
from exception import *
import spack.error as serr
class VersionParseError(serr.SpackError):
"""Raised when the version module can't parse something."""
def __init__(self, msg, spec):
super(VersionParseError, self).__init__(msg)
self.spec = spec
class UndetectableVersionError(VersionParseError):
"""Raised when we can't parse a version from a string."""
def __init__(self, spec):
super(UndetectableVersionError, self).__init__(
"Couldn't detect version in: " + spec, spec)
class UndetectableNameError(VersionParseError):
"""Raised when we can't parse a package name from a string."""
def __init__(self, spec):
super(UndetectableNameError, self).__init__(
"Couldn't parse package name in: " + spec)
class Version(object):
"""Class to represent versions"""
@ -32,6 +54,15 @@ def component(self, i):
else:
return None
def up_to(self, index):
"""Return a version string up to the specified component, exclusive.
e.g., if this is 10.8.2, self.up_to(2) will return '10.8'.
"""
return '.'.join(str(x) for x in self[:index])
def __getitem__(self, idx):
return tuple(self.version[idx])
def __repr__(self):
return self.version_string
@ -123,7 +154,7 @@ def parse_version_string_with_indices(spec):
if match and match.group(1) is not None:
return match.group(1), match.start(1), match.end(1)
raise UndetectableVersionException(spec)
raise UndetectableVersionError(spec)
def parse_version(spec):
@ -162,7 +193,7 @@ def parse_name(spec, ver=None):
match = re.search(nt, spec)
if match:
return match.group(1)
raise UndetectableNameException(spec)
raise UndetectableNameError(spec)
def parse(spec):
ver = parse_version(spec)