Adding dotkit support to TAU.

- New spack.hooks package
  - contains modules with pre and post install hooks

- New dotkit hook module
  - generates/removes dotkits on install/uninstall

- New spack use, spack unuse commands
  - use same syntax as install/uninstall

- New setup-env.bash script
  - Sets up path, dotkit support

- new spack dotkit command
  - used by script to parse specs, generate
    specs of installed pckages for dotkit file names
This commit is contained in:
Todd Gamblin 2014-07-08 01:56:32 -07:00
parent 0551638944
commit 295ffd8c50
14 changed files with 552 additions and 21 deletions

1
.gitignore vendored
View file

@ -5,3 +5,4 @@
.DS_Store
.idea
/etc/spackconfig
/share/spack/dotkit

View file

@ -51,12 +51,14 @@ def msg(message, *args):
def info(message, *args, **kwargs):
format = kwargs.get('format', '*b')
cprint("@%s{==>} %s" % (format, cescape(str(message))))
stream = kwargs.get('stream', sys.stdout)
cprint("@%s{==>} %s" % (format, cescape(str(message))), stream=stream)
for arg in args:
lines = textwrap.wrap(
str(arg), initial_indent=indent, subsequent_indent=indent)
for line in lines:
print line
stream.write(line + '\n')
def verbose(message, *args):
@ -66,15 +68,15 @@ def verbose(message, *args):
def debug(message, *args):
if _debug:
info(message, *args, format='g')
info(message, *args, format='g', stream=sys.stderr)
def error(message, *args):
info("Error: " + str(message), *args, format='*r')
info("Error: " + str(message), *args, format='*r', stream=sys.stderr)
def warn(message, *args):
info("Warning: " + str(message), *args, format='*Y')
info("Warning: " + str(message), *args, format='*Y', stream=sys.stderr)
def die(message, *args):

View file

@ -53,9 +53,12 @@
module_path = join_path(lib_path, "spack")
compilers_path = join_path(module_path, "compilers")
test_path = join_path(module_path, "test")
hooks_path = join_path(module_path, "hooks")
var_path = join_path(prefix, "var", "spack")
stage_path = join_path(var_path, "stage")
install_path = join_path(prefix, "opt")
share_path = join_path(prefix, "share", "spack")
dotkit_path = join_path(share_path, "dotkit")
#
# Set up the packages database.
@ -90,7 +93,7 @@
# Version information
from spack.version import Version
spack_version = Version("1.0")
spack_version = Version("0.8")
#
# Executables used by Spack

View file

@ -0,0 +1,99 @@
##############################################################################
# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://scalability-llnl.github.io/spack
# Please also see the LICENSE file for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License (as published by
# the Free Software Foundation) version 2.1 dated February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software Foundation,
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
import sys
import os
import shutil
import argparse
import llnl.util.tty as tty
from llnl.util.lang import partition_list
from llnl.util.filesystem import mkdirp
import spack.cmd
import spack.hooks.dotkit
from spack.spec import Spec
description ="Find dotkits for packages if they exist."
def setup_parser(subparser):
subparser.add_argument(
'--refresh', action='store_true', help='Regenerate all dotkits')
subparser.add_argument(
'spec', nargs=argparse.REMAINDER, help='spec to find a dotkit for.')
def dotkit_find(parser, args):
if not args.spec:
parser.parse_args(['dotkit', '-h'])
spec = spack.cmd.parse_specs(args.spec)
if len(spec) > 1:
tty.die("You can only pass one spec.")
spec = spec[0]
if not spack.db.exists(spec.name):
tty.die("No such package: %s" % spec.name)
specs = [s for s in spack.db.installed_package_specs() if s.satisfies(spec)]
if len(specs) == 0:
tty.die("No installed packages match spec %s" % spec)
if len(specs) > 1:
tty.error("Multiple matches for spec %s. Choose one:" % spec)
for s in specs:
sys.stderr.write(s.tree(color=True))
sys.exit(1)
match = specs[0]
if not os.path.isfile(spack.hooks.dotkit.dotkit_file(match.package)):
tty.die("No dotkit is installed for package %s." % spec)
print match.format('$_$@$+$%@$=$#')
def dotkit_refresh(parser, args):
query_specs = spack.cmd.parse_specs(args.spec)
specs = spack.db.installed_package_specs()
if query_specs:
specs = [s for s in specs
if any(s.satisfies(q) for q in query_specs)]
else:
shutil.rmtree(spack.dotkit_path, ignore_errors=False)
mkdirp(spack.dotkit_path)
for spec in specs:
spack.hooks.dotkit.post_install(spec.package)
def dotkit(parser, args):
if args.refresh:
dotkit_refresh(parser, args)
else:
dotkit_find(parser, args)

View file

@ -22,6 +22,7 @@
# along with this program; if not, write to the Free Software Foundation,
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
import sys
import collections
import argparse
from StringIO import StringIO
@ -37,12 +38,14 @@
description ="Find installed spack packages"
def setup_parser(subparser):
subparser.add_argument(
format_group = subparser.add_mutually_exclusive_group()
format_group.add_argument(
'-p', '--paths', action='store_true', dest='paths',
help='Show paths to package install directories')
subparser.add_argument(
format_group.add_argument(
'-l', '--long', action='store_true', dest='full_specs',
help='Show full-length specs of installed packages')
subparser.add_argument(
'query_specs', nargs=argparse.REMAINDER,
help='optional specs to filter results')
@ -56,13 +59,16 @@ def find(parser, args):
if nonexisting:
msg = "No such package%s: " % ('s' if len(nonexisting) > 1 else '')
tty.msg(msg + ", ".join(s.name for s in nonexisting))
msg += ", ".join(s.name for s in nonexisting)
tty.msg(msg)
if not query_specs:
return
# Make a dict with specs keyed by architecture and compiler.
specs = [s for s in spack.db.installed_package_specs()
if not query_specs or any(s.satisfies(q) for q in query_specs)]
# Make a dict with specs keyed by architecture and compiler.
index = index_by(specs, 'architecture', 'compiler')
# Traverse the index and print out each package

View file

@ -0,0 +1,36 @@
##############################################################################
# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://scalability-llnl.github.io/spack
# Please also see the LICENSE file for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License (as published by
# the Free Software Foundation) version 2.1 dated February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software Foundation,
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
import argparse
import spack.cmd.use
description ="Remove package from environment using dotkit."
def setup_parser(subparser):
subparser.add_argument(
'spec', nargs=argparse.REMAINDER, help='Spec of package to remove.')
def unuse(parser, args):
spack.cmd.use.print_help()

View file

@ -0,0 +1,50 @@
##############################################################################
# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://scalability-llnl.github.io/spack
# Please also see the LICENSE file for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License (as published by
# the Free Software Foundation) version 2.1 dated February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software Foundation,
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
import argparse
import llnl.util.tty as tty
import spack
description ="Add package to environment using dotkit."
def setup_parser(subparser):
subparser.add_argument(
'spec', nargs=argparse.REMAINDER, help='Spec of package to add.')
def print_help():
tty.msg("Spack dotkit support is not initialized.",
"",
"To use dotkit with Spack, you must first run the command",
"below, which you can copy and paste:",
"",
"For bash:",
" . %s/setup-env.bash" % spack.share_path,
"",
"ksh/csh/tcsh shells are currently unsupported",
"")
def use(parser, args):
print_help()

View file

@ -0,0 +1,71 @@
##############################################################################
# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://scalability-llnl.github.io/spack
# Please also see the LICENSE file for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License (as published by
# the Free Software Foundation) version 2.1 dated February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software Foundation,
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
"""This package contains modules with hooks for various stages in the
Spack install process. You can add modules here and they'll be
executaed by package at various times during the package lifecycle.
Each hook is just a function that takes a package as a parameter.
Hooks are not executed in any particular order.
Currently the following hooks are supported:
* post_install()
* post_uninstall()
This can be used to implement support for things like module
systems (e.g. modules, dotkit, etc.) or to add other custom
features.
"""
import imp
from llnl.util.lang import memoized, list_modules
from llnl.util.filesystem import join_path
import spack
@memoized
def all_hook_modules():
modules = []
for name in list_modules(spack.hooks_path):
path = join_path(spack.hooks_path, name) + ".py"
modules.append(imp.load_source('spack.hooks', path))
return modules
class HookRunner(object):
def __init__(self, hook_name):
self.hook_name = hook_name
def __call__(self, pkg):
for module in all_hook_modules():
if hasattr(module, self.hook_name):
hook = getattr(module, self.hook_name)
if hasattr(hook, '__call__'):
hook(pkg)
#
# Define some functions that can be called to fire off hooks.
#
post_install = HookRunner('post_install')
post_uninstall = HookRunner('post_uninstall')

View file

@ -0,0 +1,83 @@
##############################################################################
# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://scalability-llnl.github.io/spack
# Please also see the LICENSE file for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License (as published by
# the Free Software Foundation) version 2.1 dated February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software Foundation,
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
import os
import re
import textwrap
import shutil
from contextlib import closing
from llnl.util.filesystem import join_path
import spack
def dotkit_file(pkg):
dk_file_name = pkg.spec.format('$_$@$%@$+$=$#') + ".dk"
return join_path(spack.dotkit_path, dk_file_name)
def post_install(pkg):
if not os.path.exists(spack.dotkit_path):
mkdirp(spack.dotkit_path)
alterations = []
for var, path in [
('PATH', pkg.prefix.bin),
('MANPATH', pkg.prefix.man),
('MANPATH', pkg.prefix.share_man),
('LD_LIBRARY_PATH', pkg.prefix.lib),
('LD_LIBRARY_PATH', pkg.prefix.lib64)]:
if os.path.isdir(path):
alterations.append("dk_alter %s %s\n" % (var, path))
if not alterations:
return
alterations.append("dk_alter CMAKE_PREFIX_PATH %s\n" % pkg.prefix)
dk_file = dotkit_file(pkg)
with closing(open(dk_file, 'w')) as dk:
# Put everything in the spack category.
dk.write('#c spack\n')
dk.write('#d %s\n' % pkg.spec.format("$_ $@"))
# Recycle the description
if pkg.__doc__:
doc = re.sub(r'\s+', ' ', pkg.__doc__)
for line in textwrap.wrap(doc, 72):
dk.write("#h %s\n" % line)
# Write alterations
for alter in alterations:
dk.write(alter)
def post_uninstall(pkg):
dk_file = dotkit_file(pkg)
if os.path.exists(dk_file):
shutil.rmtree(dk_file, ignore_errors=True)

View file

@ -48,6 +48,7 @@
import spack
import spack.spec
import spack.error
import spack.hooks
import spack.build_environment as build_env
import spack.url as url
from spack.version import *
@ -495,7 +496,7 @@ def installed_dependents(self):
on this one."""
dependents = []
for spec in spack.db.installed_package_specs():
if self.name in spec.dependencies:
if self in spec.dependencies:
dependents.append(spec)
return dependents
@ -703,6 +704,10 @@ def do_install(self, **kwargs):
sys.exit(1)
# Once everything else is done, run post install hooks
spack.hooks.post_install(self)
def do_install_dependencies(self):
# Pass along paths of dependencies here
for dep in self.spec.dependencies.values():
@ -731,13 +736,17 @@ def do_uninstall(self, **kwargs):
if not force:
deps = self.installed_dependents
formatted_deps = [s.format('$_$@$%@$+$=$#') for s in deps]
if deps: raise InstallError(
"Cannot uninstall %s. The following installed packages depend on it: %s"
% (self.spec, deps))
% (self.spec, formatted_deps))
self.remove_prefix()
tty.msg("Successfully uninstalled %s." % self.spec)
# Once everything else is done, run post install hooks
spack.hooks.post_uninstall(self)
def do_clean(self):
if self.stage.expanded_archive_path:

View file

@ -69,7 +69,8 @@ def __new__(cls, path):
s.share = join_path(s, 'share')
s.doc = join_path(s.share, 'doc')
s.info = join_path(s.share, 'info')
s.man = join_path(s.share, 'man')
s.man = join_path(s, 'man')
s.man1 = join_path(s.man, 'man1')
s.man2 = join_path(s.man, 'man2')
s.man3 = join_path(s.man, 'man3')
@ -78,4 +79,15 @@ def __new__(cls, path):
s.man6 = join_path(s.man, 'man6')
s.man7 = join_path(s.man, 'man7')
s.man8 = join_path(s.man, 'man8')
s.share_man = join_path(s.share, 'man')
s.share_man1 = join_path(s.share_man, 'man1')
s.share_man2 = join_path(s.share_man, 'man2')
s.share_man3 = join_path(s.share_man, 'man3')
s.share_man4 = join_path(s.share_man, 'man4')
s.share_man5 = join_path(s.share_man, 'man5')
s.share_man6 = join_path(s.share_man, 'man6')
s.share_man7 = join_path(s.share_man, 'man7')
s.share_man8 = join_path(s.share_man, 'man8')
return s

132
share/spack/setup-env.bash Executable file
View file

@ -0,0 +1,132 @@
##############################################################################
# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://scalability-llnl.github.io/spack
# Please also see the LICENSE file for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License (as published by
# the Free Software Foundation) version 2.1 dated February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software Foundation,
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
#
#
# This file is part of Spack and sets up the spack environment for
# bash shells. This includes dotkit support as well as putting spack
# in your path. Source it like this:
#
# . /path/to/spack/share/spack/setup-env.bash
#
#
########################################################################
# This is a wrapper around the spack command that forwards calls to
# 'spack use' and 'spack unuse' to shell functions. This in turn
# allows them to be used to invoke dotkit functions.
#
# 'spack use' is smarter than just 'use' because it converts its
# arguments into a unique spack spec that is then passed to dotkit
# commands. This allows the user to use packages without knowing all
# their installation details.
#
# e.g., rather than requring a full spec for libelf, the user can type:
#
# spack use libelf
#
# This will first find the available libelf dotkits and use a
# matching one. If there are two versions of libelf, the user would
# need to be more specific, e.g.:
#
# spack use libelf@0.8.13
#
# This is very similar to how regular spack commands work and it
# avoids the need to come up with a user-friendly naming scheme for
# spack dotfiles.
########################################################################
function spack {
_spack_subcommand=$1; shift
_spack_spec="$@"
# Filter out use and unuse. For any other commands, just run the
# command.
case $_spack_subcommand in
"use") ;;
"unuse") ;;
*)
command spack $_spack_subcommand "$@"
return
;;
esac
# If no args or -h, just run that command as well.
if [ -z "$1" -o "$1" = "-h" ]; then
command spack $_spack_subcommand -h
return
fi
# Shift any other args for use off before parsing spec.
_spack_use_args=""
if [[ "$1" =~ ^- ]]; then
_spack_use_args="$1"; shift
_spack_spec="$@"
fi
# Here the user has run use or unuse with a spec. Find a matching
# spec with a dotkit using spack dotkit, then use or unuse the
# result. If spack dotkit comes back with an error, do nothing.
if _spack_full_spec=$(command spack dotkit $_spack_spec); then
$_spack_subcommand $_spack_use_args $_spack_full_spec
fi
}
########################################################################
# Prepends directories to path, if they exist.
# pathadd /path/to/dir # add to PATH
# or pathadd OTHERPATH /path/to/dir # add to OTHERPATH
########################################################################
function _spack_pathadd {
# If no variable name is supplied, just append to PATH
# otherwise append to that variable.
varname=PATH
path="$1"
if [ -n "$2" ]; then
varname="$1"
path="$2"
fi
# Do the actual prepending here.
eval "oldvalue=\"\$$varname\""
if [ -d "$path" ] && [[ ":$oldvalue:" != *":$path:"* ]]; then
if [ -n "$oldvalue" ]; then
eval "export $varname=\"$path:$oldvalue\""
else
export $varname="$path"
fi
fi
}
#
# Set up dotkit and path in the user environment
#
_spack_share_dir="$(dirname ${BASH_SOURCE[0]})"
_spack_prefix="$(dirname $(dirname $_spack_share_dir))"
_spack_pathadd DK_NODE "$_spack_share_dir/dotkit"
_spack_pathadd PATH "$_spack_prefix/bin"

View file

@ -29,6 +29,17 @@
dwarf_dirs = ['libdwarf', 'dwarfdump2']
class Libdwarf(Package):
"""The DWARF Debugging Information Format is of interest to
programmers working on compilers and debuggers (and any one
interested in reading or writing DWARF information). It was
developed by a committee (known as the PLSIG at the time)
starting around 1991. Starting around 1991 SGI developed the
libdwarf and dwarfdump tools for internal use and as part of
SGI IRIX developer tools. Since that time dwarfdump and
libdwarf have been shipped (as an executable and archive
respectively, not source) with every release of the SGI
MIPS/IRIX C compiler."""
homepage = "http://www.prevanders.net/dwarf.html"
url = "http://www.prevanders.net/libdwarf-20130729.tar.gz"
list_url = homepage

View file

@ -1,5 +1,8 @@
from spack import *
import os
from llnl.util.filesystem import join_path
class Tau(Package):
"""A portable profiling and tracing toolkit for performance
analysis of parallel programs written in Fortran, C, C++, UPC,
@ -18,3 +21,16 @@ def install(self, spec, prefix):
# After that, it's relatively standard.
configure("-prefix=%s" % prefix)
make("install")
# Link arch-specific directories into prefix since there is
# only one arch per prefix the way spack installs.
self.link_tau_arch_dirs()
def link_tau_arch_dirs(self):
for subdir in os.listdir(self.prefix):
for d in ('bin', 'lib'):
src = join_path(self.prefix, subdir, d)
dest = join_path(self.prefix, d)
if os.path.isdir(src) and not os.path.exists(dest):
os.symlink(join_path(subdir, d), dest)