From 49956faab97e2dea52371755d0fcb9b013567ac7 Mon Sep 17 00:00:00 2001 From: Brett Viren Date: Wed, 24 Feb 2016 12:19:06 +0100 Subject: [PATCH 01/13] First try to make file system views of specs. --- lib/spack/spack/cmd/view.py | 108 ++++++++++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 lib/spack/spack/cmd/view.py diff --git a/lib/spack/spack/cmd/view.py b/lib/spack/spack/cmd/view.py new file mode 100644 index 0000000000..c7f1c21f67 --- /dev/null +++ b/lib/spack/spack/cmd/view.py @@ -0,0 +1,108 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import os +import argparse + +import spack +import spack.cmd +import llnl.util.tty as tty + +description = "Produce a single-rooted directory view of a spec." + +def setup_parser(subparser): + setup_parser.parser = subparser + + subparser.add_argument('prefix', nargs=1, + help="Path to top-level directory for the view.") + subparser.add_argument('specs', nargs=argparse.REMAINDER, + help="specs of packages to expose in the view.") + +def assuredir(path): + 'Assure path exists as a directory' + if not os.path.exists(path): + os.makedirs(path) + +def relative_to(prefix, path): + assert 0 == path.find(prefix) + reldir = path[len(prefix):] + if reldir.startswith('/'): + reldir = reldir[1:] + return reldir + +def view_one(prefix, spec): + print spec.name,spec.prefix + + dotspack = os.path.join(prefix, '.spack', spec.name) + + if os.path.exists(os.path.join(dotspack)): + tty.warn("Skipping previously added package %s"%spec.name) + return + + + for dirpath,dirnames,filenames in os.walk(spec.prefix): + if not filenames: + continue # avoid explicitly making empty dirs + + reldir = relative_to(spec.prefix, dirpath) + + # fixme: assumes .spack has no subdirs + if dirpath.endswith('.spack'): + targdir = dotspack + else: + targdir = os.path.join(prefix, reldir) + + assuredir(targdir) + + print '\t%s...'%reldir, + nlinks = 0 + for fname in filenames: + src = os.path.join(dirpath, fname) + dst = os.path.join(targdir, fname) + if os.path.exists(dst): + tty.warn("Skipping existing file for view: %s" % dst) + continue + os.symlink(src,dst) + nlinks += 1 + print nlinks + + +def view(parser, args): + # if not args.specs: + # tty.die("view creation requires at least one spec argument") + + specs = spack.cmd.parse_specs(args.specs, normalize=True, concretize=True) + if not specs: + setup_parser.parser.print_help() + return 1 + + prefix = args.prefix[0] + assuredir(prefix) + + flat = set() + for spec in specs: + flat.update(spec.normalized().traverse()) + + for spec in flat: + view_one(prefix, spec) From a2b9a000dcb18868ea9f1c110632a251fbea92f3 Mon Sep 17 00:00:00 2001 From: Brett Viren Date: Fri, 29 Apr 2016 15:08:54 -0400 Subject: [PATCH 02/13] Add add removal and status actions in addition to link and add various ways to filter what is done. --- lib/spack/spack/cmd/view.py | 130 +++++++++++++++++++++++++++++------- 1 file changed, 107 insertions(+), 23 deletions(-) diff --git a/lib/spack/spack/cmd/view.py b/lib/spack/spack/cmd/view.py index c7f1c21f67..dd698350b2 100644 --- a/lib/spack/spack/cmd/view.py +++ b/lib/spack/spack/cmd/view.py @@ -1,3 +1,8 @@ +''' +Produce a file-system "view" of a Spack DAG. + +Concept from Nix, implemented by brett.viren@gmail.com ca 2016. +''' ############################################################################## # Copyright (c) 2013, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory. @@ -22,7 +27,9 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## + import os +import re import argparse import spack @@ -34,8 +41,15 @@ def setup_parser(subparser): setup_parser.parser = subparser - subparser.add_argument('prefix', nargs=1, - help="Path to top-level directory for the view.") + subparser.add_argument('-e','--exclude', action='append', default=[], + help="exclude any packages which the given re pattern") + subparser.add_argument('-a','--action', default='link', + choices=['link','remove','status'], # names match action_*() functions below + help="what action to perform on the view") + subparser.add_argument('--no-dependencies', action='store_true', default=False, + help="just operate on named packages and do not follow dependencies") + subparser.add_argument('-p','--prefix', required=True, + help="Path to a top-level directory to receive the view.") subparser.add_argument('specs', nargs=argparse.REMAINDER, help="specs of packages to expose in the view.") @@ -45,64 +59,134 @@ def assuredir(path): os.makedirs(path) def relative_to(prefix, path): + 'Return end of `path` relative to `prefix`' assert 0 == path.find(prefix) reldir = path[len(prefix):] if reldir.startswith('/'): reldir = reldir[1:] return reldir -def view_one(prefix, spec): - print spec.name,spec.prefix + +def transform_path(spec, path, prefix=None): + 'Return the a relative path corresponding to given path spec.prefix' + if os.path.isabs(path): + path = relative_to(spec.prefix, path) + subdirs = path.split(os.path.sep) + if subdirs[0] == '.spack': + lst = ['.spack',spec.name]+subdirs[1:] + path = os.path.join(*lst) + if prefix: + path = os.path.join(prefix, path) + return path +def action_status(spec, prefix): + 'Check status of view in prefix against spec' dotspack = os.path.join(prefix, '.spack', spec.name) - if os.path.exists(os.path.join(dotspack)): - tty.warn("Skipping previously added package %s"%spec.name) + tty.info("Package added: %s"%spec.name) + return + tty.info("Package missing: %s"%spec.name) + return + +def action_remove(spec, prefix): + 'Remove any files found in spec from prefix and purge empty directories.' + + if not os.path.exists(prefix): return + dotspack = transform_path(spec, '.spack', prefix) + if not os.path.exists(dotspack): + tty.info("Skipping nonexistent package %s"%spec.name) + return + + for dirpath,dirnames,filenames in os.walk(spec.prefix): + if not filenames: + continue + + targdir = transform_path(spec, dirpath, prefix) + for fname in filenames: + src = os.path.join(dirpath, fname) + dst = os.path.join(targdir, fname) + if not os.path.exists(dst): + #tty.warn("Skipping nonexistent file for view: %s" % dst) + continue + os.unlink(dst) + + +def action_link(spec, prefix): + 'Symlink all files in `spec` into directory `prefix`.' + + dotspack = transform_path(spec, '.spack', prefix) + if os.path.exists(dotspack): + tty.warn("Skipping previously added package %s"%spec.name) + return for dirpath,dirnames,filenames in os.walk(spec.prefix): if not filenames: continue # avoid explicitly making empty dirs - reldir = relative_to(spec.prefix, dirpath) - - # fixme: assumes .spack has no subdirs - if dirpath.endswith('.spack'): - targdir = dotspack - else: - targdir = os.path.join(prefix, reldir) - + targdir = transform_path(spec, dirpath, prefix) assuredir(targdir) - print '\t%s...'%reldir, - nlinks = 0 for fname in filenames: src = os.path.join(dirpath, fname) dst = os.path.join(targdir, fname) if os.path.exists(dst): + if '.spack' in dst.split(os.path.sep): + continue # silence these tty.warn("Skipping existing file for view: %s" % dst) continue os.symlink(src,dst) - nlinks += 1 - print nlinks - + + + +def purge_empty_directories(path): + 'Ascend up from the leaves accessible from `path` and remove empty directories.' + for dirpath, subdirs, files in os.walk(path, topdown=False): + for sd in subdirs: + sdp = os.path.join(dirpath,sd) + try: + os.rmdir(sdp) + except OSError: + tty.warn("Not removing directory with contents: %s" % sdp) + def view(parser, args): - # if not args.specs: - # tty.die("view creation requires at least one spec argument") + 'The view command.' + to_exclude = [re.compile(e) for e in args.exclude] + def exclude(spec): + for e in to_exclude: + if e.match(spec.name): + return True + return False specs = spack.cmd.parse_specs(args.specs, normalize=True, concretize=True) if not specs: setup_parser.parser.print_help() return 1 - prefix = args.prefix[0] + prefix = args.prefix assuredir(prefix) flat = set() for spec in specs: + if args.no_dependencies: + flat.add(spec) + continue flat.update(spec.normalized().traverse()) + action = args.action.lower() + method = eval('action_' + action) + for spec in flat: - view_one(prefix, spec) + if exclude(spec): + tty.info('Skipping excluded package: "%s"' % spec.name) + continue + if not os.path.exists(spec.prefix): + tty.warn('Skipping unknown package: %s in %s' % (spec.name, spec.prefix)) + continue + tty.info("%s %s" % (action, spec.name)) + method(spec, prefix) + + if action in ['remove']: + purge_empty_directories(prefix) From 8c140f4725608510d0d7ecc6195736ca513b02ab Mon Sep 17 00:00:00 2001 From: Brett Viren Date: Sat, 30 Apr 2016 08:46:47 -0400 Subject: [PATCH 03/13] Change the action from parameter to subcommand to fit Spack convention. --- lib/spack/spack/cmd/view.py | 58 ++++++++++++++++++++++++++----------- 1 file changed, 41 insertions(+), 17 deletions(-) diff --git a/lib/spack/spack/cmd/view.py b/lib/spack/spack/cmd/view.py index dd698350b2..a82c034398 100644 --- a/lib/spack/spack/cmd/view.py +++ b/lib/spack/spack/cmd/view.py @@ -41,17 +41,30 @@ def setup_parser(subparser): setup_parser.parser = subparser - subparser.add_argument('-e','--exclude', action='append', default=[], - help="exclude any packages which the given re pattern") - subparser.add_argument('-a','--action', default='link', - choices=['link','remove','status'], # names match action_*() functions below - help="what action to perform on the view") - subparser.add_argument('--no-dependencies', action='store_true', default=False, - help="just operate on named packages and do not follow dependencies") - subparser.add_argument('-p','--prefix', required=True, - help="Path to a top-level directory to receive the view.") - subparser.add_argument('specs', nargs=argparse.REMAINDER, - help="specs of packages to expose in the view.") + sp = subparser.add_subparsers(metavar='ACTION', dest='action') + + # The action parameterizes the command but in keeping with Spack + # patterns we make it a subcommand. + sps = [ + sp.add_parser('link', aliases=['add'], + help='Add packages to the view, create view if needed.'), + sp.add_parser('remove', aliases=['rm'], + help='Remove packages from the view, and view if empty.'), + sp.add_parser('status', aliases=['check'], + help='Check status of packages in the view.') + ] + + # All these options and arguments are common to every action. + for p in sps: + p.add_argument('-e','--exclude', action='append', default=[], + help="exclude any packages which the given re pattern") + p.add_argument('--no-dependencies', action='store_true', default=False, + help="just operate on named packages and do not follow dependencies") + p.add_argument('prefix', nargs=1, + help="Path to a top-level directory to receive the view.") + p.add_argument('specs', nargs=argparse.REMAINDER, + help="specs of packages to expose in the view.") + def assuredir(path): 'Assure path exists as a directory' @@ -151,7 +164,9 @@ def purge_empty_directories(path): tty.warn("Not removing directory with contents: %s" % sdp) -def view(parser, args): + + +def view_action(action, parser, args): 'The view command.' to_exclude = [re.compile(e) for e in args.exclude] def exclude(spec): @@ -162,7 +177,7 @@ def exclude(spec): specs = spack.cmd.parse_specs(args.specs, normalize=True, concretize=True) if not specs: - setup_parser.parser.print_help() + parser.print_help() return 1 prefix = args.prefix @@ -175,9 +190,6 @@ def exclude(spec): continue flat.update(spec.normalized().traverse()) - action = args.action.lower() - method = eval('action_' + action) - for spec in flat: if exclude(spec): tty.info('Skipping excluded package: "%s"' % spec.name) @@ -186,7 +198,19 @@ def exclude(spec): tty.warn('Skipping unknown package: %s in %s' % (spec.name, spec.prefix)) continue tty.info("%s %s" % (action, spec.name)) - method(spec, prefix) + action(spec, prefix) if action in ['remove']: purge_empty_directories(prefix) + + +def view(parser, args): + action = { + 'add': action_link, + 'link': action_link, + 'remove': action_remove, + 'rm': action_remove, + 'status': action_status, + 'check': action_status + }[args.action] + view_action(action, parser, args) From 346f1022343d608ffabd565636ec8358c90464c8 Mon Sep 17 00:00:00 2001 From: Brett Viren Date: Sun, 1 May 2016 13:54:07 -0400 Subject: [PATCH 04/13] Bug fixes, tty tweaks. --- lib/spack/spack/cmd/view.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/lib/spack/spack/cmd/view.py b/lib/spack/spack/cmd/view.py index a82c034398..1bdaaecadc 100644 --- a/lib/spack/spack/cmd/view.py +++ b/lib/spack/spack/cmd/view.py @@ -46,7 +46,7 @@ def setup_parser(subparser): # The action parameterizes the command but in keeping with Spack # patterns we make it a subcommand. sps = [ - sp.add_parser('link', aliases=['add'], + sp.add_parser('add', aliases=['link'], help='Add packages to the view, create view if needed.'), sp.add_parser('remove', aliases=['rm'], help='Remove packages from the view, and view if empty.'), @@ -112,6 +112,7 @@ def action_remove(spec, prefix): tty.info("Skipping nonexistent package %s"%spec.name) return + tty.info("remove %s"%spec.name) for dirpath,dirnames,filenames in os.walk(spec.prefix): if not filenames: continue @@ -125,7 +126,6 @@ def action_remove(spec, prefix): continue os.unlink(dst) - def action_link(spec, prefix): 'Symlink all files in `spec` into directory `prefix`.' @@ -134,6 +134,7 @@ def action_link(spec, prefix): tty.warn("Skipping previously added package %s"%spec.name) return + tty.info("link %s" % spec.name) for dirpath,dirnames,filenames in os.walk(spec.prefix): if not filenames: continue # avoid explicitly making empty dirs @@ -151,8 +152,6 @@ def action_link(spec, prefix): continue os.symlink(src,dst) - - def purge_empty_directories(path): 'Ascend up from the leaves accessible from `path` and remove empty directories.' for dirpath, subdirs, files in os.walk(path, topdown=False): @@ -161,13 +160,14 @@ def purge_empty_directories(path): try: os.rmdir(sdp) except OSError: - tty.warn("Not removing directory with contents: %s" % sdp) + #tty.warn("Not removing directory with contents: %s" % sdp) + pass def view_action(action, parser, args): - 'The view command.' + 'The view command parameterized by the action.' to_exclude = [re.compile(e) for e in args.exclude] def exclude(spec): for e in to_exclude: @@ -180,7 +180,7 @@ def exclude(spec): parser.print_help() return 1 - prefix = args.prefix + prefix = args.prefix[0] assuredir(prefix) flat = set() @@ -197,14 +197,15 @@ def exclude(spec): if not os.path.exists(spec.prefix): tty.warn('Skipping unknown package: %s in %s' % (spec.name, spec.prefix)) continue - tty.info("%s %s" % (action, spec.name)) action(spec, prefix) - if action in ['remove']: + if args.action in ['remove','rm']: purge_empty_directories(prefix) + def view(parser, args): + 'The view command.' action = { 'add': action_link, 'link': action_link, From 26c5bc9d979b95df560bdb7d7a6bd747ef93f83d Mon Sep 17 00:00:00 2001 From: Brett Viren Date: Sun, 1 May 2016 17:10:32 -0400 Subject: [PATCH 05/13] A few minimal test packages, fodder for testing 'spack view' command. --- .../builtin.mock/packages/test-a/package.py | 29 ++++++++++++++++ .../packages/test-a/test-a-0.0.tar.gz | Bin 0 -> 112 bytes .../builtin.mock/packages/test-b/package.py | 31 +++++++++++++++++ .../packages/test-b/test-b-0.0.tar.gz | Bin 0 -> 112 bytes .../builtin.mock/packages/test-c/package.py | 31 +++++++++++++++++ .../packages/test-c/test-c-0.0.tar.gz | Bin 0 -> 112 bytes .../builtin.mock/packages/test-d/package.py | 32 ++++++++++++++++++ .../packages/test-d/test-d-0.0.tar.gz | Bin 0 -> 112 bytes 8 files changed, 123 insertions(+) create mode 100644 var/spack/repos/builtin.mock/packages/test-a/package.py create mode 100644 var/spack/repos/builtin.mock/packages/test-a/test-a-0.0.tar.gz create mode 100644 var/spack/repos/builtin.mock/packages/test-b/package.py create mode 100644 var/spack/repos/builtin.mock/packages/test-b/test-b-0.0.tar.gz create mode 100644 var/spack/repos/builtin.mock/packages/test-c/package.py create mode 100644 var/spack/repos/builtin.mock/packages/test-c/test-c-0.0.tar.gz create mode 100644 var/spack/repos/builtin.mock/packages/test-d/package.py create mode 100644 var/spack/repos/builtin.mock/packages/test-d/test-d-0.0.tar.gz diff --git a/var/spack/repos/builtin.mock/packages/test-a/package.py b/var/spack/repos/builtin.mock/packages/test-a/package.py new file mode 100644 index 0000000000..2f72370580 --- /dev/null +++ b/var/spack/repos/builtin.mock/packages/test-a/package.py @@ -0,0 +1,29 @@ +from spack import * + +import os + +mydir = os.path.dirname(__file__) +source = os.path.join(mydir,'test-a-0.0.tar.gz') + +class TestA(Package): + """The test-a package""" + + url = 'file://'+source + + version('0.0', '4e823d0af4154fcf52b75dad41b7fd63') + + variant('nom', default=True, description='Nominal variant') + variant('var', default=False, description='Variant variant') + + def install(self, spec, prefix): + bindir = os.path.join(prefix,'bin') + os.makedirs(bindir) + script = os.path.join(bindir, 'test-a') + with open(script,'w') as fp: + fp.write("""#!/bin/bash +echo 'name: %s' +echo 'prefix: %s' +echo 'spec: %s' + """ % (spec.name, prefix, spec)) + + os.chmod(script, 0555) diff --git a/var/spack/repos/builtin.mock/packages/test-a/test-a-0.0.tar.gz b/var/spack/repos/builtin.mock/packages/test-a/test-a-0.0.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..3588e05765cd799dfe19e1f77e057bf7508ec47e GIT binary patch literal 112 zcmb2|=3uzVq!!M={Pvt8-ysJPmJ7D0<~95|Kkb{1f}-PrX$3W18BI)~SE|#F^vw8N zv-i!f&C_|0CdssyKPx$LK4$72|C!Dyr^A#M>z4NF`mcRfZGSIIclq(}ch}1?z`(y3 OtXZo*ykgK`U;qF>c`_gX literal 0 HcmV?d00001 diff --git a/var/spack/repos/builtin.mock/packages/test-b/package.py b/var/spack/repos/builtin.mock/packages/test-b/package.py new file mode 100644 index 0000000000..db64b0a556 --- /dev/null +++ b/var/spack/repos/builtin.mock/packages/test-b/package.py @@ -0,0 +1,31 @@ +from spack import * + +import os + +mydir = os.path.dirname(__file__) +source = os.path.join(mydir,'test-b-0.0.tar.gz') + +class TestB(Package): + """The test-b package""" + + url = 'file://'+source + + version('0.0', '4e823d0af4154fcf52b75dad41b7fd63') + + variant('nom', default=True, description='Nominal variant') + variant('var', default=False, description='Variant variant') + + depends_on('test-a') + + def install(self, spec, prefix): + bindir = os.path.join(prefix,'bin') + os.makedirs(bindir) + script = os.path.join(bindir, 'test-b') + with open(script,'w') as fp: + fp.write("""#!/bin/bash +echo 'name: %s' +echo 'prefix: %s' +echo 'spec: %s' + """ % (spec.name, prefix, spec)) + + os.chmod(script, 0555) diff --git a/var/spack/repos/builtin.mock/packages/test-b/test-b-0.0.tar.gz b/var/spack/repos/builtin.mock/packages/test-b/test-b-0.0.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..3588e05765cd799dfe19e1f77e057bf7508ec47e GIT binary patch literal 112 zcmb2|=3uzVq!!M={Pvt8-ysJPmJ7D0<~95|Kkb{1f}-PrX$3W18BI)~SE|#F^vw8N zv-i!f&C_|0CdssyKPx$LK4$72|C!Dyr^A#M>z4NF`mcRfZGSIIclq(}ch}1?z`(y3 OtXZo*ykgK`U;qF>c`_gX literal 0 HcmV?d00001 diff --git a/var/spack/repos/builtin.mock/packages/test-c/package.py b/var/spack/repos/builtin.mock/packages/test-c/package.py new file mode 100644 index 0000000000..0b03037466 --- /dev/null +++ b/var/spack/repos/builtin.mock/packages/test-c/package.py @@ -0,0 +1,31 @@ +from spack import * + +import os + +mydir = os.path.dirname(__file__) +source = os.path.join(mydir,'test-c-0.0.tar.gz') + +class TestC(Package): + """The test-c package""" + + url = 'file://'+source + + version('0.0', '4e823d0af4154fcf52b75dad41b7fd63') + + variant('nom', default=True, description='Nominal variant') + variant('var', default=False, description='Variant variant') + + depends_on('test-a+var',when='+var') + + def install(self, spec, prefix): + bindir = os.path.join(prefix,'bin') + os.makedirs(bindir) + script = os.path.join(bindir, 'test-c') + with open(script,'w') as fp: + fp.write("""#!/bin/bash +echo 'name: %s' +echo 'prefix: %s' +echo 'spec: %s' + """ % (spec.name, prefix, spec)) + + os.chmod(script, 0555) diff --git a/var/spack/repos/builtin.mock/packages/test-c/test-c-0.0.tar.gz b/var/spack/repos/builtin.mock/packages/test-c/test-c-0.0.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..3588e05765cd799dfe19e1f77e057bf7508ec47e GIT binary patch literal 112 zcmb2|=3uzVq!!M={Pvt8-ysJPmJ7D0<~95|Kkb{1f}-PrX$3W18BI)~SE|#F^vw8N zv-i!f&C_|0CdssyKPx$LK4$72|C!Dyr^A#M>z4NF`mcRfZGSIIclq(}ch}1?z`(y3 OtXZo*ykgK`U;qF>c`_gX literal 0 HcmV?d00001 diff --git a/var/spack/repos/builtin.mock/packages/test-d/package.py b/var/spack/repos/builtin.mock/packages/test-d/package.py new file mode 100644 index 0000000000..5cb7dcb2cb --- /dev/null +++ b/var/spack/repos/builtin.mock/packages/test-d/package.py @@ -0,0 +1,32 @@ +from spack import * + +import os + +mydir = os.path.dirname(__file__) +source = os.path.join(mydir,'test-d-0.0.tar.gz') + +class TestD(Package): + """The test-d package""" + + url = 'file://'+source + + version('0.0', '4e823d0af4154fcf52b75dad41b7fd63') + + variant('nom', default=True, description='Nominal variant') + variant('var', default=False, description='Variant variant') + + depends_on('test-b') + depends_on('test-c') + + def install(self, spec, prefix): + bindir = os.path.join(prefix,'bin') + os.makedirs(bindir) + script = os.path.join(bindir, 'test-d') + with open(script,'w') as fp: + fp.write("""#!/bin/bash +echo 'name: %s' +echo 'prefix: %s' +echo 'spec: %s' + """ % (spec.name, prefix, spec)) + + os.chmod(script, 0555) diff --git a/var/spack/repos/builtin.mock/packages/test-d/test-d-0.0.tar.gz b/var/spack/repos/builtin.mock/packages/test-d/test-d-0.0.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..3588e05765cd799dfe19e1f77e057bf7508ec47e GIT binary patch literal 112 zcmb2|=3uzVq!!M={Pvt8-ysJPmJ7D0<~95|Kkb{1f}-PrX$3W18BI)~SE|#F^vw8N zv-i!f&C_|0CdssyKPx$LK4$72|C!Dyr^A#M>z4NF`mcRfZGSIIclq(}ch}1?z`(y3 OtXZo*ykgK`U;qF>c`_gX literal 0 HcmV?d00001 From 2d1430da130e33fdd8d8345ce9b4cbe3dbdc2871 Mon Sep 17 00:00:00 2001 From: Brett Viren Date: Sat, 7 May 2016 18:24:24 -0400 Subject: [PATCH 06/13] Address all coments in @trws's latest comment in PR #869. I addressed them by factoring the code better to follow the visitor pattern. This will allow actions to be easily added in the future. These may not even be file sytsem views. One could add actions to generate shell init scripts, JSON DAG-dumpers, GraphViz DOT file generators, etc (yes, some of these are alread in there - just to give the idea). Also added is a top-level test $ source share/spack/setup-env.sh $ ./share/spack/examples/test_view.sh Read the top of that script first. --- lib/spack/spack/cmd/view.py | 318 +++++++++++------- share/spack/examples/test_view.sh | 31 ++ .../builtin.mock/packages/test-a/package.py | 1 + .../builtin.mock/packages/test-b/package.py | 1 + .../builtin.mock/packages/test-c/package.py | 1 + .../builtin.mock/packages/test-d/package.py | 1 + 6 files changed, 231 insertions(+), 122 deletions(-) create mode 100755 share/spack/examples/test_view.sh diff --git a/lib/spack/spack/cmd/view.py b/lib/spack/spack/cmd/view.py index 1bdaaecadc..31937b39a2 100644 --- a/lib/spack/spack/cmd/view.py +++ b/lib/spack/spack/cmd/view.py @@ -1,8 +1,3 @@ -''' -Produce a file-system "view" of a Spack DAG. - -Concept from Nix, implemented by brett.viren@gmail.com ca 2016. -''' ############################################################################## # Copyright (c) 2013, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory. @@ -27,6 +22,45 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +'''Produce a "view" of a Spack DAG. + +A "view" is the product of applying a function on a set of package specs. + +This set consists of: + +- specs resolved from the package names given by the user (the seeds) +- all depenencies of the seeds unless user specifies `--no-depenencies` +- less any specs with names matching the regular expressions given by `--exclude` + +The `view` command provides a number of functions (the "actions"): + +- symlink :: a file system view which is a directory hierarchy that is + the union of the hierarchies of the installed packages in the DAG + where installed files are referenced via symlinks. + +- hardlink :: like the symlink view but hardlinks are used + +- statlink :: a view producing a status report of a symlink or + hardlink view. + + +The file system view concept is imspired by Nix, implemented by +brett.viren@gmail.com ca 2016. + +''' +# Implementation notes: +# +# This is implemented as a visitor pattern on the set of package specs. +# +# The command line ACTION maps to a visitor_*() function which takes +# the set of package specs and any args which may be specific to the +# ACTION. +# +# To add a new view: +# 1. add a new cmd line args sub parser ACTION +# 2. add any action-specific options/arguments, most likely a list of specs. +# 3. add a visitor_MYACTION() function +# 4. add any visitor_MYALIAS assignments to match any command line aliases import os import re @@ -38,33 +72,48 @@ description = "Produce a single-rooted directory view of a spec." -def setup_parser(subparser): - setup_parser.parser = subparser +def setup_parser(sp): + setup_parser.parser = sp - sp = subparser.add_subparsers(metavar='ACTION', dest='action') + sp.add_argument('-v','--verbose', action='store_true', default=False, + help="Display verbose output.") + sp.add_argument('-e','--exclude', action='append', default=[], + help="Exclude packages with names matching the given regex pattern.") + sp.add_argument('-d', '--dependencies', choices=['true','false','yes','no'], + default='true', + help="Follow dependencies.") + + + ssp = sp.add_subparsers(metavar='ACTION', dest='action') # The action parameterizes the command but in keeping with Spack # patterns we make it a subcommand. - sps = [ - sp.add_parser('add', aliases=['link'], - help='Add packages to the view, create view if needed.'), - sp.add_parser('remove', aliases=['rm'], - help='Remove packages from the view, and view if empty.'), - sp.add_parser('status', aliases=['check'], - help='Check status of packages in the view.') + file_system_view_actions = [ + ssp.add_parser('symlink', aliases=['add','soft'], + help='Add package files to a filesystem view via symbolic links.'), + ssp.add_parser('hardlink', aliases=['hard'], + help='Add packages files to a filesystem via via hard links.'), + ssp.add_parser('remove', aliases=['rm'], + help='Remove packages from a filesystem view.'), + ssp.add_parser('statlink', aliases=['status','check'], + help='Check status of packages in a filesystem view.') ] - # All these options and arguments are common to every action. - for p in sps: - p.add_argument('-e','--exclude', action='append', default=[], - help="exclude any packages which the given re pattern") - p.add_argument('--no-dependencies', action='store_true', default=False, - help="just operate on named packages and do not follow dependencies") - p.add_argument('prefix', nargs=1, - help="Path to a top-level directory to receive the view.") - p.add_argument('specs', nargs=argparse.REMAINDER, - help="specs of packages to expose in the view.") + for act in file_system_view_actions: + act.add_argument('path', nargs=1, + help="Path to file system view directory.") + act.add_argument('specs', metavar='spec', nargs='+', + help="Seed specs of the packages to view.") + ## Other VIEW ACTIONS might be added here. + ## Some ideas are the following (and some are redundant with existing cmds) + ## A JSON view that dumps a DAG to a JSON file + ## A DOT view that dumps to a GraphViz file + ## A SHELL INIT view that dumps bash/csh script setting up to use packages in the view + return + + +### Util functions def assuredir(path): 'Assure path exists as a directory' @@ -79,7 +128,6 @@ def relative_to(prefix, path): reldir = reldir[1:] return reldir - def transform_path(spec, path, prefix=None): 'Return the a relative path corresponding to given path spec.prefix' if os.path.isabs(path): @@ -92,66 +140,6 @@ def transform_path(spec, path, prefix=None): path = os.path.join(prefix, path) return path -def action_status(spec, prefix): - 'Check status of view in prefix against spec' - dotspack = os.path.join(prefix, '.spack', spec.name) - if os.path.exists(os.path.join(dotspack)): - tty.info("Package added: %s"%spec.name) - return - tty.info("Package missing: %s"%spec.name) - return - -def action_remove(spec, prefix): - 'Remove any files found in spec from prefix and purge empty directories.' - - if not os.path.exists(prefix): - return - - dotspack = transform_path(spec, '.spack', prefix) - if not os.path.exists(dotspack): - tty.info("Skipping nonexistent package %s"%spec.name) - return - - tty.info("remove %s"%spec.name) - for dirpath,dirnames,filenames in os.walk(spec.prefix): - if not filenames: - continue - - targdir = transform_path(spec, dirpath, prefix) - for fname in filenames: - src = os.path.join(dirpath, fname) - dst = os.path.join(targdir, fname) - if not os.path.exists(dst): - #tty.warn("Skipping nonexistent file for view: %s" % dst) - continue - os.unlink(dst) - -def action_link(spec, prefix): - 'Symlink all files in `spec` into directory `prefix`.' - - dotspack = transform_path(spec, '.spack', prefix) - if os.path.exists(dotspack): - tty.warn("Skipping previously added package %s"%spec.name) - return - - tty.info("link %s" % spec.name) - for dirpath,dirnames,filenames in os.walk(spec.prefix): - if not filenames: - continue # avoid explicitly making empty dirs - - targdir = transform_path(spec, dirpath, prefix) - assuredir(targdir) - - for fname in filenames: - src = os.path.join(dirpath, fname) - dst = os.path.join(targdir, fname) - if os.path.exists(dst): - if '.spack' in dst.split(os.path.sep): - continue # silence these - tty.warn("Skipping existing file for view: %s" % dst) - continue - os.symlink(src,dst) - def purge_empty_directories(path): 'Ascend up from the leaves accessible from `path` and remove empty directories.' for dirpath, subdirs, files in os.walk(path, topdown=False): @@ -160,58 +148,144 @@ def purge_empty_directories(path): try: os.rmdir(sdp) except OSError: - #tty.warn("Not removing directory with contents: %s" % sdp) pass - - - -def view_action(action, parser, args): - 'The view command parameterized by the action.' - to_exclude = [re.compile(e) for e in args.exclude] +def filter_exclude(specs, exclude): + 'Filter specs given sequence of exclude regex' + to_exclude = [re.compile(e) for e in exclude] def exclude(spec): for e in to_exclude: if e.match(spec.name): return True return False + return [s for s in specs if not exclude(s)] - specs = spack.cmd.parse_specs(args.specs, normalize=True, concretize=True) - if not specs: - parser.print_help() - return 1 - - prefix = args.prefix[0] - assuredir(prefix) - +def flatten(seeds, descend=True): + 'Normalize and flattend seed specs and descend hiearchy' flat = set() - for spec in specs: - if args.no_dependencies: + for spec in seeds: + if not descend: flat.add(spec) continue flat.update(spec.normalized().traverse()) + return flat - for spec in flat: - if exclude(spec): - tty.info('Skipping excluded package: "%s"' % spec.name) + +### Action-specific helpers + +def check_one(spec, path, verbose=False): + 'Check status of view in path against spec' + dotspack = os.path.join(path, '.spack', spec.name) + if os.path.exists(os.path.join(dotspack)): + tty.info('Package in view: "%s"'%spec.name) + return + tty.info('Package not in view: "%s"'%spec.name) + return + +def remove_one(spec, path, verbose=False): + 'Remove any files found in `spec` from `path` and purge empty directories.' + + if not os.path.exists(path): + return # done, short circuit + + dotspack = transform_path(spec, '.spack', path) + if not os.path.exists(dotspack): + if verbose: + tty.info('Skipping nonexistent package: "%s"'%spec.name) + return + + if verbose: + tty.info('Removing package: "%s"'%spec.name) + for dirpath,dirnames,filenames in os.walk(spec.prefix): + if not filenames: continue - if not os.path.exists(spec.prefix): - tty.warn('Skipping unknown package: %s in %s' % (spec.name, spec.prefix)) - continue - action(spec, prefix) + targdir = transform_path(spec, dirpath, path) + for fname in filenames: + dst = os.path.join(targdir, fname) + if not os.path.exists(dst): + continue + os.unlink(dst) - if args.action in ['remove','rm']: - purge_empty_directories(prefix) +def link_one(spec, path, link = os.symlink, verbose=False): + 'Link all files in `spec` into directory `path`.' + dotspack = transform_path(spec, '.spack', path) + if os.path.exists(dotspack): + tty.warn('Skipping existing package: "%s"'%spec.name) + return + + if verbose: + tty.info('Linking package: "%s"' % spec.name) + for dirpath,dirnames,filenames in os.walk(spec.prefix): + if not filenames: + continue # avoid explicitly making empty dirs + + targdir = transform_path(spec, dirpath, path) + assuredir(targdir) + + for fname in filenames: + src = os.path.join(dirpath, fname) + dst = os.path.join(targdir, fname) + if os.path.exists(dst): + if '.spack' in dst.split(os.path.sep): + continue # silence these + tty.warn("Skipping existing file: %s" % dst) + continue + link(src,dst) + + +### The canonically named visitor_* functions and their alias maps. +### One for each action. + +def visitor_symlink(specs, args): + 'Symlink all files found in specs' + path = args.path[0] + assuredir(path) + for spec in specs: + link_one(spec, path, verbose=args.verbose) +visitor_add = visitor_symlink +visitor_soft = visitor_symlink + +def visitor_hardlink(specs, args): + 'Hardlink all files found in specs' + path = args.path[0] + assuredir(path) + for spec in specs: + link_one(spec, path, os.link, verbose=args.verbose) +visitor_hard = visitor_hardlink + +def visitor_remove(specs, args): + 'Remove all files and directories found in specs from args.path' + path = args.path[0] + for spec in specs: + remove_one(spec, path, verbose=args.verbose) + purge_empty_directories(path) +visitor_rm = visitor_remove + +def visitor_statlink(specs, args): + 'Give status of view in args.path relative to specs' + path = args.path[0] + for spec in specs: + check_one(spec, path, verbose=args.verbose) +visitor_status = visitor_statlink +visitor_check = visitor_statlink + + +# Finally, the actual "view" command. There should be no need to +# modify anything below when new actions are added. def view(parser, args): - 'The view command.' - action = { - 'add': action_link, - 'link': action_link, - 'remove': action_remove, - 'rm': action_remove, - 'status': action_status, - 'check': action_status - }[args.action] - view_action(action, parser, args) + 'Produce a view of a set of packages.' + + # Process common args + seeds = [spack.cmd.disambiguate_spec(s) for s in args.specs] + specs = flatten(seeds, args.dependencies.lower() in ['yes','true']) + specs = filter_exclude(specs, args.exclude) + + # Execute the visitation. + try: + visitor = globals()['visitor_' + args.action] + except KeyError: + tty.error('Unknown action: "%s"' % args.action) + visitor(specs, args) diff --git a/share/spack/examples/test_view.sh b/share/spack/examples/test_view.sh new file mode 100755 index 0000000000..b3e7102192 --- /dev/null +++ b/share/spack/examples/test_view.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# This will install a few bogus/test packages in order to test the +# `spack view` command. It assumes you have "spack" in your path. + +# It makes sub-directories in your CWD and installs and uninstalls +# Spack packages named test-*. + +set -x +set -e + +view="spack -m view -v" +for variant in +nom ~nom+var +nom+var +do + spack -m uninstall -f -a -y test-d + spack -m install test-d$variant + testdir=test_view + rm -rf $testdir + echo "hardlink may fail if Spack install area and CWD are not same FS" + for action in symlink hardlink + do + $view --dependencies=no $action $testdir test-d + $view -e test-a -e test-b $action $testdir test-d + $view $action $testdir test-d + $view status $testdir test-d + $view -d false remove $testdir test-a + $view remove $testdir test-d + rmdir $testdir # should not fail + done +done +echo "Warnings about skipping existing in the above are okay" diff --git a/var/spack/repos/builtin.mock/packages/test-a/package.py b/var/spack/repos/builtin.mock/packages/test-a/package.py index 2f72370580..ff45344b7a 100644 --- a/var/spack/repos/builtin.mock/packages/test-a/package.py +++ b/var/spack/repos/builtin.mock/packages/test-a/package.py @@ -9,6 +9,7 @@ class TestA(Package): """The test-a package""" url = 'file://'+source + homepage = "http://www.example.com/" version('0.0', '4e823d0af4154fcf52b75dad41b7fd63') diff --git a/var/spack/repos/builtin.mock/packages/test-b/package.py b/var/spack/repos/builtin.mock/packages/test-b/package.py index db64b0a556..f6d30bb85f 100644 --- a/var/spack/repos/builtin.mock/packages/test-b/package.py +++ b/var/spack/repos/builtin.mock/packages/test-b/package.py @@ -9,6 +9,7 @@ class TestB(Package): """The test-b package""" url = 'file://'+source + homepage = "http://www.example.com/" version('0.0', '4e823d0af4154fcf52b75dad41b7fd63') diff --git a/var/spack/repos/builtin.mock/packages/test-c/package.py b/var/spack/repos/builtin.mock/packages/test-c/package.py index 0b03037466..641e46ee50 100644 --- a/var/spack/repos/builtin.mock/packages/test-c/package.py +++ b/var/spack/repos/builtin.mock/packages/test-c/package.py @@ -9,6 +9,7 @@ class TestC(Package): """The test-c package""" url = 'file://'+source + homepage = "http://www.example.com/" version('0.0', '4e823d0af4154fcf52b75dad41b7fd63') diff --git a/var/spack/repos/builtin.mock/packages/test-d/package.py b/var/spack/repos/builtin.mock/packages/test-d/package.py index 5cb7dcb2cb..b46c680edd 100644 --- a/var/spack/repos/builtin.mock/packages/test-d/package.py +++ b/var/spack/repos/builtin.mock/packages/test-d/package.py @@ -9,6 +9,7 @@ class TestD(Package): """The test-d package""" url = 'file://'+source + homepage = "http://www.example.com/" version('0.0', '4e823d0af4154fcf52b75dad41b7fd63') From f1900f6a7b080a6b45392d356749e2a98de46398 Mon Sep 17 00:00:00 2001 From: Brett Viren Date: Sun, 8 May 2016 10:25:21 -0400 Subject: [PATCH 07/13] Add a 'print' view allowing genreation of arbitrary strings based on format using package/spec parameters. --- lib/spack/spack/cmd/view.py | 64 +++++++++++++++++++++++++++++++++++-- 1 file changed, 61 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/cmd/view.py b/lib/spack/spack/cmd/view.py index 31937b39a2..b504dfd2b1 100644 --- a/lib/spack/spack/cmd/view.py +++ b/lib/spack/spack/cmd/view.py @@ -38,11 +38,12 @@ the union of the hierarchies of the installed packages in the DAG where installed files are referenced via symlinks. -- hardlink :: like the symlink view but hardlinks are used +- hardlink :: like the symlink view but hardlinks are used. - statlink :: a view producing a status report of a symlink or hardlink view. +- format :: a view printing one string per spec following a given format. The file system view concept is imspired by Nix, implemented by brett.viren@gmail.com ca 2016. @@ -64,6 +65,7 @@ import os import re +import sys import argparse import spack @@ -86,6 +88,9 @@ def setup_parser(sp): ssp = sp.add_subparsers(metavar='ACTION', dest='action') + specs_opts = dict(metavar='spec', nargs='+', + help="Seed specs of the packages to view.") + # The action parameterizes the command but in keeping with Spack # patterns we make it a subcommand. file_system_view_actions = [ @@ -102,9 +107,15 @@ def setup_parser(sp): for act in file_system_view_actions: act.add_argument('path', nargs=1, help="Path to file system view directory.") - act.add_argument('specs', metavar='spec', nargs='+', - help="Seed specs of the packages to view.") + act.add_argument('specs', **specs_opts) + # The formatted print action. + act = ssp.add_parser('print', + help="Print a string to stdout based on given format") + act.add_argument('format', nargs=1, + help="Format describing per-package printout.") + act.add_argument('specs', **specs_opts) + ## Other VIEW ACTIONS might be added here. ## Some ideas are the following (and some are redundant with existing cmds) ## A JSON view that dumps a DAG to a JSON file @@ -170,6 +181,39 @@ def flatten(seeds, descend=True): flat.update(spec.normalized().traverse()) return flat +def spec2dict(spec): + 'Convert info in a spec into a simple dictionary.' + + # Expclitly convert instead of just returning spec.__dict__ as + # some things need processing or are properties. + # + pkg = spec.package + ret = dict(name = spec.name, + short = spec.short_spec, + cshort = spec.cshort_spec, # color + root = spec.root, + prefix = spec.prefix, + version = spec.version, + variants = spec.variants, + namespace = spec.namespace, + compiler = spec.compiler, + architecture = spec.architecture, + dependencies = ','.join(spec.dependencies.keys()), + dependents = ','.join(spec.dependents.keys()), + external = spec.external or "False", + hash = spec.dag_hash(), + + # package related: + url = pkg.url, + stage = pkg.stage.path, + installed = pkg.installed, + installed_dependents = ','.join([s.name for s in pkg.installed_dependents]), + build_log = pkg.build_log_path, + rpath = ':'.join(pkg.rpath), + + # ... + ) + return ret ### Action-specific helpers @@ -270,6 +314,20 @@ def visitor_statlink(specs, args): visitor_status = visitor_statlink visitor_check = visitor_statlink +def visitor_print(specs, args): + 'Print a string for each spec using args.format.' + fmt = args.format[0] + for spec in specs: + kwds = spec2dict(spec) + try: + string = fmt.format(**kwds) + except KeyError: + tty.error("Format error, use keywords: %s" % (', '.join(kwds.keys()), )) + raise + # argparser escapes these + string = string.replace(r'\n', '\n').replace(r'\t', '\t') + sys.stdout.write(string) + # Finally, the actual "view" command. There should be no need to # modify anything below when new actions are added. From 8ddc1f89778335dda3d1dd213e75bc3a92d99712 Mon Sep 17 00:00:00 2001 From: Brett Viren Date: Sat, 14 May 2016 09:11:04 -0400 Subject: [PATCH 08/13] Move from str.format() to string.Template. --- lib/spack/spack/cmd/view.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/cmd/view.py b/lib/spack/spack/cmd/view.py index b504dfd2b1..caf14bfa53 100644 --- a/lib/spack/spack/cmd/view.py +++ b/lib/spack/spack/cmd/view.py @@ -317,16 +317,19 @@ def visitor_statlink(specs, args): def visitor_print(specs, args): 'Print a string for each spec using args.format.' fmt = args.format[0] + from string import Template + t = Template(fmt) + for spec in specs: kwds = spec2dict(spec) try: - string = fmt.format(**kwds) + text = t.substitute(kwds) except KeyError: tty.error("Format error, use keywords: %s" % (', '.join(kwds.keys()), )) raise # argparser escapes these - string = string.replace(r'\n', '\n').replace(r'\t', '\t') - sys.stdout.write(string) + text = text.replace(r'\n', '\n').replace(r'\t', '\t') + sys.stdout.write(text) # Finally, the actual "view" command. There should be no need to From 8078845273021dae01e3fdde2a5e3b10aea2f845 Mon Sep 17 00:00:00 2001 From: Brett Viren Date: Sat, 14 May 2016 09:11:41 -0400 Subject: [PATCH 09/13] Remove problematic tests, deal with this issue outside of PR #869. --- share/spack/examples/test_view.sh | 31 ---------------- .../builtin.mock/packages/test-a/package.py | 30 ---------------- .../packages/test-a/test-a-0.0.tar.gz | Bin 112 -> 0 bytes .../builtin.mock/packages/test-b/package.py | 32 ----------------- .../packages/test-b/test-b-0.0.tar.gz | Bin 112 -> 0 bytes .../builtin.mock/packages/test-c/package.py | 32 ----------------- .../packages/test-c/test-c-0.0.tar.gz | Bin 112 -> 0 bytes .../builtin.mock/packages/test-d/package.py | 33 ------------------ .../packages/test-d/test-d-0.0.tar.gz | Bin 112 -> 0 bytes 9 files changed, 158 deletions(-) delete mode 100755 share/spack/examples/test_view.sh delete mode 100644 var/spack/repos/builtin.mock/packages/test-a/package.py delete mode 100644 var/spack/repos/builtin.mock/packages/test-a/test-a-0.0.tar.gz delete mode 100644 var/spack/repos/builtin.mock/packages/test-b/package.py delete mode 100644 var/spack/repos/builtin.mock/packages/test-b/test-b-0.0.tar.gz delete mode 100644 var/spack/repos/builtin.mock/packages/test-c/package.py delete mode 100644 var/spack/repos/builtin.mock/packages/test-c/test-c-0.0.tar.gz delete mode 100644 var/spack/repos/builtin.mock/packages/test-d/package.py delete mode 100644 var/spack/repos/builtin.mock/packages/test-d/test-d-0.0.tar.gz diff --git a/share/spack/examples/test_view.sh b/share/spack/examples/test_view.sh deleted file mode 100755 index b3e7102192..0000000000 --- a/share/spack/examples/test_view.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -# This will install a few bogus/test packages in order to test the -# `spack view` command. It assumes you have "spack" in your path. - -# It makes sub-directories in your CWD and installs and uninstalls -# Spack packages named test-*. - -set -x -set -e - -view="spack -m view -v" -for variant in +nom ~nom+var +nom+var -do - spack -m uninstall -f -a -y test-d - spack -m install test-d$variant - testdir=test_view - rm -rf $testdir - echo "hardlink may fail if Spack install area and CWD are not same FS" - for action in symlink hardlink - do - $view --dependencies=no $action $testdir test-d - $view -e test-a -e test-b $action $testdir test-d - $view $action $testdir test-d - $view status $testdir test-d - $view -d false remove $testdir test-a - $view remove $testdir test-d - rmdir $testdir # should not fail - done -done -echo "Warnings about skipping existing in the above are okay" diff --git a/var/spack/repos/builtin.mock/packages/test-a/package.py b/var/spack/repos/builtin.mock/packages/test-a/package.py deleted file mode 100644 index ff45344b7a..0000000000 --- a/var/spack/repos/builtin.mock/packages/test-a/package.py +++ /dev/null @@ -1,30 +0,0 @@ -from spack import * - -import os - -mydir = os.path.dirname(__file__) -source = os.path.join(mydir,'test-a-0.0.tar.gz') - -class TestA(Package): - """The test-a package""" - - url = 'file://'+source - homepage = "http://www.example.com/" - - version('0.0', '4e823d0af4154fcf52b75dad41b7fd63') - - variant('nom', default=True, description='Nominal variant') - variant('var', default=False, description='Variant variant') - - def install(self, spec, prefix): - bindir = os.path.join(prefix,'bin') - os.makedirs(bindir) - script = os.path.join(bindir, 'test-a') - with open(script,'w') as fp: - fp.write("""#!/bin/bash -echo 'name: %s' -echo 'prefix: %s' -echo 'spec: %s' - """ % (spec.name, prefix, spec)) - - os.chmod(script, 0555) diff --git a/var/spack/repos/builtin.mock/packages/test-a/test-a-0.0.tar.gz b/var/spack/repos/builtin.mock/packages/test-a/test-a-0.0.tar.gz deleted file mode 100644 index 3588e05765cd799dfe19e1f77e057bf7508ec47e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 112 zcmb2|=3uzVq!!M={Pvt8-ysJPmJ7D0<~95|Kkb{1f}-PrX$3W18BI)~SE|#F^vw8N zv-i!f&C_|0CdssyKPx$LK4$72|C!Dyr^A#M>z4NF`mcRfZGSIIclq(}ch}1?z`(y3 OtXZo*ykgK`U;qF>c`_gX diff --git a/var/spack/repos/builtin.mock/packages/test-b/package.py b/var/spack/repos/builtin.mock/packages/test-b/package.py deleted file mode 100644 index f6d30bb85f..0000000000 --- a/var/spack/repos/builtin.mock/packages/test-b/package.py +++ /dev/null @@ -1,32 +0,0 @@ -from spack import * - -import os - -mydir = os.path.dirname(__file__) -source = os.path.join(mydir,'test-b-0.0.tar.gz') - -class TestB(Package): - """The test-b package""" - - url = 'file://'+source - homepage = "http://www.example.com/" - - version('0.0', '4e823d0af4154fcf52b75dad41b7fd63') - - variant('nom', default=True, description='Nominal variant') - variant('var', default=False, description='Variant variant') - - depends_on('test-a') - - def install(self, spec, prefix): - bindir = os.path.join(prefix,'bin') - os.makedirs(bindir) - script = os.path.join(bindir, 'test-b') - with open(script,'w') as fp: - fp.write("""#!/bin/bash -echo 'name: %s' -echo 'prefix: %s' -echo 'spec: %s' - """ % (spec.name, prefix, spec)) - - os.chmod(script, 0555) diff --git a/var/spack/repos/builtin.mock/packages/test-b/test-b-0.0.tar.gz b/var/spack/repos/builtin.mock/packages/test-b/test-b-0.0.tar.gz deleted file mode 100644 index 3588e05765cd799dfe19e1f77e057bf7508ec47e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 112 zcmb2|=3uzVq!!M={Pvt8-ysJPmJ7D0<~95|Kkb{1f}-PrX$3W18BI)~SE|#F^vw8N zv-i!f&C_|0CdssyKPx$LK4$72|C!Dyr^A#M>z4NF`mcRfZGSIIclq(}ch}1?z`(y3 OtXZo*ykgK`U;qF>c`_gX diff --git a/var/spack/repos/builtin.mock/packages/test-c/package.py b/var/spack/repos/builtin.mock/packages/test-c/package.py deleted file mode 100644 index 641e46ee50..0000000000 --- a/var/spack/repos/builtin.mock/packages/test-c/package.py +++ /dev/null @@ -1,32 +0,0 @@ -from spack import * - -import os - -mydir = os.path.dirname(__file__) -source = os.path.join(mydir,'test-c-0.0.tar.gz') - -class TestC(Package): - """The test-c package""" - - url = 'file://'+source - homepage = "http://www.example.com/" - - version('0.0', '4e823d0af4154fcf52b75dad41b7fd63') - - variant('nom', default=True, description='Nominal variant') - variant('var', default=False, description='Variant variant') - - depends_on('test-a+var',when='+var') - - def install(self, spec, prefix): - bindir = os.path.join(prefix,'bin') - os.makedirs(bindir) - script = os.path.join(bindir, 'test-c') - with open(script,'w') as fp: - fp.write("""#!/bin/bash -echo 'name: %s' -echo 'prefix: %s' -echo 'spec: %s' - """ % (spec.name, prefix, spec)) - - os.chmod(script, 0555) diff --git a/var/spack/repos/builtin.mock/packages/test-c/test-c-0.0.tar.gz b/var/spack/repos/builtin.mock/packages/test-c/test-c-0.0.tar.gz deleted file mode 100644 index 3588e05765cd799dfe19e1f77e057bf7508ec47e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 112 zcmb2|=3uzVq!!M={Pvt8-ysJPmJ7D0<~95|Kkb{1f}-PrX$3W18BI)~SE|#F^vw8N zv-i!f&C_|0CdssyKPx$LK4$72|C!Dyr^A#M>z4NF`mcRfZGSIIclq(}ch}1?z`(y3 OtXZo*ykgK`U;qF>c`_gX diff --git a/var/spack/repos/builtin.mock/packages/test-d/package.py b/var/spack/repos/builtin.mock/packages/test-d/package.py deleted file mode 100644 index b46c680edd..0000000000 --- a/var/spack/repos/builtin.mock/packages/test-d/package.py +++ /dev/null @@ -1,33 +0,0 @@ -from spack import * - -import os - -mydir = os.path.dirname(__file__) -source = os.path.join(mydir,'test-d-0.0.tar.gz') - -class TestD(Package): - """The test-d package""" - - url = 'file://'+source - homepage = "http://www.example.com/" - - version('0.0', '4e823d0af4154fcf52b75dad41b7fd63') - - variant('nom', default=True, description='Nominal variant') - variant('var', default=False, description='Variant variant') - - depends_on('test-b') - depends_on('test-c') - - def install(self, spec, prefix): - bindir = os.path.join(prefix,'bin') - os.makedirs(bindir) - script = os.path.join(bindir, 'test-d') - with open(script,'w') as fp: - fp.write("""#!/bin/bash -echo 'name: %s' -echo 'prefix: %s' -echo 'spec: %s' - """ % (spec.name, prefix, spec)) - - os.chmod(script, 0555) diff --git a/var/spack/repos/builtin.mock/packages/test-d/test-d-0.0.tar.gz b/var/spack/repos/builtin.mock/packages/test-d/test-d-0.0.tar.gz deleted file mode 100644 index 3588e05765cd799dfe19e1f77e057bf7508ec47e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 112 zcmb2|=3uzVq!!M={Pvt8-ysJPmJ7D0<~95|Kkb{1f}-PrX$3W18BI)~SE|#F^vw8N zv-i!f&C_|0CdssyKPx$LK4$72|C!Dyr^A#M>z4NF`mcRfZGSIIclq(}ch}1?z`(y3 OtXZo*ykgK`U;qF>c`_gX From c98afe2205ec90bc98c7f1c33b3e8e05403c7c22 Mon Sep 17 00:00:00 2001 From: Brett Viren Date: Sat, 14 May 2016 13:14:51 -0400 Subject: [PATCH 10/13] Pick a better name for the 'spec' parameters. --- lib/spack/spack/cmd/view.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/cmd/view.py b/lib/spack/spack/cmd/view.py index caf14bfa53..8767459c08 100644 --- a/lib/spack/spack/cmd/view.py +++ b/lib/spack/spack/cmd/view.py @@ -189,8 +189,8 @@ def spec2dict(spec): # pkg = spec.package ret = dict(name = spec.name, - short = spec.short_spec, - cshort = spec.cshort_spec, # color + spec = spec.short_spec, + colorspec = spec.cshort_spec, # color root = spec.root, prefix = spec.prefix, version = spec.version, From b3ede099e2d2dd684ebe478dd711a979ebb3bf98 Mon Sep 17 00:00:00 2001 From: Brett Viren Date: Sat, 14 May 2016 13:15:25 -0400 Subject: [PATCH 11/13] Document command. --- lib/spack/docs/basic_usage.rst | 233 +++++++++++++++++++++++++++++++++ 1 file changed, 233 insertions(+) diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst index 29791d98c4..e340881a93 100644 --- a/lib/spack/docs/basic_usage.rst +++ b/lib/spack/docs/basic_usage.rst @@ -330,6 +330,125 @@ of libelf would look like this: The full spec syntax is discussed in detail in :ref:`sec-specs`. +``spack view print`` +~~~~~~~~~~~~~~~~~~~~~~ + +``spack view print`` is a subcommand of ``spack view`` which displays information about one or more installed packages and their dependencies using a user-provided format string. The string can reverence variables in a shell-like manner for example ``$variable`` or ``${variable}text``. It can also include ``\t`` for tabs and ``\n`` for new lines. + +Some of the supported variables are: + +``name`` + The package name. + +``version`` + The package version. + +``spec`` + The package specification. + +``root`` + The root specification. + +``prefix`` + The installation directory. + +``variants`` + The collection of variants, if any. + +``namespace`` + The package repository name space. + +``compiler`` + The compiler \`name@versoin\` used to build the package. + +``architecture`` + The architecture targeted by the compiler. + +``dependencies`` + A comma-separated list of names of packages on which the package depends. + +``dependents`` + A comma-separated list of names of packages which depend on the package. + +``hash`` + The Spack hash for the package. + +``url`` + The source URL for the package. + +``stage`` + The directory for staging the build. + +``build_log`` + The path to the build log file. + +``rpath`` + The colon-separated library \`RPATH\` used in building the package. + +Here are some example uses of `spack view print`. A simple line-oriented report of information can be produced: + +.. code-block:: sh + + $ spack view print '$name\t$version\t$hash\n' cmake@3.5.2 + ncurses 6.0 bvbu4ixbnvtodpik4qzljlx3ukpyfrcz + zlib 1.2.8 ckki7zlryxrsetfqkgoxxahlhqqjni7n + openssl 1.0.2g 6zbar63sciso253nptxyrnhupymo7oyi + cmake 3.5.2 wprvmoczkpw4tiy5ybuk5zr7saus2d7g + +There are better ways to do this but a slow-and-dirty shell init procedure can be formed: + +.. code-block:: sh + + $ spack view print 'export PATH="${prefix}/bin:$$PATH"\n' m4 + export PATH="/spack/opt/spack/linux-x86_64/gcc-5.2.1/libsigsegv-2.10-h6hsv76hffcjoe3nsaihzxemniwiedu2/bin:$PATH" + export PATH="/spack/opt/spack/linux-x86_64/gcc-5.2.1/m4-1.4.17-a4ikhddnk2zonr66mbwaqb226uhigcrp/bin:$PATH" + $ eval $(spack view print 'export PATH="${prefix}/bin:$$PATH"\n' m4) + +Or, maybe you have some external application that can chew on Spack data in which case you can dump it to some easily parsed markup syntax such as YAML: + +.. code-block:: sh + + + $ spack view print '${name}:\n - version: ${version}\n - url : ${url}\n - spec: ${spec}\n - prefix: ${prefix}\n - root : ${root}\n - stage : ${stage}\n - log: ${build_log}\n' m4 + +Which might produce something like: + +.. code-block:: yaml + + libsigsegv: + - version: 2.10 + - url : ftp://ftp.gnu.org/gnu/libsigsegv/libsigsegv-2.10.tar.gz + - spec: libsigsegv@2.10%gcc@5.2.1=linux-x86_64-h6hsv76 + - prefix: /spack/opt/spack/linux-x86_64/gcc-5.2.1/libsigsegv-2.10-h6hsv76hffcjoe3nsaihzxemniwiedu2 + - root : m4@1.4.17%gcc@5.2.1+sigsegv=linux-x86_64^libsigsegv@2.10%gcc@5.2.1=linux-x86_64 + - stage : /spack/var/spack/stage/libsigsegv-2.10-h6hsv76hffcjoe3nsaihzxemniwiedu2 + - log: /spack/opt/spack/linux-x86_64/gcc-5.2.1/libsigsegv-2.10-h6hsv76hffcjoe3nsaihzxemniwiedu2/.spack/build.out + m4: + - version: 1.4.17 + - url : ftp://ftp.gnu.org/gnu/m4/m4-1.4.17.tar.gz + - spec: m4@1.4.17%gcc@5.2.1+sigsegv=linux-x86_64-a4ikhdd + - prefix: /spack/opt/spack/linux-x86_64/gcc-5.2.1/m4-1.4.17-a4ikhddnk2zonr66mbwaqb226uhigcrp + - root : m4@1.4.17%gcc@5.2.1+sigsegv=linux-x86_64^libsigsegv@2.10%gcc@5.2.1=linux-x86_64 + - stage : /spack/var/spack/stage/m4-1.4.17-a4ikhddnk2zonr66mbwaqb226uhigcrp + - log: /spack/opt/spack/linux-x86_64/gcc-5.2.1/m4-1.4.17-a4ikhddnk2zonr66mbwaqb226uhigcrp/.spack/build.out + +Or, maybe you want to do something with information about package dependencies by generating some ready-to-import Python code holding the tree: + +.. code-block:: sh + + $ spack view print '$name = dict(name = "$name", parents = [$dependencies], children = [$dependents])\n' cmake@3.5.2 + +Producing this Python code + +.. code-block:: python + + ncurses = dict(name = "ncurses", parents = []]) + zlib = dict(name = "zlib", parents = []]) + openssl = dict(name = "openssl", parents = [zlib]]) + cmake = dict(name = "cmake", parents = [ncurses,openssl]]) + + + Compiler configuration ----------------------------------- @@ -1035,6 +1154,120 @@ regenerate all module and dotkit files from scratch: .. _extensions: +Filesystem Views +------------------------------- + +.. Maybe this is not the right location for this documentation. + +The Spack installation area allows for many package installation trees +to coexist and gives the user choices as to what versions and variants +of packages to use. To use them, the user must rely on a way to +aggregate a subset of those packages. The section on Environment +Modules gives one good way to do that which relies on setting various +environment variables. An alternative way to aggregate is through +**filesystem views**. + +A filesystem view is a single directory tree which is the union of the +directory hierarchies of the individual package installation trees +that have been included. The files of the view's installed packages +are brought into the view by symbolic or hard links back to their +location in the original Spack installation area. As the view is +formed, any clashes due to a file having the exact same path in its +package installation tree are handled in a first-come-first-served +basis and a warning is printed. Packages and their dependencies can +be both added and removed. During removal, empty directories will be +purged. These operations can be limited to pertain to just the +packages listed by the user or to exclude specific dependencies and +they allow for software installed outside of Spack to coexist inside +the filesystem view tree. + +By its nature, a filesystem view represents a particular choice of one +set of packages among all the versions and variants that are available +in the Spack installation area. It is thus equivalent to the +directory hiearchy that might exist under ``/usr/local``. While this +limits a view to including only one version/variant of any package, it +provides the benefits of having a simpler and traditional layout which +may be used without any particular knowledge that its packages were +built by Spack. + +Views can be used for a variety of purposes including: + +- A central installation in a traditional layout, eg ``/usr/local`` maintained over time by the sysadmin. +- A self-contained installation area which may for the basis of a top-level atomic versioning scheme, eg ``/opt/pro`` vs ``/opt/dev``. +- Providing an atomic and monolithic binary distribution, eg for delivery as a single tarball. +- Producing ephemeral testing or developing environments. + +Using Filesystem Views +~~~~~~~~~~~~~~~~~~~~~~ + +A filesystem view is created and packages are linked in by the ``spack +view`` command's ``symlink`` and ``hardlink`` sub-commands. The +``spack view remove`` command can be used to unlink some or all of the +filesystem view. + +The following example creates a filesystem view based +on an installed ``cmake`` package and then removes from the view the +files in the ``cmake`` package while retaining its dependencies. + +.. code-block:: sh + + + $ spack view -v symlink myview cmake@3.5.2 + ==> Linking package: "ncurses" + ==> Linking package: "zlib" + ==> Linking package: "openssl" + ==> Linking package: "cmake" + + $ ls myview/ + bin doc etc include lib share + + $ ls myview/bin/ + captoinfo clear cpack ctest infotocap openssl tabs toe tset + ccmake cmake c_rehash infocmp ncurses6-config reset tic tput + + $ spack view -v -d false rm myview cmake@3.5.2 + ==> Removing package: "cmake" + + $ ls myview/bin/ + captoinfo c_rehash infotocap openssl tabs toe tset + clear infocmp ncurses6-config reset tic tput + + +Limitations of Filesystem Views +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This section describes some limitations that should be considered in +using filesystems views. + +Filesystem views are merely organizational. The binary executable +programs, shared libraries and other build products found in a view +are mere links into the "real" Spack installation area. If a view is +built with symbolic links it requires the Spack-installed package to +be kept in place. Building a view with hardlinks removes this +requirement but any internal paths (eg, rpath or ``#!`` interpreter +specifications) will still require the Spack-installed package files +to be in place. + +.. FIXME: reference the relocation work of Hegner and Gartung. + +As described above, when a view is built only a single instance of a +file may exist in the unified filesystem tree. If more than one +package provides a file at the same path (relative to its own root) +then it is the first package added to the view that "wins". A warning +is printed and it is up to the user to determine if the conflict +matters. + +It is up to the user to assure a consistent view is produced. In +particular if the user excludes packages, limits the following of +dependencies or removes packages the view may become inconsistent. In +particular, if two packages require the same sub-tree of dependencies, +removing one package (recursively) will remove its dependencies and +leave the other package broken. + + + + + Extensions & Python support ------------------------------------ From 77a34ebdf94dd39be338ab85deb45e564c0a563b Mon Sep 17 00:00:00 2001 From: Brett Viren Date: Mon, 16 May 2016 10:08:59 -0400 Subject: [PATCH 12/13] flake8-clean. --- lib/spack/spack/cmd/view.py | 159 +++++++++++++++++++----------------- 1 file changed, 84 insertions(+), 75 deletions(-) diff --git a/lib/spack/spack/cmd/view.py b/lib/spack/spack/cmd/view.py index 8767459c08..4861383eaa 100644 --- a/lib/spack/spack/cmd/view.py +++ b/lib/spack/spack/cmd/view.py @@ -29,14 +29,17 @@ This set consists of: - specs resolved from the package names given by the user (the seeds) + - all depenencies of the seeds unless user specifies `--no-depenencies` -- less any specs with names matching the regular expressions given by `--exclude` + +- less any specs with names matching the regular expressions given by + `--exclude` The `view` command provides a number of functions (the "actions"): - symlink :: a file system view which is a directory hierarchy that is the union of the hierarchies of the installed packages in the DAG - where installed files are referenced via symlinks. + where installed files are referenced via symlinks. - hardlink :: like the symlink view but hardlinks are used. @@ -66,71 +69,70 @@ import os import re import sys -import argparse - import spack import spack.cmd import llnl.util.tty as tty description = "Produce a single-rooted directory view of a spec." + def setup_parser(sp): setup_parser.parser = sp - sp.add_argument('-v','--verbose', action='store_true', default=False, - help="Display verbose output.") - sp.add_argument('-e','--exclude', action='append', default=[], - help="Exclude packages with names matching the given regex pattern.") - sp.add_argument('-d', '--dependencies', choices=['true','false','yes','no'], - default='true', - help="Follow dependencies.") - + sp.add_argument( + '-v', '--verbose', action='store_true', default=False, + help="Display verbose output.") + sp.add_argument( + '-e', '--exclude', action='append', default=[], + help="Exclude packages with names matching the given regex pattern.") + sp.add_argument( + '-d', '--dependencies', choices=['true', 'false', 'yes', 'no'], + default='true', + help="Follow dependencies.") ssp = sp.add_subparsers(metavar='ACTION', dest='action') - specs_opts = dict(metavar='spec', nargs='+', + specs_opts = dict(metavar='spec', nargs='+', help="Seed specs of the packages to view.") # The action parameterizes the command but in keeping with Spack # patterns we make it a subcommand. file_system_view_actions = [ - ssp.add_parser('symlink', aliases=['add','soft'], - help='Add package files to a filesystem view via symbolic links.'), - ssp.add_parser('hardlink', aliases=['hard'], - help='Add packages files to a filesystem via via hard links.'), - ssp.add_parser('remove', aliases=['rm'], - help='Remove packages from a filesystem view.'), - ssp.add_parser('statlink', aliases=['status','check'], - help='Check status of packages in a filesystem view.') + ssp.add_parser( + 'symlink', aliases=['add', 'soft'], + help='Add package files to a filesystem view via symbolic links.'), + ssp.add_parser( + 'hardlink', aliases=['hard'], + help='Add packages files to a filesystem via via hard links.'), + ssp.add_parser( + 'remove', aliases=['rm'], + help='Remove packages from a filesystem view.'), + ssp.add_parser( + 'statlink', aliases=['status', 'check'], + help='Check status of packages in a filesystem view.') ] # All these options and arguments are common to every action. for act in file_system_view_actions: act.add_argument('path', nargs=1, help="Path to file system view directory.") act.add_argument('specs', **specs_opts) - + # The formatted print action. act = ssp.add_parser('print', help="Print a string to stdout based on given format") - act.add_argument('format', nargs=1, + act.add_argument('format', nargs=1, help="Format describing per-package printout.") act.add_argument('specs', **specs_opts) - ## Other VIEW ACTIONS might be added here. - ## Some ideas are the following (and some are redundant with existing cmds) - ## A JSON view that dumps a DAG to a JSON file - ## A DOT view that dumps to a GraphViz file - ## A SHELL INIT view that dumps bash/csh script setting up to use packages in the view return -### Util functions - def assuredir(path): 'Assure path exists as a directory' if not os.path.exists(path): os.makedirs(path) - + + def relative_to(prefix, path): 'Return end of `path` relative to `prefix`' assert 0 == path.find(prefix) @@ -139,31 +141,36 @@ def relative_to(prefix, path): reldir = reldir[1:] return reldir + def transform_path(spec, path, prefix=None): 'Return the a relative path corresponding to given path spec.prefix' if os.path.isabs(path): path = relative_to(spec.prefix, path) subdirs = path.split(os.path.sep) if subdirs[0] == '.spack': - lst = ['.spack',spec.name]+subdirs[1:] + lst = ['.spack', spec.name] + subdirs[1:] path = os.path.join(*lst) if prefix: path = os.path.join(prefix, path) return path + def purge_empty_directories(path): - 'Ascend up from the leaves accessible from `path` and remove empty directories.' + '''Ascend up from the leaves accessible from `path` + and remove empty directories.''' for dirpath, subdirs, files in os.walk(path, topdown=False): for sd in subdirs: - sdp = os.path.join(dirpath,sd) + sdp = os.path.join(dirpath, sd) try: os.rmdir(sdp) except OSError: pass + def filter_exclude(specs, exclude): 'Filter specs given sequence of exclude regex' to_exclude = [re.compile(e) for e in exclude] + def exclude(spec): for e in to_exclude: if e.match(spec.name): @@ -171,6 +178,7 @@ def exclude(spec): return False return [s for s in specs if not exclude(s)] + def flatten(seeds, descend=True): 'Normalize and flattend seed specs and descend hiearchy' flat = set() @@ -181,6 +189,7 @@ def flatten(seeds, descend=True): flat.update(spec.normalized().traverse()) return flat + def spec2dict(spec): 'Convert info in a spec into a simple dictionary.' @@ -188,44 +197,45 @@ def spec2dict(spec): # some things need processing or are properties. # pkg = spec.package - ret = dict(name = spec.name, - spec = spec.short_spec, - colorspec = spec.cshort_spec, # color - root = spec.root, - prefix = spec.prefix, - version = spec.version, - variants = spec.variants, - namespace = spec.namespace, - compiler = spec.compiler, - architecture = spec.architecture, - dependencies = ','.join(spec.dependencies.keys()), - dependents = ','.join(spec.dependents.keys()), - external = spec.external or "False", - hash = spec.dag_hash(), + inst_deps = ','.join([s.name for s in pkg.installed_dependents]), + ret = dict(name=spec.name, + spec=spec.short_spec, + colorspec=spec.cshort_spec, # color + root=spec.root, + prefix=spec.prefix, + version=spec.version, + variants=spec.variants, + namespace=spec.namespace, + compiler=spec.compiler, + architecture=spec.architecture, + dependencies=','.join(spec.dependencies.keys()), + dependents=','.join(spec.dependents.keys()), + external=spec.external or "False", + hash=spec.dag_hash(), # package related: - url = pkg.url, - stage = pkg.stage.path, - installed = pkg.installed, - installed_dependents = ','.join([s.name for s in pkg.installed_dependents]), - build_log = pkg.build_log_path, - rpath = ':'.join(pkg.rpath), + url=pkg.url, + stage=pkg.stage.path, + installed=pkg.installed, + installed_dependents=inst_deps, + build_log=pkg.build_log_path, + rpath=':'.join(pkg.rpath), # ... ) return ret -### Action-specific helpers def check_one(spec, path, verbose=False): 'Check status of view in path against spec' dotspack = os.path.join(path, '.spack', spec.name) if os.path.exists(os.path.join(dotspack)): - tty.info('Package in view: "%s"'%spec.name) + tty.info('Package in view: "%s"' % spec.name) return - tty.info('Package not in view: "%s"'%spec.name) + tty.info('Package not in view: "%s"' % spec.name) return + def remove_one(spec, path, verbose=False): 'Remove any files found in `spec` from `path` and purge empty directories.' @@ -235,12 +245,12 @@ def remove_one(spec, path, verbose=False): dotspack = transform_path(spec, '.spack', path) if not os.path.exists(dotspack): if verbose: - tty.info('Skipping nonexistent package: "%s"'%spec.name) + tty.info('Skipping nonexistent package: "%s"' % spec.name) return if verbose: - tty.info('Removing package: "%s"'%spec.name) - for dirpath,dirnames,filenames in os.walk(spec.prefix): + tty.info('Removing package: "%s"' % spec.name) + for dirpath, dirnames, filenames in os.walk(spec.prefix): if not filenames: continue targdir = transform_path(spec, dirpath, path) @@ -250,17 +260,18 @@ def remove_one(spec, path, verbose=False): continue os.unlink(dst) -def link_one(spec, path, link = os.symlink, verbose=False): + +def link_one(spec, path, link=os.symlink, verbose=False): 'Link all files in `spec` into directory `path`.' dotspack = transform_path(spec, '.spack', path) if os.path.exists(dotspack): - tty.warn('Skipping existing package: "%s"'%spec.name) + tty.warn('Skipping existing package: "%s"' % spec.name) return if verbose: tty.info('Linking package: "%s"' % spec.name) - for dirpath,dirnames,filenames in os.walk(spec.prefix): + for dirpath, dirnames, filenames in os.walk(spec.prefix): if not filenames: continue # avoid explicitly making empty dirs @@ -275,12 +286,9 @@ def link_one(spec, path, link = os.symlink, verbose=False): continue # silence these tty.warn("Skipping existing file: %s" % dst) continue - link(src,dst) + link(src, dst) -### The canonically named visitor_* functions and their alias maps. -### One for each action. - def visitor_symlink(specs, args): 'Symlink all files found in specs' path = args.path[0] @@ -290,6 +298,7 @@ def visitor_symlink(specs, args): visitor_add = visitor_symlink visitor_soft = visitor_symlink + def visitor_hardlink(specs, args): 'Hardlink all files found in specs' path = args.path[0] @@ -298,6 +307,7 @@ def visitor_hardlink(specs, args): link_one(spec, path, os.link, verbose=args.verbose) visitor_hard = visitor_hardlink + def visitor_remove(specs, args): 'Remove all files and directories found in specs from args.path' path = args.path[0] @@ -306,6 +316,7 @@ def visitor_remove(specs, args): purge_empty_directories(path) visitor_rm = visitor_remove + def visitor_statlink(specs, args): 'Give status of view in args.path relative to specs' path = args.path[0] @@ -314,6 +325,7 @@ def visitor_statlink(specs, args): visitor_status = visitor_statlink visitor_check = visitor_statlink + def visitor_print(specs, args): 'Print a string for each spec using args.format.' fmt = args.format[0] @@ -325,25 +337,22 @@ def visitor_print(specs, args): try: text = t.substitute(kwds) except KeyError: - tty.error("Format error, use keywords: %s" % (', '.join(kwds.keys()), )) + tty.error("Format error, use keywords: %s" % + (', '.join(kwds.keys()), )) raise # argparser escapes these text = text.replace(r'\n', '\n').replace(r'\t', '\t') sys.stdout.write(text) -# Finally, the actual "view" command. There should be no need to -# modify anything below when new actions are added. - - def view(parser, args): 'Produce a view of a set of packages.' - # Process common args + # Process common args seeds = [spack.cmd.disambiguate_spec(s) for s in args.specs] - specs = flatten(seeds, args.dependencies.lower() in ['yes','true']) + specs = flatten(seeds, args.dependencies.lower() in ['yes', 'true']) specs = filter_exclude(specs, args.exclude) - + # Execute the visitation. try: visitor = globals()['visitor_' + args.action] From 126fc10b9dc56a51eb75bb7cfb88619b7029097c Mon Sep 17 00:00:00 2001 From: Brett Viren Date: Thu, 19 May 2016 08:35:54 -0600 Subject: [PATCH 13/13] Remove `view print` for now. --- lib/spack/docs/basic_usage.rst | 118 --------------------------------- lib/spack/spack/cmd/view.py | 72 +------------------- 2 files changed, 3 insertions(+), 187 deletions(-) diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst index 4af4387846..e108e393d7 100644 --- a/lib/spack/docs/basic_usage.rst +++ b/lib/spack/docs/basic_usage.rst @@ -330,124 +330,6 @@ of libelf would look like this: The full spec syntax is discussed in detail in :ref:`sec-specs`. -``spack view print`` -~~~~~~~~~~~~~~~~~~~~~~ - -``spack view print`` is a subcommand of ``spack view`` which displays information about one or more installed packages and their dependencies using a user-provided format string. The string can reverence variables in a shell-like manner for example ``$variable`` or ``${variable}text``. It can also include ``\t`` for tabs and ``\n`` for new lines. - -Some of the supported variables are: - -``name`` - The package name. - -``version`` - The package version. - -``spec`` - The package specification. - -``root`` - The root specification. - -``prefix`` - The installation directory. - -``variants`` - The collection of variants, if any. - -``namespace`` - The package repository name space. - -``compiler`` - The compiler \`name@versoin\` used to build the package. - -``architecture`` - The architecture targeted by the compiler. - -``dependencies`` - A comma-separated list of names of packages on which the package depends. - -``dependents`` - A comma-separated list of names of packages which depend on the package. - -``hash`` - The Spack hash for the package. - -``url`` - The source URL for the package. - -``stage`` - The directory for staging the build. - -``build_log`` - The path to the build log file. - -``rpath`` - The colon-separated library \`RPATH\` used in building the package. - -Here are some example uses of `spack view print`. A simple line-oriented report of information can be produced: - -.. code-block:: sh - - $ spack view print '$name\t$version\t$hash\n' cmake@3.5.2 - ncurses 6.0 bvbu4ixbnvtodpik4qzljlx3ukpyfrcz - zlib 1.2.8 ckki7zlryxrsetfqkgoxxahlhqqjni7n - openssl 1.0.2g 6zbar63sciso253nptxyrnhupymo7oyi - cmake 3.5.2 wprvmoczkpw4tiy5ybuk5zr7saus2d7g - -There are better ways to do this but a slow-and-dirty shell init procedure can be formed: - -.. code-block:: sh - - $ spack view print 'export PATH="${prefix}/bin:$$PATH"\n' m4 - export PATH="/spack/opt/spack/linux-x86_64/gcc-5.2.1/libsigsegv-2.10-h6hsv76hffcjoe3nsaihzxemniwiedu2/bin:$PATH" - export PATH="/spack/opt/spack/linux-x86_64/gcc-5.2.1/m4-1.4.17-a4ikhddnk2zonr66mbwaqb226uhigcrp/bin:$PATH" - $ eval $(spack view print 'export PATH="${prefix}/bin:$$PATH"\n' m4) - -Or, maybe you have some external application that can chew on Spack data in which case you can dump it to some easily parsed markup syntax such as YAML: - -.. code-block:: sh - - - $ spack view print '${name}:\n - version: ${version}\n - url : ${url}\n - spec: ${spec}\n - prefix: ${prefix}\n - root : ${root}\n - stage : ${stage}\n - log: ${build_log}\n' m4 - -Which might produce something like: - -.. code-block:: yaml - - libsigsegv: - - version: 2.10 - - url : ftp://ftp.gnu.org/gnu/libsigsegv/libsigsegv-2.10.tar.gz - - spec: libsigsegv@2.10%gcc@5.2.1=linux-x86_64-h6hsv76 - - prefix: /spack/opt/spack/linux-x86_64/gcc-5.2.1/libsigsegv-2.10-h6hsv76hffcjoe3nsaihzxemniwiedu2 - - root : m4@1.4.17%gcc@5.2.1+sigsegv=linux-x86_64^libsigsegv@2.10%gcc@5.2.1=linux-x86_64 - - stage : /spack/var/spack/stage/libsigsegv-2.10-h6hsv76hffcjoe3nsaihzxemniwiedu2 - - log: /spack/opt/spack/linux-x86_64/gcc-5.2.1/libsigsegv-2.10-h6hsv76hffcjoe3nsaihzxemniwiedu2/.spack/build.out - m4: - - version: 1.4.17 - - url : ftp://ftp.gnu.org/gnu/m4/m4-1.4.17.tar.gz - - spec: m4@1.4.17%gcc@5.2.1+sigsegv=linux-x86_64-a4ikhdd - - prefix: /spack/opt/spack/linux-x86_64/gcc-5.2.1/m4-1.4.17-a4ikhddnk2zonr66mbwaqb226uhigcrp - - root : m4@1.4.17%gcc@5.2.1+sigsegv=linux-x86_64^libsigsegv@2.10%gcc@5.2.1=linux-x86_64 - - stage : /spack/var/spack/stage/m4-1.4.17-a4ikhddnk2zonr66mbwaqb226uhigcrp - - log: /spack/opt/spack/linux-x86_64/gcc-5.2.1/m4-1.4.17-a4ikhddnk2zonr66mbwaqb226uhigcrp/.spack/build.out - -Or, maybe you want to do something with information about package dependencies by generating some ready-to-import Python code holding the tree: - -.. code-block:: sh - - $ spack view print '$name = dict(name = "$name", parents = [$dependencies], children = [$dependents])\n' cmake@3.5.2 - -Producing this Python code - -.. code-block:: python - - ncurses = dict(name = "ncurses", parents = []]) - zlib = dict(name = "zlib", parents = []]) - openssl = dict(name = "openssl", parents = [zlib]]) - cmake = dict(name = "cmake", parents = [ncurses,openssl]]) - - Compiler configuration diff --git a/lib/spack/spack/cmd/view.py b/lib/spack/spack/cmd/view.py index 4861383eaa..8f1fc9be74 100644 --- a/lib/spack/spack/cmd/view.py +++ b/lib/spack/spack/cmd/view.py @@ -24,9 +24,8 @@ ############################################################################## '''Produce a "view" of a Spack DAG. -A "view" is the product of applying a function on a set of package specs. - -This set consists of: +A "view" is file hierarchy representing the union of a number of +Spack-installed package file hierarchies. The union is formed from: - specs resolved from the package names given by the user (the seeds) @@ -35,7 +34,7 @@ - less any specs with names matching the regular expressions given by `--exclude` -The `view` command provides a number of functions (the "actions"): +The `view` can be built and tore down via a number of methods (the "actions"): - symlink :: a file system view which is a directory hierarchy that is the union of the hierarchies of the installed packages in the DAG @@ -46,8 +45,6 @@ - statlink :: a view producing a status report of a symlink or hardlink view. -- format :: a view printing one string per spec following a given format. - The file system view concept is imspired by Nix, implemented by brett.viren@gmail.com ca 2016. @@ -68,7 +65,6 @@ import os import re -import sys import spack import spack.cmd import llnl.util.tty as tty @@ -117,13 +113,6 @@ def setup_parser(sp): help="Path to file system view directory.") act.add_argument('specs', **specs_opts) - # The formatted print action. - act = ssp.add_parser('print', - help="Print a string to stdout based on given format") - act.add_argument('format', nargs=1, - help="Format describing per-package printout.") - act.add_argument('specs', **specs_opts) - return @@ -190,42 +179,6 @@ def flatten(seeds, descend=True): return flat -def spec2dict(spec): - 'Convert info in a spec into a simple dictionary.' - - # Expclitly convert instead of just returning spec.__dict__ as - # some things need processing or are properties. - # - pkg = spec.package - inst_deps = ','.join([s.name for s in pkg.installed_dependents]), - ret = dict(name=spec.name, - spec=spec.short_spec, - colorspec=spec.cshort_spec, # color - root=spec.root, - prefix=spec.prefix, - version=spec.version, - variants=spec.variants, - namespace=spec.namespace, - compiler=spec.compiler, - architecture=spec.architecture, - dependencies=','.join(spec.dependencies.keys()), - dependents=','.join(spec.dependents.keys()), - external=spec.external or "False", - hash=spec.dag_hash(), - - # package related: - url=pkg.url, - stage=pkg.stage.path, - installed=pkg.installed, - installed_dependents=inst_deps, - build_log=pkg.build_log_path, - rpath=':'.join(pkg.rpath), - - # ... - ) - return ret - - def check_one(spec, path, verbose=False): 'Check status of view in path against spec' dotspack = os.path.join(path, '.spack', spec.name) @@ -326,25 +279,6 @@ def visitor_statlink(specs, args): visitor_check = visitor_statlink -def visitor_print(specs, args): - 'Print a string for each spec using args.format.' - fmt = args.format[0] - from string import Template - t = Template(fmt) - - for spec in specs: - kwds = spec2dict(spec) - try: - text = t.substitute(kwds) - except KeyError: - tty.error("Format error, use keywords: %s" % - (', '.join(kwds.keys()), )) - raise - # argparser escapes these - text = text.replace(r'\n', '\n').replace(r'\t', '\t') - sys.stdout.write(text) - - def view(parser, args): 'Produce a view of a set of packages.'