commands: add --update option to spack list

- Add a `--update FILE` option to `spack list`
- Output is written to the file only if any package is newer than the file
- Simplify the code in docs/conf.py using this new option

lib/spack/docs/conf.py

@@ -54,8 +54,8 @@
 os.environ['COLUMNS'] = '120'
 
 # Generate full package list if needed
-subprocess.Popen(
+subprocess.call([
-    ['spack', 'list', '--format=html', '--update=package_list.html'])
+    'spack', 'list', '--format=html', '--update=package_list.html'])
 
 # Generate a command index if an update is needed
 subprocess.call([

lib/spack/spack/cmd/list.py

@@ -9,6 +9,7 @@
 import argparse
 import cgi
 import fnmatch
+import os
 import re
 import sys
 import math
@@ -46,6 +47,9 @@ def setup_parser(subparser):
     subparser.add_argument(
         '--format', default='name_only', choices=formatters,
         help='format to be used to print the output [default: name_only]')
+    subparser.add_argument(
+        '--update', metavar='FILE', default=None, action='store',
+        help='write output to the specified file, if any package is newer')
 
     arguments.add_common_arguments(subparser, ['tags'])
 
@@ -90,11 +94,11 @@ def match(p, f):
 
 
 @formatter
-def name_only(pkgs):
+def name_only(pkgs, out):
     indent = 0
-    if sys.stdout.isatty():
+    if out.isatty():
         tty.msg("%d packages." % len(pkgs))
-    colify(pkgs, indent=indent)
+    colify(pkgs, indent=indent, output=out)
 
 
 def github_url(pkg):
@@ -123,64 +127,69 @@ def rows_for_ncols(elts, ncols):
 
 
 @formatter
-def rst(pkg_names):
+def rst(pkg_names, out):
     """Print out information on all packages in restructured text."""
 
     pkgs = [spack.repo.get(name) for name in pkg_names]
 
-    print('.. _package-list:')
+    out.write('.. _package-list:\n')
-    print()
+    out.write('\n')
-    print('============')
+    out.write('============\n')
-    print('Package List')
+    out.write('Package List\n')
-    print('============')
+    out.write('============\n')
-    print()
+    out.write('\n')
-    print('This is a list of things you can install using Spack.  It is')
+    out.write('This is a list of things you can install using Spack.  It is\n')
-    print('automatically generated based on the packages in the latest Spack')
+    out.write(
-    print('release.')
+        'automatically generated based on the packages in the latest Spack\n')
-    print()
+    out.write('release.\n')
-    print('Spack currently has %d mainline packages:' % len(pkgs))
+    out.write('\n')
-    print()
+    out.write('Spack currently has %d mainline packages:\n' % len(pkgs))
-    print(rst_table('`%s`_' % p for p in pkg_names))
+    out.write('\n')
-    print()
+    out.write(rst_table('`%s`_' % p for p in pkg_names))
+    out.write('\n')
+    out.write('\n')
 
     # Output some text for each package.
     for pkg in pkgs:
-        print('-----')
+        out.write('-----\n')
-        print()
+        out.write('\n')
-        print('.. _%s:' % pkg.name)
+        out.write('.. _%s:\n' % pkg.name)
-        print()
+        out.write('\n')
         # Must be at least 2 long, breaks for single letter packages like R.
-        print('-' * max(len(pkg.name), 2))
+        out.write('-' * max(len(pkg.name), 2))
-        print(pkg.name)
+        out.write('\n')
-        print('-' * max(len(pkg.name), 2))
+        out.write(pkg.name)
-        print()
+        out.write('\n')
-        print('Homepage:')
+        out.write('-' * max(len(pkg.name), 2))
-        print('  * `%s <%s>`__' % (cgi.escape(pkg.homepage), pkg.homepage))
+        out.write('\n\n')
-        print()
+        out.write('Homepage:\n')
-        print('Spack package:')
+        out.write(
-        print('  * `%s/package.py <%s>`__' % (pkg.name, github_url(pkg)))
+            '  * `%s <%s>`__\n' % (cgi.escape(pkg.homepage), pkg.homepage))
-        print()
+        out.write('\n')
+        out.write('Spack package:\n')
+        out.write('  * `%s/package.py <%s>`__\n' % (pkg.name, github_url(pkg)))
+        out.write('\n')
         if pkg.versions:
-            print('Versions:')
+            out.write('Versions:\n')
-            print('  ' + ', '.join(str(v) for v in
+            out.write('  ' + ', '.join(str(v) for v in
-                                   reversed(sorted(pkg.versions))))
+                                       reversed(sorted(pkg.versions))))
-            print()
+            out.write('\n\n')
 
         for deptype in spack.dependency.all_deptypes:
             deps = pkg.dependencies_of_type(deptype)
             if deps:
-                print('%s Dependencies' % deptype.capitalize())
+                out.write('%s Dependencies\n' % deptype.capitalize())
-                print('  ' + ', '.join('%s_' % d if d in pkg_names
+                out.write('  ' + ', '.join('%s_' % d if d in pkg_names
-                                       else d for d in deps))
+                                           else d for d in deps))
-                print()
+                out.write('\n\n')
 
-        print('Description:')
+        out.write('Description:\n')
-        print(pkg.format_doc(indent=2))
+        out.write(pkg.format_doc(indent=2))
-        print()
+        out.write('\n\n')
 
 
 @formatter
-def html(pkg_names):
+def html(pkg_names, out):
     """Print out information on all packages in Sphinx HTML.
 
     This is intended to be inlined directly into Sphinx documentation.
@@ -199,83 +208,90 @@ def html(pkg_names):
     def head(n, span_id, title, anchor=None):
         if anchor is None:
             anchor = title
-        print(('<span id="id%d"></span>'
+        out.write(('<span id="id%d"></span>'
-               '<h1>%s<a class="headerlink" href="#%s" '
+                   '<h1>%s<a class="headerlink" href="#%s" '
-               'title="Permalink to this headline">&para;</a>'
+                   'title="Permalink to this headline">&para;</a>'
-               '</h1>') % (span_id, title, anchor))
+                   '</h1>\n') % (span_id, title, anchor))
 
     # Start with the number of packages, skipping the title and intro
     # blurb, which we maintain in the RST file.
-    print('<p>')
+    out.write('<p>\n')
-    print('Spack currently has %d mainline packages:' % len(pkgs))
+    out.write('Spack currently has %d mainline packages:\n' % len(pkgs))
-    print('</p>')
+    out.write('</p>\n')
 
     # Table of links to all packages
-    print('<table border="1" class="docutils">')
+    out.write('<table border="1" class="docutils">\n')
-    print('<tbody valign="top">')
+    out.write('<tbody valign="top">\n')
     for i, row in enumerate(rows_for_ncols(pkg_names, 3)):
-        print('<tr class="row-odd">' if i % 2 == 0 else
+        out.write('<tr class="row-odd">\n' if i % 2 == 0 else
-              '<tr class="row-even">')
+                  '<tr class="row-even">\n')
         for name in row:
-            print('<td>')
+            out.write('<td>\n')
-            print('<a class="reference internal" href="#%s">%s</a></td>'
+            out.write('<a class="reference internal" href="#%s">%s</a></td>\n'
-                  % (name, name))
+                      % (name, name))
-            print('</td>')
+            out.write('</td>\n')
-        print('</tr>')
+        out.write('</tr>\n')
-    print('</tbody>')
+    out.write('</tbody>\n')
-    print('</table>')
+    out.write('</table>\n')
-    print('<hr class="docutils"/>')
+    out.write('<hr class="docutils"/>\n')
 
     # Output some text for each package.
     for pkg in pkgs:
-        print('<div class="section" id="%s">' % pkg.name)
+        out.write('<div class="section" id="%s">\n' % pkg.name)
         head(2, span_id, pkg.name)
         span_id += 1
 
-        print('<dl class="docutils">')
+        out.write('<dl class="docutils">\n')
 
-        print('<dt>Homepage:</dt>')
+        out.write('<dt>Homepage:</dt>\n')
-        print('<dd><ul class="first last simple">')
+        out.write('<dd><ul class="first last simple">\n')
-        print(('<li>'
+        out.write(('<li>'
-               '<a class="reference external" href="%s">%s</a>'
+                   '<a class="reference external" href="%s">%s</a>'
-               '</li>') % (pkg.homepage, cgi.escape(pkg.homepage)))
+                   '</li>\n') % (pkg.homepage, cgi.escape(pkg.homepage)))
-        print('</ul></dd>')
+        out.write('</ul></dd>\n')
 
-        print('<dt>Spack package:</dt>')
+        out.write('<dt>Spack package:</dt>\n')
-        print('<dd><ul class="first last simple">')
+        out.write('<dd><ul class="first last simple">\n')
-        print(('<li>'
+        out.write(('<li>'
-               '<a class="reference external" href="%s">%s/package.py</a>'
+                   '<a class="reference external" href="%s">%s/package.py</a>'
-               '</li>') % (github_url(pkg), pkg.name))
+                   '</li>\n') % (github_url(pkg), pkg.name))
-        print('</ul></dd>')
+        out.write('</ul></dd>\n')
 
         if pkg.versions:
-            print('<dt>Versions:</dt>')
+            out.write('<dt>Versions:</dt>\n')
-            print('<dd>')
+            out.write('<dd>\n')
-            print(', '.join(str(v) for v in reversed(sorted(pkg.versions))))
+            out.write(', '.join(
-            print('</dd>')
+                str(v) for v in reversed(sorted(pkg.versions))))
+            out.write('\n')
+            out.write('</dd>\n')
 
         for deptype in spack.dependency.all_deptypes:
             deps = pkg.dependencies_of_type(deptype)
             if deps:
-                print('<dt>%s Dependencies:</dt>' % deptype.capitalize())
+                out.write('<dt>%s Dependencies:</dt>\n' % deptype.capitalize())
-                print('<dd>')
+                out.write('<dd>\n')
-                print(', '.join(
+                out.write(', '.join(
                     d if d not in pkg_names else
                     '<a class="reference internal" href="#%s">%s</a>' % (d, d)
                     for d in deps))
-                print('</dd>')
+                out.write('\n')
+                out.write('</dd>\n')
 
-        print('<dt>Description:</dt>')
+        out.write('<dt>Description:</dt>\n')
-        print('<dd>')
+        out.write('<dd>\n')
-        print(cgi.escape(pkg.format_doc(indent=2)))
+        out.write(cgi.escape(pkg.format_doc(indent=2)))
-        print('</dd>')
+        out.write('\n')
-        print('</dl>')
+        out.write('</dd>\n')
+        out.write('</dl>\n')
 
-        print('<hr class="docutils"/>')
+        out.write('<hr class="docutils"/>\n')
-        print('</div>')
+        out.write('</div>\n')
 
 
 def list(parser, args):
+    # retrieve the formatter to use from args
+    formatter = formatters[args.format]
+
     # Retrieve the names of all the packages
     pkgs = set(spack.repo.all_package_names())
     # Filter the set appropriately
@@ -288,5 +304,17 @@ def list(parser, args):
         sorted_packages = set(sorted_packages) & packages_with_tags
         sorted_packages = sorted(sorted_packages)
 
-    # Print to stdout
+    if args.update:
-    formatters[args.format](sorted_packages)
+        # change output stream if user asked for update
+        if os.path.exists(args.update):
+            if os.path.getmtime(args.update) > spack.repo.path.last_mtime():
+                tty.msg('File is up to date: %s' % args.update)
+                return
+
+        tty.msg('Updating file: %s' % args.update)
+        with open(args.update, 'w') as f:
+            formatter(sorted_packages, f)
+
+    else:
+        # Print to stdout
+        formatter(sorted_packages, sys.stdout)

lib/spack/spack/repo.py

@@ -183,6 +183,10 @@ def _create_new_cache(self):
 
         return cache
 
+    def last_mtime(self):
+        return max(
+            sinfo.st_mtime for sinfo in self._packages_to_stats.values())
+
     def __getitem__(self, item):
         return self._packages_to_stats[item]
 
@@ -607,6 +611,10 @@ def load_module(self, fullname):
         sys.modules[fullname] = module
         return module
 
+    def last_mtime(self):
+        """Time a package file in this repo was last updated."""
+        return max(repo.last_mtime() for repo in self.repos)
+
     def repo_for_pkg(self, spec):
         """Given a spec, get the repository for its package."""
         # We don't @_autospec this function b/c it's called very frequently
@@ -1018,6 +1026,10 @@ def exists(self, pkg_name):
         """Whether a package with the supplied name exists."""
         return pkg_name in self._pkg_checker
 
+    def last_mtime(self):
+        """Time a package file in this repo was last updated."""
+        return self._pkg_checker.last_mtime()
+
     def is_virtual(self, pkg_name):
         """True if the package with this name is virtual, False otherwise."""
         return self.provider_index.contains(pkg_name)

lib/spack/spack/test/cmd/list.py

@@ -59,3 +59,30 @@ def test_list_format_html():
 
     assert '<div class="section" id="hdf5">' in output
     assert '<h1>hdf5' in output
+
+
+def test_list_update(tmpdir):
+    update_file = tmpdir.join('output')
+
+    # not yet created when list is run
+    list('--update', str(update_file))
+    assert update_file.exists()
+    with update_file.open() as f:
+        assert f.read()
+
+    # created but older than any package
+    with update_file.open('w') as f:
+        f.write('empty\n')
+    update_file.setmtime(0)
+    list('--update', str(update_file))
+    assert update_file.exists()
+    with update_file.open() as f:
+        assert f.read() != 'empty\n'
+
+    # newer than any packages
+    with update_file.open('w') as f:
+        f.write('empty\n')
+    list('--update', str(update_file))
+    assert update_file.exists()
+    with update_file.open() as f:
+        assert f.read() == 'empty\n'

lib/spack/spack/test/repo.py

@@ -3,6 +3,7 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
+import os
 import pytest
 
 import spack.repo
@@ -56,3 +57,10 @@ def test_repo_pkg_with_unknown_namespace(repo_for_test):
 def test_repo_unknown_pkg(repo_for_test):
     with pytest.raises(spack.repo.UnknownPackageError):
         repo_for_test.get('builtin.mock.nonexistentpackage')
+
+
+@pytest.mark.maybeslow
+def test_repo_last_mtime():
+    latest_mtime = max(os.path.getmtime(p.module.__file__)
+                       for p in spack.repo.path.all_packages())
+    assert spack.repo.path.last_mtime() == latest_mtime