Allow spack install -j to throttle make jobs.

bin/spack

@@ -56,18 +56,18 @@ from external import argparse
 # Command parsing
 parser = argparse.ArgumentParser(
     description='Spack: the Supercomputing PACKage Manager.')
-parser.add_argument('-V', '--version', action='version',
-                    version="%s" % spack.spack_version)
-parser.add_argument('-v', '--verbose', action='store_true',
-                    help="Print additional output during builds")
 parser.add_argument('-d', '--debug', action='store_true',
                     help="Write out debug logs during compile")
 parser.add_argument('-k', '--insecure', action='store_true',
-                    help="Do not check ssl certificates when downloading archives.")
+                    help="Do not check ssl certificates when downloading.")
 parser.add_argument('-m', '--mock', action='store_true',
                     help="Use mock packages instead of real ones.")
 parser.add_argument('-p', '--profile', action='store_true',
                     help="Profile execution using cProfile.")
+parser.add_argument('-v', '--verbose', action='store_true',
+                    help="Print additional output during builds")
+parser.add_argument('-V', '--version', action='version',
+                    version="%s" % spack.spack_version)
 
 # each command module implements a parser() function, to which we pass its
 # subparser for setup.

lib/spack/spack/build_environment.py

@@ -68,16 +68,16 @@ class MakeExecutable(Executable):
        Note that if the SPACK_NO_PARALLEL_MAKE env var is set it overrides
        everything.
     """
-    def __init__(self, name, parallel):
+    def __init__(self, name, jobs):
         super(MakeExecutable, self).__init__(name)
-        self.parallel = parallel
+        self.jobs = jobs
 
     def __call__(self, *args, **kwargs):
-        parallel = kwargs.get('parallel', self.parallel)
+        parallel = kwargs.get('parallel', self.jobs > 1)
         disable_parallel = env_flag(SPACK_NO_PARALLEL_MAKE)
 
-        if parallel and not disable_parallel:
+        if self.jobs > 1 and not disable_parallel:
-            jobs = "-j%d" % multiprocessing.cpu_count()
+            jobs = "-j%d" % self.jobs
             args = (jobs,) + args
 
         super(MakeExecutable, self).__call__(*args, **kwargs)
@@ -163,15 +163,21 @@ def set_module_variables_for_package(pkg):
     """
     m = pkg.module
 
-    m.make  = MakeExecutable('make', pkg.parallel)
+    # number of jobs spack will to build with.
-    m.gmake = MakeExecutable('gmake', pkg.parallel)
+    jobs = multiprocessing.cpu_count()
+    if not pkg.parallel:
+        jobs = 1
+    elif pkg.make_jobs:
+        jobs = pkg.make_jobs
+    m.make_jobs = jobs
+
+    # TODO: make these build deps that can be installed if not found.
+    m.make  = MakeExecutable('make', jobs)
+    m.gmake = MakeExecutable('gmake', jobs)
 
     # easy shortcut to os.environ
     m.env = os.environ
 
-    # number of jobs spack prefers to build with.
-    m.make_jobs = multiprocessing.cpu_count()
-
     # Find the configure script in the archive path
     # Don't use which for this; we want to find it in the current dir.
     m.configure = Executable('./configure')

lib/spack/spack/cmd/install.py

@@ -25,6 +25,8 @@
 import sys
 from external import argparse
 
+import llnl.util.tty as tty
+
 import spack
 import spack.cmd
 
@@ -34,6 +36,9 @@ def setup_parser(subparser):
     subparser.add_argument(
         '-i', '--ignore-dependencies', action='store_true', dest='ignore_deps',
         help="Do not try to install dependencies of requested packages.")
+    subparser.add_argument(
+        '-j', '--jobs', action='store', type=int,
+        help="Explicitly set number of make jobs.  Default is #cpus.")
     subparser.add_argument(
         '--keep-prefix', action='store_true', dest='keep_prefix',
         help="Don't remove the install prefix if installation fails.")
@@ -54,13 +59,19 @@ def install(parser, args):
     if not args.packages:
         tty.die("install requires at least one package argument")
 
+    if args.jobs is not None:
+        if args.jobs <= 0:
+            tty.die("The -j option must be a positive integer!")
+
     if args.no_checksum:
         spack.do_checksum = False
 
     specs = spack.cmd.parse_specs(args.packages, concretize=True)
     for spec in specs:
         package = spack.db.get(spec)
-        package.do_install(keep_prefix=args.keep_prefix,
+        package.do_install(
-                           keep_stage=args.keep_stage,
+            keep_prefix=args.keep_prefix,
-                           ignore_deps=args.ignore_deps,
+            keep_stage=args.keep_stage,
-                           fake=args.fake)
+            ignore_deps=args.ignore_deps,
+            make_jobs=args.jobs,
+            fake=args.fake)

lib/spack/spack/package.py

@@ -333,6 +333,9 @@ class SomePackage(Package):
     """By default we build in parallel.  Subclasses can override this."""
     parallel = True
 
+    """# jobs to use for parallel make. If set, overrides default of ncpus."""
+    make_jobs = None
+
     """Most packages are NOT extendable.  Set to True if you want extensions."""
     extendable = False
 
@@ -781,9 +784,12 @@ def do_install(self, **kwargs):
         """
         # whether to keep the prefix on failure.  Default is to destroy it.
         keep_prefix  = kwargs.get('keep_prefix', False)
-        keep_stage   = kwargs.get('keep_stage', False)
+        keep_stage   = kwargs.get('keep_stage',  False)
         ignore_deps  = kwargs.get('ignore_deps', False)
-        fake_install = kwargs.get('fake', False)
+        fake_install = kwargs.get('fake',        False)
+
+        # Override builtin number of make jobs.
+        self.make_jobs    = kwargs.get('make_jobs',   None)
 
         if not self.spec.concrete:
             raise ValueError("Can only install concrete packages.")