flux: update flux-core package and add a flux-sched package (#8818)

Renames the flux package to flux-core and updates its dependencies, versions,
and variants. Adds a flux-sched package.

This also updates two flux-core dependencies: lua and czmq.

var/spack/repos/builtin/packages/czmq/package.py

@@ -30,6 +30,7 @@ class Czmq(AutotoolsPackage):
     homepage = "http://czmq.zeromq.org"
     url      = "https://github.com/zeromq/czmq/archive/v4.0.2.tar.gz"
 
+    version('4.1.1', 'a2ab03cddd14399c6ba75b030a256211')
     version('4.0.2', 'a65317a3fb8238cf70e3e992e381f9cc')
     version('3.0.2', '23e9885f7ee3ce88d99d0425f52e9be1')
 
@@ -37,6 +38,7 @@ class Czmq(AutotoolsPackage):
     depends_on('automake', type='build')
     depends_on('autoconf', type='build')
     depends_on('pkgconfig', type='build')
+    depends_on("libuuid")
     depends_on('zeromq')
 
     def configure_args(self):

var/spack/repos/builtin/packages/flux-core/package.py

@@ -0,0 +1,131 @@
+##############################################################################
+# Copyright (c) 2013-2018, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://github.com/spack/spack
+# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License (as
+# published by the Free Software Foundation) version 2.1, February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+from spack import *
+import os
+
+
+class FluxCore(AutotoolsPackage):
+    """ A next-generation resource manager (pre-alpha) """
+
+    homepage = "https://github.com/flux-framework/flux-core"
+    url      = "https://github.com/flux-framework/flux-core/releases/download/v0.8.0/flux-core-0.8.0.tar.gz"
+    git      = "https://github.com/flux-framework/flux-core.git"
+
+    version('master',  branch='master')
+    version('0.10.0', 'a84a1ed53a69c805c253bc940540cbf667a059b2008fd2a6a9bb890a985ead08e88dcbba68c01567f887357306fbfded41b93cc33edfa7809955ba5ba5870284')
+    version('0.9.0',  '70eaec1005aa49e8d8cf397570789cebedfb5d917efe963390d456ee4c473eefb15b0c81ea83f60a1fd057fe7be356bbafdebcae64b499844d194c48f6aefa05')
+    version('0.8.0',  'b0fec05acedc530bcdf75b2477ac22f39d2adddc7af8ff76496208a5e1e8185b1b4a18677871d95c3cfbf34b05f391953651200917fe029931f4e2beb79d70df')
+
+    # Avoid the infinite symlink issue
+    # This workaround is documented in PR #3543
+    build_directory = 'spack-build'
+
+    variant('doc', default=False, description='Build flux manpages')
+    variant('cuda', default=False, description='Build dependencies with support for CUDA')
+
+    depends_on("zeromq@4.0.4:")
+    depends_on("czmq@2.2:")
+    depends_on("hwloc@1.11.1:1.99")
+    depends_on("hwloc +cuda", when='+cuda')
+    depends_on("lua", type=('build', 'run', 'link'))
+    depends_on("lua@5.1:5.1.99", when="@0.1.0:0.9.0")
+    depends_on("lua@5.1:5.2.99", when="@0.10.0:,master")
+    depends_on("lua-luaposix")
+    depends_on("munge")
+    depends_on("libuuid")
+    depends_on("python", type=('build', 'run'))
+    depends_on("py-cffi", type=('build', 'run'))
+    depends_on("jansson")
+    depends_on("yaml-cpp")
+
+    # versions up to 0.8.0 uses pylint to check Flux's python binding
+    # later versions provide a configure flag and disable the check by default
+    depends_on("py-pylint", when='@:0.8.0', type='build')
+
+    depends_on("asciidoc", type='build', when="+docs")
+
+    # Need autotools when building on master:
+    depends_on("autoconf", type='build', when='@master')
+    depends_on("automake", type='build', when='@master')
+    depends_on("libtool", type='build', when='@master')
+
+    def setup(self):
+        pass
+
+    @when('@master')
+    def setup(self):
+        # Check in case we are running `spack diy` from an "unshallow" clone
+        if os.path.exists('.git/shallow'):
+            # Allow git-describe to get last tag so flux-version works:
+            git = which('git')
+            git('fetch', '--unshallow')
+
+    def autoreconf(self, spec, prefix):
+        self.setup()
+        if not os.path.exists('configure'):
+            # Bootstrap with autotools
+            bash = which('bash')
+            bash('./autogen.sh')
+
+    @property
+    def lua_version(self):
+        return self.spec['lua'].version.up_to(2)
+
+    @property
+    def lua_share_dir(self):
+        return os.path.join('share', 'lua', str(self.lua_version))
+
+    @property
+    def lua_lib_dir(self):
+        return os.path.join('lib', 'lua', str(self.lua_version))
+
+    def setup_environment(self, spack_env, run_env):
+        #  Ensure ./fluxometer.lua can be found during flux's make check
+        spack_env.append_path('LUA_PATH', './?.lua', separator=';')
+
+        run_env.prepend_path(
+            'LUA_PATH',
+            os.path.join(self.spec.prefix, self.lua_share_dir, '?.lua'),
+            separator=';')
+        run_env.prepend_path(
+            'LUA_CPATH',
+            os.path.join(self.spec.prefix, self.lua_lib_dir, '?.so'),
+            separator=';')
+        run_env.prepend_path(
+            'PYTHONPATH',
+            os.path.join(
+                self.spec.prefix.lib,
+                "python{0}".format(self.spec['python'].version.up_to(2)),
+                "site-packages"),
+        )
+        run_env.prepend_path('FLUX_MODULE_PATH', self.prefix.lib.flux.modules)
+        run_env.prepend_path('FLUX_EXEC_PATH', self.prefix.libexec.flux.cmd)
+        run_env.prepend_path('FLUX_RC_PATH', self.prefix.etc.flux)
+
+    def configure_args(self):
+        args = ['--enable-pylint=no']
+        if '+docs' not in self.spec:
+            args.append('--disable-docs')
+        return args

var/spack/repos/builtin/packages/flux-sched/package.py

@@ -0,0 +1,105 @@
+##############################################################################
+# Copyright (c) 2013-2018, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://github.com/spack/spack
+# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License (as
+# published by the Free Software Foundation) version 2.1, February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+from spack import *
+import os
+
+
+class FluxSched(AutotoolsPackage):
+    """ A scheduler for flux-core (pre-alpha) """
+
+    homepage = "https://github.com/flux-framework/flux-sched"
+    url      = "https://github.com/flux-framework/flux-sched/releases/download/v0.5.0/flux-sched-0.5.0.tar.gz"
+    git      = "https://github.com/flux-framework/flux-sched.git"
+
+    version('master', branch='master')
+    version('0.5.0', 'a9835c9c478aa41123a4e12672500052228aaf1ea770f74cb0901dbf4a049bd7d329e99d8d3484e39cfed1f911705030b2775dcfede39bc8bea59c6afe2549b1')
+    version('0.4.0', '82732641ac4594ffe9b94ca442a99e92bf5f91bc14745af92203a887a40610dd44edda3ae07f9b6c8d63799b2968d87c8da28f1488edef1310d0d12be9bd6319')
+
+    variant('cuda', default=False, description='Build dependencies with support for CUDA')
+
+    depends_on("boost+graph", when='@0.5.0:,master')
+
+    depends_on("flux-core", type=('build', 'link', 'run'))
+    depends_on("flux-core+cuda", when='+cuda')
+    depends_on("flux-core@0.8.0", when='@0.4.0')
+    depends_on("flux-core@0.9.0", when='@0.5.0')
+    depends_on("flux-core@0.10.0", when='@0.6.0')
+    depends_on("flux-core@master", when='@master')
+
+    # Need autotools when building on master:
+    depends_on("autoconf", type='build', when='@master')
+    depends_on("automake", type='build', when='@master')
+    depends_on("libtool", type='build', when='@master')
+
+    def setup(self):
+        pass
+
+    @when('@master')
+    def setup(self):
+        # Check in case we are running `spack diy` from an "unshallow" clone
+        if os.path.exists('.git/shallow'):
+            # Allow git-describe to get last tag so flux-version works:
+            git = which('git')
+            git('fetch', '--unshallow')
+
+    def autoreconf(self, spec, prefix):
+        self.setup()
+        if not os.path.exists('configure'):
+            # Bootstrap with autotools
+            bash = which('bash')
+            bash('./autogen.sh')
+
+    def configure_args(self):
+        # flux-sched's ax_boost is sometimes weird about non-system locations
+        # explicitly setting the path guarantees success
+        return ['--with-boost={0}'.format(self.spec['boost'].prefix)]
+
+    @property
+    def lua_version(self):
+        return self.spec['lua'].version.up_to(2)
+
+    @property
+    def lua_share_dir(self):
+        return os.path.join('share', 'lua', str(self.lua_version))
+
+    @property
+    def lua_lib_dir(self):
+        return os.path.join('lib', 'lua', str(self.lua_version))
+
+    def setup_environment(self, spack_env, run_env):
+        run_env.prepend_path(
+            'LUA_PATH',
+            os.path.join(self.spec.prefix, self.lua_share_dir, '?.lua'),
+            separator=';')
+        run_env.prepend_path(
+            'LUA_CPATH',
+            os.path.join(self.spec.prefix, self.lua_lib_dir, '?.so'),
+            separator=';')
+
+        run_env.prepend_path('FLUX_MODULE_PATH', self.prefix.lib.flux.modules)
+        run_env.prepend_path('FLUX_MODULE_PATH',
+                             self.prefix.lib.flux.modules.sched)
+        run_env.prepend_path('FLUX_EXEC_PATH', self.prefix.libexec.flux.cmd)
+        run_env.prepend_path('FLUX_RC_EXTRA', self.prefix.etc.flux)

var/spack/repos/builtin/packages/flux/package.py

@@ -1,80 +0,0 @@
-##############################################################################
-# Copyright (c) 2013-2018, Lawrence Livermore National Security, LLC.
-# Produced at the Lawrence Livermore National Laboratory.
-#
-# This file is part of Spack.
-# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
-# LLNL-CODE-647188
-#
-# For details, see https://github.com/spack/spack
-# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU Lesser General Public License (as
-# published by the Free Software Foundation) version 2.1, February 1999.
-#
-# This program is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
-# conditions of the GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-##############################################################################
-from spack import *
-import os
-
-
-class Flux(AutotoolsPackage):
-    """ A next-generation resource manager (pre-alpha) """
-
-    homepage = "https://github.com/flux-framework/flux-core"
-    url      = "https://github.com/flux-framework/flux-core/releases/download/v0.8.0/flux-core-0.8.0.tar.gz"
-    git      = "https://github.com/flux-framework/flux-core.git"
-
-    version('master', branch='master')
-    version('0.8.0', md5='9ee12974a8b2ab9a30533f69826f3bec')
-
-    variant('doc', default=False, description='Build flux manpages')
-
-    depends_on("zeromq@4.0.4:")
-    depends_on("czmq@2.2:")
-    depends_on("hwloc")
-    depends_on("lua@5.1:5.1.99")
-    depends_on("lua-luaposix")
-    depends_on("munge")
-    depends_on("libuuid")
-    depends_on("python")
-    depends_on("py-cffi", type=('build', 'run'))
-    depends_on("jansson")
-
-    depends_on("asciidoc", type='build', when="+docs")
-
-    # Need autotools when building on master:
-    depends_on("autoconf", type='build', when='@master')
-    depends_on("automake", type='build', when='@master')
-    depends_on("libtool", type='build', when='@master')
-
-    def setup():
-        pass
-
-    @when('@master')
-    def setup(self):
-        # Allow git-describe to get last tag so flux-version works:
-        git = which('git')
-        git('pull', '--depth=50', '--tags')
-
-    def autoreconf(self, spec, prefix):
-        self.setup()
-        if os.path.exists('autogen.sh'):
-            # Bootstrap with autotools
-            bash = which('bash')
-            bash('./autogen.sh')
-
-    def setup_environment(self, spack_env, run_env):
-        #  Ensure ./fluxometer.lua can be found during flux's make check
-        spack_env.append_path('LUA_PATH', './?.lua', separator=';')
-
-    def configure_args(self):
-        return ['--disable-docs'] if '+docs' not in self.spec else []

var/spack/repos/builtin/packages/lua/package.py

@@ -64,28 +64,41 @@ def install(self, spec, prefix):
         else:
             target = 'linux'
         make('INSTALL_TOP=%s' % prefix,
-             'MYCFLAGS=%s' % (
-                 self.compiler.pic_flag),
              'MYLDFLAGS=-L%s -L%s' % (
                  spec['readline'].prefix.lib,
                  spec['ncurses'].prefix.lib),
              'MYLIBS=-lncursesw',
-             'CC=%s -std=gnu99' % spack_cc,
+             'CC=%s -std=gnu99 %s' % (spack_cc,
+                                      self.compiler.pic_flag),
              target)
         make('INSTALL_TOP=%s' % prefix,
-             'MYCFLAGS=%s' % (
-                 self.compiler.pic_flag),
-             'MYLDFLAGS=-L%s -L%s' % (
-                 spec['readline'].prefix.lib,
-                 spec['ncurses'].prefix.lib),
-             'MYLIBS=-lncursesw',
-             'CC=%s -std=gnu99' % spack_cc,
              'install')
 
         static_to_shared_library(join_path(prefix.lib, 'liblua.a'),
                                  arguments=['-lm'], version=self.version,
                                  compat_version=self.version.up_to(2))
 
+        # compatibility with ax_lua.m4 from autoconf-archive
+        # https://www.gnu.org/software/autoconf-archive/ax_lua.html
+        with working_dir(prefix.lib):
+            # e.g., liblua.so.5.1.5
+            src_path = 'liblua.{0}.{1}'.format(dso_suffix,
+                                               str(self.version.up_to(3)))
+
+            # For lua version 5.1.X, the symlinks should be:
+            # liblua5.1.so
+            # liblua51.so
+            # liblua-5.1.so
+            # liblua-51.so
+            version_formats = [str(self.version.up_to(2)),
+                               Version(str(self.version.up_to(2))).joined]
+            for version_str in version_formats:
+                for joiner in ['', '-']:
+                    dest_path = 'liblua{0}{1}.{2}'.format(joiner,
+                                                          version_str,
+                                                          dso_suffix)
+                    os.symlink(src_path, dest_path)
+
         with working_dir(os.path.join('luarocks', 'luarocks')):
             configure('--prefix=' + prefix, '--with-lua=' + prefix)
             make('build')