Singularity package: new version, legacy builds, and root permissions (#11094)

* Remove old Singularity versions, add version 3.1.1
* Recent versions of Singularity build with go rather than autotools:
  update dependencies and build logic
* Move old Singularity versions to new singularity-legacy package
  which uses the autotools build system
* Some binaries built by Singularity need to be run as root: include
  a script that the user can run after the Spack Singularity install
  to change these permissions.
* The Singularity go build expects to work with a custom stage
  directory relative to GOPATH: override the stage phase to create
  this expected path.
* Update Singularity install config to point to Spack-installed
  makesquashfs dependency

var/spack/repos/builtin/packages/singularity-legacy/package.py

@@ -0,0 +1,36 @@
+# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack import *
+
+
+class SingularityLegacy(AutotoolsPackage):
+    """Singularity is a container platform focused on supporting 'Mobility of
+       Compute'. The software changed the installation method from AutoTools
+       to GoLang, so we have two separate package names to support that. The
+       legacy package is pre-version 3.0.0
+    """
+
+    homepage = "https://www.sylabs.io/singularity/"
+    url      = "https://github.com/sylabs/singularity/releases/download/2.5.2/singularity-2.5.2.tar.gz"
+    git      = "https://github.com/sylabs/singularity.git"
+
+    # Versions before 2.5.2 suffer from a serious security problem.
+    # https://nvd.nist.gov/vuln/detail/CVE-2018-12021
+    version('2.6-release', branch='vault/2.6-release')
+    version('2.6.1', sha256='f38d46a225e8368eb4693137806d2dc96e925a50bdf7f6983662848831041df2')
+    version('2.6.0', sha256='7c425211a099f6fa6f74037e6e17be58fb5923b0bd11aea745e48ef83c488b49')
+    version('2.5.2', '2edc1a8ac9a4d7d26fba6244f1c5fd95')
+
+    depends_on('libarchive', when='@2.5.2:')
+    # these are only needed if we're grabbing the unreleased tree
+    depends_on('m4',       type='build', when='@2.6-release')
+    depends_on('autoconf', type='build', when='@2.6-release')
+    depends_on('automake', type='build', when='@2.6-release')
+    depends_on('libtool',  type='build', when='@2.6-release')
+
+    # When installing as root, the copy has to run before chmod runs
+    def install(self, spec, prefix):
+        make('install', parallel=False)

var/spack/repos/builtin/packages/singularity/package.py

@@ -5,29 +5,140 @@
 
 from spack import *
 
+import llnl.util.tty as tty
+import os
+import shutil
 
-class Singularity(AutotoolsPackage):
+
-    """Singularity is a container platform focused on supporting 'Mobility of
+class Singularity(MakefilePackage):
-       Compute'"""
+    '''Singularity is a container technology focused on building portable
+       encapsulated environments to support "Mobility of Compute" For older
+       versions of Singularity (pre 3.0) you should use singularity-legacy,
+       which has a different install base (Autotools).
+
+       Needs post-install chmod/chown steps to enable full functionality.
+       See package definition for details.
+    '''
 
     homepage = "https://www.sylabs.io/singularity/"
-    url      = "https://github.com/singularityware/singularity/releases/download/2.5.2/singularity-2.5.2.tar.gz"
+    url      = "https://github.com/sylabs/singularity/releases/download/v3.1.1/singularity-3.1.1.tar.gz"
-    git      = "https://github.com/singularityware/singularity.git"
+    git      = "https://github.com/sylabs/singularity.git"
 
-    # Versions before 2.5.2 suffer from a serious security problem.
-    # https://nvd.nist.gov/vuln/detail/CVE-2018-12021
     version('develop', branch='master')
-    version('2.6.1', sha256='f38d46a225e8368eb4693137806d2dc96e925a50bdf7f6983662848831041df2')
+    version('3.1.1', '158f58a79db5337e1d655ee0159b641e42ea7435')
-    version('2.6.0', sha256='7c425211a099f6fa6f74037e6e17be58fb5923b0bd11aea745e48ef83c488b49')
-    version('2.5.2', '2edc1a8ac9a4d7d26fba6244f1c5fd95')
 
-    depends_on('libarchive', when='@2.5.2:')
+    depends_on('go')
-    # these are only needed if we're grabbing the unreleased tree
+    depends_on('libuuid')
-    depends_on('m4',       type='build', when='@develop')
+    depends_on('libgpg-error')
-    depends_on('autoconf', type='build', when='@develop')
+    depends_on('squashfs', type='run')
-    depends_on('automake', type='build', when='@develop')
+    depends_on('git', when='@develop')  # mconfig uses it for version info
-    depends_on('libtool',  type='build', when='@develop')
 
-    # When installing as root, the copy has to run before chmod runs
+    # Go has novel ideas about how projects should be organized.
-    def install(self, spec, prefix):
+    # We'll point GOPATH at the stage dir, and move the unpacked src
-        make('install', parallel=False)
+    # tree into the proper subdir in our overridden do_stage below.
+    @property
+    def gopath(self):
+        return join_path(self.stage.path)
+
+    @property
+    def sylabs_gopath_dir(self):
+        return join_path(self.gopath, 'src/github.com/sylabs/')
+
+    @property
+    def singularity_gopath_dir(self):
+        return join_path(self.sylabs_gopath_dir, 'singularity')
+
+    # Unpack the tarball as usual, then move the src dir into
+    # its home within GOPATH.
+    def do_stage(self, mirror_only=False):
+        super(Singularity, self).do_stage(mirror_only)
+        source_path = self.stage.source_path
+        if not os.path.exists(self.singularity_gopath_dir):
+            tty.debug("Moving {0} to {1}".format(
+                source_path, self.singularity_gopath_dir))
+            mkdirp(self.sylabs_gopath_dir)
+            shutil.move(source_path,
+                        self.singularity_gopath_dir)
+
+    # MakefilePackage's stages use this via working_dir()
+    @property
+    def build_directory(self):
+        return self.singularity_gopath_dir
+
+    # Hijack the edit stage to run mconfig.
+    def edit(self, spec, prefix):
+        with working_dir(self.build_directory):
+            configure = Executable('./mconfig --prefix=%s' % prefix)
+            configure()
+
+    # Set these for use by MakefilePackage's default build/install methods.
+    build_targets = ['-C', 'builddir', 'parallel=False']
+    install_targets = ['install', '-C', 'builddir', 'parallel=False']
+
+    def setup_environment(self, spack_env, run_env):
+        # Point GOPATH at the top of the staging dir for the build
+        # step.
+        spack_env.prepend_path('GOPATH', self.gopath)
+
+    # `singularity` has a fixed path where it will look for
+    # mksquashfs.  If it lives somewhere else you need to specify the
+    # full path in the config file.  This bit uses filter_file to edit
+    # the config file, uncommenting and setting the mksquashfs path.
+    @run_after('install')
+    def fix_mksquashfs_path(self):
+        prefix = self.spec.prefix
+        squash_path = join_path(self.spec['squashfs'].prefix.bin, 'mksquashfs')
+        filter_file(r'^# mksquashfs path =',
+                    'mksquashfs path = {0}'.format(squash_path),
+                    join_path(prefix.etc, 'singularity', 'singularity.conf'))
+
+    #
+    # Assemble a script that fixes the ownership and permissions of several
+    # key files, install it, and tty.warn() the user.
+    # HEADSUP: https://github.com/spack/spack/pull/10412.
+    #
+    def perm_script(self):
+        return 'spack_perms_fix.sh'
+
+    def perm_script_tmpl(self):
+        return "{0}.j2".format(self.perm_script())
+
+    def perm_script_path(self):
+        return join_path(self.spec.prefix.bin, self.perm_script())
+
+    def _build_script(self, filename, variable_data):
+        with open(filename, 'w') as f:
+            env = spack.tengine.make_environment(dirs=self.package_dir)
+            t = env.get_template(self.perm_script_tmpl())
+            f.write(t.render(variable_data))
+
+    @run_after('install')
+    def build_perms_script(self):
+        script = self.perm_script_path()
+        chown_files = ['libexec/singularity/bin/starter-suid',
+                       'etc/singularity/singularity.conf',
+                       'etc/singularity/capability.json',
+                       'etc/singularity/ecl.toml']
+        setuid_files = ['libexec/singularity/bin/starter-suid']
+        self._build_script(script, {'prefix': self.spec.prefix,
+                                    'chown_files': chown_files,
+                                    'setuid_files': setuid_files})
+        chmod = which('chmod')
+        chmod('555', script)
+
+    # Until tty output works better from build steps, this ends up in
+    # the build log.  See https://github.com/spack/spack/pull/10412.
+    @run_after('install')
+    def caveats(self):
+        tty.warn("""
+        For full functionality, you'll need to chown and chmod some files
+        after installing the package.  This has security implications.
+        See: https://singularity.lbl.gov/docs-security for details.
+
+        We've installed a script that will make the necessary changes;
+        read through it and then execute it as root (e.g. via sudo).
+
+        The script is named:
+
+        {0}
+        """.format(self.perm_script_path()))

var/spack/repos/builtin/packages/singularity/spack_perms_fix.sh.j2

@@ -0,0 +1,11 @@
+#!/bin/sh -eu
+
+{% for cf  in chown_files %}
+chown root {{ prefix }}/{{ cf }}
+{% endfor %}
+
+{% for sf in setuid_files %}
+chmod 4555 {{ prefix }}/{{ sf }}
+{% endfor %}
+
+# end