From 4aeb0d19bde272cc90312867955044d97b5b2d66 Mon Sep 17 00:00:00 2001 From: Vanessasaurus Date: Thu, 2 May 2019 21:27:13 -0400 Subject: [PATCH] Singularity package: new version, legacy builds, and root permissions (#11094) * Remove old Singularity versions, add version 3.1.1 * Recent versions of Singularity build with go rather than autotools: update dependencies and build logic * Move old Singularity versions to new singularity-legacy package which uses the autotools build system * Some binaries built by Singularity need to be run as root: include a script that the user can run after the Spack Singularity install to change these permissions. * The Singularity go build expects to work with a custom stage directory relative to GOPATH: override the stage phase to create this expected path. * Update Singularity install config to point to Spack-installed makesquashfs dependency --- .../packages/singularity-legacy/package.py | 36 +++++ .../builtin/packages/singularity/package.py | 149 +++++++++++++++--- .../singularity/spack_perms_fix.sh.j2 | 11 ++ 3 files changed, 177 insertions(+), 19 deletions(-) create mode 100644 var/spack/repos/builtin/packages/singularity-legacy/package.py create mode 100644 var/spack/repos/builtin/packages/singularity/spack_perms_fix.sh.j2 diff --git a/var/spack/repos/builtin/packages/singularity-legacy/package.py b/var/spack/repos/builtin/packages/singularity-legacy/package.py new file mode 100644 index 0000000000..52caa53450 --- /dev/null +++ b/var/spack/repos/builtin/packages/singularity-legacy/package.py @@ -0,0 +1,36 @@ +# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +from spack import * + + +class SingularityLegacy(AutotoolsPackage): + """Singularity is a container platform focused on supporting 'Mobility of + Compute'. The software changed the installation method from AutoTools + to GoLang, so we have two separate package names to support that. The + legacy package is pre-version 3.0.0 + """ + + homepage = "https://www.sylabs.io/singularity/" + url = "https://github.com/sylabs/singularity/releases/download/2.5.2/singularity-2.5.2.tar.gz" + git = "https://github.com/sylabs/singularity.git" + + # Versions before 2.5.2 suffer from a serious security problem. + # https://nvd.nist.gov/vuln/detail/CVE-2018-12021 + version('2.6-release', branch='vault/2.6-release') + version('2.6.1', sha256='f38d46a225e8368eb4693137806d2dc96e925a50bdf7f6983662848831041df2') + version('2.6.0', sha256='7c425211a099f6fa6f74037e6e17be58fb5923b0bd11aea745e48ef83c488b49') + version('2.5.2', '2edc1a8ac9a4d7d26fba6244f1c5fd95') + + depends_on('libarchive', when='@2.5.2:') + # these are only needed if we're grabbing the unreleased tree + depends_on('m4', type='build', when='@2.6-release') + depends_on('autoconf', type='build', when='@2.6-release') + depends_on('automake', type='build', when='@2.6-release') + depends_on('libtool', type='build', when='@2.6-release') + + # When installing as root, the copy has to run before chmod runs + def install(self, spec, prefix): + make('install', parallel=False) diff --git a/var/spack/repos/builtin/packages/singularity/package.py b/var/spack/repos/builtin/packages/singularity/package.py index 273d73c210..4287773a77 100644 --- a/var/spack/repos/builtin/packages/singularity/package.py +++ b/var/spack/repos/builtin/packages/singularity/package.py @@ -5,29 +5,140 @@ from spack import * +import llnl.util.tty as tty +import os +import shutil -class Singularity(AutotoolsPackage): - """Singularity is a container platform focused on supporting 'Mobility of - Compute'""" + +class Singularity(MakefilePackage): + '''Singularity is a container technology focused on building portable + encapsulated environments to support "Mobility of Compute" For older + versions of Singularity (pre 3.0) you should use singularity-legacy, + which has a different install base (Autotools). + + Needs post-install chmod/chown steps to enable full functionality. + See package definition for details. + ''' homepage = "https://www.sylabs.io/singularity/" - url = "https://github.com/singularityware/singularity/releases/download/2.5.2/singularity-2.5.2.tar.gz" - git = "https://github.com/singularityware/singularity.git" + url = "https://github.com/sylabs/singularity/releases/download/v3.1.1/singularity-3.1.1.tar.gz" + git = "https://github.com/sylabs/singularity.git" - # Versions before 2.5.2 suffer from a serious security problem. - # https://nvd.nist.gov/vuln/detail/CVE-2018-12021 version('develop', branch='master') - version('2.6.1', sha256='f38d46a225e8368eb4693137806d2dc96e925a50bdf7f6983662848831041df2') - version('2.6.0', sha256='7c425211a099f6fa6f74037e6e17be58fb5923b0bd11aea745e48ef83c488b49') - version('2.5.2', '2edc1a8ac9a4d7d26fba6244f1c5fd95') + version('3.1.1', '158f58a79db5337e1d655ee0159b641e42ea7435') - depends_on('libarchive', when='@2.5.2:') - # these are only needed if we're grabbing the unreleased tree - depends_on('m4', type='build', when='@develop') - depends_on('autoconf', type='build', when='@develop') - depends_on('automake', type='build', when='@develop') - depends_on('libtool', type='build', when='@develop') + depends_on('go') + depends_on('libuuid') + depends_on('libgpg-error') + depends_on('squashfs', type='run') + depends_on('git', when='@develop') # mconfig uses it for version info - # When installing as root, the copy has to run before chmod runs - def install(self, spec, prefix): - make('install', parallel=False) + # Go has novel ideas about how projects should be organized. + # We'll point GOPATH at the stage dir, and move the unpacked src + # tree into the proper subdir in our overridden do_stage below. + @property + def gopath(self): + return join_path(self.stage.path) + + @property + def sylabs_gopath_dir(self): + return join_path(self.gopath, 'src/github.com/sylabs/') + + @property + def singularity_gopath_dir(self): + return join_path(self.sylabs_gopath_dir, 'singularity') + + # Unpack the tarball as usual, then move the src dir into + # its home within GOPATH. + def do_stage(self, mirror_only=False): + super(Singularity, self).do_stage(mirror_only) + source_path = self.stage.source_path + if not os.path.exists(self.singularity_gopath_dir): + tty.debug("Moving {0} to {1}".format( + source_path, self.singularity_gopath_dir)) + mkdirp(self.sylabs_gopath_dir) + shutil.move(source_path, + self.singularity_gopath_dir) + + # MakefilePackage's stages use this via working_dir() + @property + def build_directory(self): + return self.singularity_gopath_dir + + # Hijack the edit stage to run mconfig. + def edit(self, spec, prefix): + with working_dir(self.build_directory): + configure = Executable('./mconfig --prefix=%s' % prefix) + configure() + + # Set these for use by MakefilePackage's default build/install methods. + build_targets = ['-C', 'builddir', 'parallel=False'] + install_targets = ['install', '-C', 'builddir', 'parallel=False'] + + def setup_environment(self, spack_env, run_env): + # Point GOPATH at the top of the staging dir for the build + # step. + spack_env.prepend_path('GOPATH', self.gopath) + + # `singularity` has a fixed path where it will look for + # mksquashfs. If it lives somewhere else you need to specify the + # full path in the config file. This bit uses filter_file to edit + # the config file, uncommenting and setting the mksquashfs path. + @run_after('install') + def fix_mksquashfs_path(self): + prefix = self.spec.prefix + squash_path = join_path(self.spec['squashfs'].prefix.bin, 'mksquashfs') + filter_file(r'^# mksquashfs path =', + 'mksquashfs path = {0}'.format(squash_path), + join_path(prefix.etc, 'singularity', 'singularity.conf')) + + # + # Assemble a script that fixes the ownership and permissions of several + # key files, install it, and tty.warn() the user. + # HEADSUP: https://github.com/spack/spack/pull/10412. + # + def perm_script(self): + return 'spack_perms_fix.sh' + + def perm_script_tmpl(self): + return "{0}.j2".format(self.perm_script()) + + def perm_script_path(self): + return join_path(self.spec.prefix.bin, self.perm_script()) + + def _build_script(self, filename, variable_data): + with open(filename, 'w') as f: + env = spack.tengine.make_environment(dirs=self.package_dir) + t = env.get_template(self.perm_script_tmpl()) + f.write(t.render(variable_data)) + + @run_after('install') + def build_perms_script(self): + script = self.perm_script_path() + chown_files = ['libexec/singularity/bin/starter-suid', + 'etc/singularity/singularity.conf', + 'etc/singularity/capability.json', + 'etc/singularity/ecl.toml'] + setuid_files = ['libexec/singularity/bin/starter-suid'] + self._build_script(script, {'prefix': self.spec.prefix, + 'chown_files': chown_files, + 'setuid_files': setuid_files}) + chmod = which('chmod') + chmod('555', script) + + # Until tty output works better from build steps, this ends up in + # the build log. See https://github.com/spack/spack/pull/10412. + @run_after('install') + def caveats(self): + tty.warn(""" + For full functionality, you'll need to chown and chmod some files + after installing the package. This has security implications. + See: https://singularity.lbl.gov/docs-security for details. + + We've installed a script that will make the necessary changes; + read through it and then execute it as root (e.g. via sudo). + + The script is named: + + {0} + """.format(self.perm_script_path())) diff --git a/var/spack/repos/builtin/packages/singularity/spack_perms_fix.sh.j2 b/var/spack/repos/builtin/packages/singularity/spack_perms_fix.sh.j2 new file mode 100644 index 0000000000..32baa21203 --- /dev/null +++ b/var/spack/repos/builtin/packages/singularity/spack_perms_fix.sh.j2 @@ -0,0 +1,11 @@ +#!/bin/sh -eu + +{% for cf in chown_files %} +chown root {{ prefix }}/{{ cf }} +{% endfor %} + +{% for sf in setuid_files %} +chmod 4555 {{ prefix }}/{{ sf }} +{% endfor %} + +# end