From e6ec3729260b666483b02e38501671bf32ec3af5 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Tue, 14 Jun 2016 09:59:48 -0400 Subject: [PATCH 1/3] Add packages c-blosc and hdf5-blosc c-blosc provides the Blosc compression filter, and hdf5-blosc provides a plugin for HDF5 to use that filter. --- .../repos/builtin/packages/c-blosc/package.py | 47 +++++++++++ .../builtin/packages/hdf5-blosc/package.py | 77 +++++++++++++++++++ 2 files changed, 124 insertions(+) create mode 100644 var/spack/repos/builtin/packages/c-blosc/package.py create mode 100644 var/spack/repos/builtin/packages/hdf5-blosc/package.py diff --git a/var/spack/repos/builtin/packages/c-blosc/package.py b/var/spack/repos/builtin/packages/c-blosc/package.py new file mode 100644 index 0000000000..16e8c89811 --- /dev/null +++ b/var/spack/repos/builtin/packages/c-blosc/package.py @@ -0,0 +1,47 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + +from spack import * + +class CBlosc(Package): + """Blosc, an extremely fast, multi-threaded, meta-compressor library""" + homepage = "http://www.blosc.org" + url = "https://github.com/Blosc/c-blosc/archive/v1.9.2.tar.gz" + + version('1.9.2', 'dd2d83069d74b36b8093f1c6b49defc5') + version('1.9.1', '7d708d3daadfacf984a87b71b1734ce2') + version('1.9.0', 'e4c1dc8e2c468e5cfa2bf05eeee5357a') + version('1.8.1', 'd73d5be01359cf271e9386c90dcf5b05') + version('1.8.0', '5b92ecb287695ba20cc33d30bf221c4f') + + depends_on("cmake") + depends_on("snappy") + depends_on("zlib") + + def install(self, spec, prefix): + cmake('.', *std_cmake_args) + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/hdf5-blosc/package.py b/var/spack/repos/builtin/packages/hdf5-blosc/package.py new file mode 100644 index 0000000000..897c079e4e --- /dev/null +++ b/var/spack/repos/builtin/packages/hdf5-blosc/package.py @@ -0,0 +1,77 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + +import os +import shutil +import sys + +from spack import * + +class Hdf5Blosc(Package): + """Blosc filter for HDF5""" + homepage = "https://github.com/Blosc/hdf5-blosc" + url = "https://github.com/Blosc/hdf5-blosc/archive/master.zip" + + version('master', '02c04acbf4bec66ec8a35bf157d1c9de') + + depends_on("c-blosc") + depends_on("hdf5") + depends_on("libtool") + + parallel = False + + def install(self, spec, prefix): + cmake('.', *std_cmake_args) + + make() + make("install") + if sys.platform == 'darwin': + fix_darwin_install_name(prefix.lib) + + # Build and install plugin manually + # The plugin requires at least HDF5 1.8.11: + if spec['hdf5'].satisfies('@1.8.11:'): + # The plugin does not yet work with Windows: + if sys.platform not in ('win32', 'cygwin'): + # cc = which('cc') + libtool = Executable(join_path(spec['libtool'].prefix.bin, + 'libtool')) + with working_dir('src'): + libtool('--mode=compile', '--tag=CC', + 'cc', '-g', '-O', '-c', 'blosc_plugin.c') + libtool('--mode=link', '--tag=CC', + 'cc', '-g', '-O', + '-rpath', prefix.lib, + '-o', 'libblosc_plugin.la', 'blosc_plugin.lo') + shlibext = 'so' if sys.platform!='darwin' else 'dylib' + shlib0 = 'libblosc_plugin.0.%s' % shlibext + shutil.copyfile(join_path('.libs', shlib0), + join_path(prefix.lib, shlib0)) + shlib = 'libblosc_plugin.%s' % shlibext + os.symlink(shlib0, join_path(prefix.lib, shlib)) + + def setup_dependent_environment(self, spack_env, run_env, dependent_spec): + spack_env.append_path('HDF5_PLUGIN_PATH', self.spec.prefix.lib) + run_env.append_path('HDF5_PLUGIN_PATH', self.spec.prefix.lib) From ca5180f7a24d08d49f046a4efa8ef5b48201211d Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Tue, 14 Jun 2016 17:03:00 -0400 Subject: [PATCH 2/3] Correct Blosc install on Darwin --- .../repos/builtin/packages/c-blosc/package.py | 4 + .../builtin/packages/hdf5-blosc/package.py | 177 +++++++++++++++--- 2 files changed, 153 insertions(+), 28 deletions(-) diff --git a/var/spack/repos/builtin/packages/c-blosc/package.py b/var/spack/repos/builtin/packages/c-blosc/package.py index 16e8c89811..dee332be14 100644 --- a/var/spack/repos/builtin/packages/c-blosc/package.py +++ b/var/spack/repos/builtin/packages/c-blosc/package.py @@ -23,6 +23,8 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +import sys + from spack import * class CBlosc(Package): @@ -45,3 +47,5 @@ def install(self, spec, prefix): make() make("install") + if sys.platform == 'darwin': + fix_darwin_install_name(prefix.lib) diff --git a/var/spack/repos/builtin/packages/hdf5-blosc/package.py b/var/spack/repos/builtin/packages/hdf5-blosc/package.py index 897c079e4e..680bc41c41 100644 --- a/var/spack/repos/builtin/packages/hdf5-blosc/package.py +++ b/var/spack/repos/builtin/packages/hdf5-blosc/package.py @@ -43,35 +43,156 @@ class Hdf5Blosc(Package): parallel = False def install(self, spec, prefix): - cmake('.', *std_cmake_args) + # The included cmake recipe doesn"t work for Darwin + # cmake(".", *std_cmake_args) + # + # make() + # make("install") + # if sys.platform == "darwin": + # fix_darwin_install_name(prefix.lib) - make() - make("install") - if sys.platform == 'darwin': - fix_darwin_install_name(prefix.lib) + libtool = Executable(join_path(spec["libtool"].prefix.bin, "libtool")) + if "+mpi" in spec["hdf5"]: + cc = "mpicc" + else: + cc = "cc" + shlibext = "so" if sys.platform!="darwin" else "dylib" + mkdirp(prefix.include) + mkdirp(prefix.lib) - # Build and install plugin manually - # The plugin requires at least HDF5 1.8.11: - if spec['hdf5'].satisfies('@1.8.11:'): - # The plugin does not yet work with Windows: - if sys.platform not in ('win32', 'cygwin'): - # cc = which('cc') - libtool = Executable(join_path(spec['libtool'].prefix.bin, - 'libtool')) - with working_dir('src'): - libtool('--mode=compile', '--tag=CC', - 'cc', '-g', '-O', '-c', 'blosc_plugin.c') - libtool('--mode=link', '--tag=CC', - 'cc', '-g', '-O', - '-rpath', prefix.lib, - '-o', 'libblosc_plugin.la', 'blosc_plugin.lo') - shlibext = 'so' if sys.platform!='darwin' else 'dylib' - shlib0 = 'libblosc_plugin.0.%s' % shlibext - shutil.copyfile(join_path('.libs', shlib0), - join_path(prefix.lib, shlib0)) - shlib = 'libblosc_plugin.%s' % shlibext - os.symlink(shlib0, join_path(prefix.lib, shlib)) + # Build and install filter + with working_dir("src"): + libtool("--mode=compile", "--tag=CC", + "cc", "-g", "-O", + "-c", "blosc_filter.c") + libtool("--mode=link", "--tag=CC", + "cc", "-g", "-O", + "-rpath", prefix.lib, + "-o", "libblosc_filter.la", + "blosc_filter.lo", + "-L%s" % spec["c-blosc"].prefix.lib, "-lblosc", + "-L%s" % spec["hdf5"].prefix.lib, "-lhdf5") + shlib0 = "libblosc_filter.0.%s" % shlibext + shlib = "libblosc_filter.%s" % shlibext + shutil.copyfile(join_path(".libs", shlib0), + join_path(prefix.lib, shlib0)) + os.symlink(shlib0, join_path(prefix.lib, shlib)) + + # Build and install plugin + # The plugin requires at least HDF5 1.8.11: + if spec["hdf5"].satisfies("@1.8.11:"): + libtool("--mode=compile", "--tag=CC", + "cc", "-g", "-O", + "-c", "blosc_plugin.c") + libtool("--mode=link", "--tag=CC", + "cc", "-g", "-O", + "-rpath", prefix.lib, + "-o", "libblosc_plugin.la", + "blosc_plugin.lo", + "-L%s" % prefix.lib, "-lblosc_filter", + "-L%s" % spec["c-blosc"].prefix.lib, "-lblosc", + "-L%s" % spec["hdf5"].prefix.lib, "-lhdf5") + shlib0 = "libblosc_plugin.0.%s" % shlibext + shlib = "libblosc_plugin.%s" % shlibext + shutil.copyfile(join_path(".libs", shlib0), + join_path(prefix.lib, shlib0)) + os.symlink(shlib0, join_path(prefix.lib, shlib)) + + self.check_install(spec) + def check_install(self, spec): + "Build and run a small program to test the installed HDF5 Blosc plugin" + print "Checking HDF5-Blosc plugin..." + checkdir = "spack-check" + with working_dir(checkdir, create=True): + source = r"""\ +#include +#include +#include +#include + +#define FILTER_BLOSC 32001 /* Blosc filter ID registered with the HDF group */ + +int main(int argc, char **argv) { + herr_t herr; + hid_t file = H5Fcreate("file.h5", H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + assert(file >= 0); + hsize_t dims[3] = {10, 10, 10}; + hid_t space = H5Screate_simple(3, dims, NULL); + assert(space >= 0); + hid_t create_proplist = H5Pcreate(H5P_DATASET_CREATE); + assert(create_proplist >= 0); + herr = H5Pset_chunk(create_proplist, 3, dims); + assert(herr >= 0); + herr = H5Pset_filter(create_proplist, FILTER_BLOSC, H5Z_FLAG_OPTIONAL, 0, + NULL); + assert(herr >= 0); + htri_t all_filters_avail = H5Pall_filters_avail(create_proplist); + assert(all_filters_avail > 0); + hid_t dataset = H5Dcreate(file, "dataset", H5T_NATIVE_DOUBLE, space, + H5P_DEFAULT, create_proplist, H5P_DEFAULT); + assert(dataset >= 0); + double data[10][10][10]; + for (int k=0; k<10; ++k) { + for (int j=0; j<10; ++j) { + for (int i=0; i<10; ++i) { + data[k][j][i] = 1.0 / (1.0 + i + j + k); + } + } + } + herr = H5Dwrite(dataset, H5T_NATIVE_DOUBLE, space, space, H5P_DEFAULT, + &data[0][0][0]); + assert(herr >= 0); + herr = H5Pclose(create_proplist); + assert(herr >= 0); + herr = H5Dclose(dataset); + assert(herr >= 0); + herr = H5Sclose(space); + assert(herr >= 0); + herr = H5Fclose(file); + assert(herr >= 0); + printf("Done.\n"); + return 0; +} +""" + expected = """\ +Done. +""" + with open("check.c", "w") as f: + f.write(source) + if "+mpi" in spec["hdf5"]: + cc = which("mpicc") + else: + cc = which("cc") + # TODO: Automate these path and library settings + cc("-c", "-I%s" % spec["hdf5"].prefix.include, "check.c") + cc("-o", "check", "check.o", + "-L%s" % spec["hdf5"].prefix.lib, "-lhdf5") + try: + check = Executable("./check") + output = check(return_output=True) + except: + output = "" + success = output == expected + if not success: + print "Produced output does not match expected output." + print "Expected output:" + print "-"*80 + print expected + print "-"*80 + print "Produced output:" + print "-"*80 + print output + print "-"*80 + print "Environment:" + env = which("env") + env() + raise RuntimeError("HDF5 Blosc plugin check failed") + shutil.rmtree(checkdir) + + def setup_environment(self, spack_env, run_env): + spack_env.append_path("HDF5_PLUGIN_PATH", self.spec.prefix.lib) + run_env.append_path("HDF5_PLUGIN_PATH", self.spec.prefix.lib) def setup_dependent_environment(self, spack_env, run_env, dependent_spec): - spack_env.append_path('HDF5_PLUGIN_PATH', self.spec.prefix.lib) - run_env.append_path('HDF5_PLUGIN_PATH', self.spec.prefix.lib) + spack_env.append_path("HDF5_PLUGIN_PATH", self.spec.prefix.lib) + run_env.append_path("HDF5_PLUGIN_PATH", self.spec.prefix.lib) From 05d7378da57f51a26830a61a88228fb8fc7d3108 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Tue, 14 Jun 2016 17:13:20 -0400 Subject: [PATCH 3/3] Install shared libraries correctly on Linux --- .../builtin/packages/hdf5-blosc/package.py | 28 ++++++++++++------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/var/spack/repos/builtin/packages/hdf5-blosc/package.py b/var/spack/repos/builtin/packages/hdf5-blosc/package.py index 680bc41c41..50f380083c 100644 --- a/var/spack/repos/builtin/packages/hdf5-blosc/package.py +++ b/var/spack/repos/builtin/packages/hdf5-blosc/package.py @@ -29,6 +29,21 @@ from spack import * +def _install_shlib(name, src, dst): + """Install a shared library from directory src to directory dst""" + if sys.platform == "darwin": + shlib0 = name + ".0.dylib" + shlib = name + ".dylib" + shutil.copyfile(join_path(src, shlib0), join_path(dst, shlib0)) + os.symlink(shlib0, join_path(dst, shlib)) + else: + shlib000 = name + ".so.0.0.0" + shlib0 = name + ".so.0" + shlib = name + ".dylib" + shutil.copyfile(join_path(src, shlib000), join_path(dst, shlib000)) + os.symlink(shlib000, join_path(dst, shlib0)) + os.symlink(shlib0, join_path(dst, shlib)) + class Hdf5Blosc(Package): """Blosc filter for HDF5""" homepage = "https://github.com/Blosc/hdf5-blosc" @@ -72,11 +87,7 @@ def install(self, spec, prefix): "blosc_filter.lo", "-L%s" % spec["c-blosc"].prefix.lib, "-lblosc", "-L%s" % spec["hdf5"].prefix.lib, "-lhdf5") - shlib0 = "libblosc_filter.0.%s" % shlibext - shlib = "libblosc_filter.%s" % shlibext - shutil.copyfile(join_path(".libs", shlib0), - join_path(prefix.lib, shlib0)) - os.symlink(shlib0, join_path(prefix.lib, shlib)) + _install_shlib("libblosc_filter", ".libs", prefix.lib) # Build and install plugin # The plugin requires at least HDF5 1.8.11: @@ -92,13 +103,10 @@ def install(self, spec, prefix): "-L%s" % prefix.lib, "-lblosc_filter", "-L%s" % spec["c-blosc"].prefix.lib, "-lblosc", "-L%s" % spec["hdf5"].prefix.lib, "-lhdf5") - shlib0 = "libblosc_plugin.0.%s" % shlibext - shlib = "libblosc_plugin.%s" % shlibext - shutil.copyfile(join_path(".libs", shlib0), - join_path(prefix.lib, shlib0)) - os.symlink(shlib0, join_path(prefix.lib, shlib)) + _install_shlib("libblosc_plugin", ".libs", prefix.lib) self.check_install(spec) + def check_install(self, spec): "Build and run a small program to test the installed HDF5 Blosc plugin" print "Checking HDF5-Blosc plugin..."