From c6cd8ae243c07fa80d54207c334883585eca0d97 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 1 Oct 2015 21:35:25 -0700 Subject: [PATCH 001/481] Use more recent version of libffi. 3.1 has a known bug that is fixed in current version --- var/spack/packages/libffi/package.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/var/spack/packages/libffi/package.py b/var/spack/packages/libffi/package.py index 2c1c4eed4d..18acabb00c 100644 --- a/var/spack/packages/libffi/package.py +++ b/var/spack/packages/libffi/package.py @@ -6,9 +6,9 @@ class Libffi(Package): to call any function specified by a call interface description at run time.""" homepage = "https://sourceware.org/libffi/" - url = "ftp://sourceware.org/pub/libffi/libffi-3.1.tar.gz" - - version('3.1', 'f5898b29bbfd70502831a212d9249d10') + + version('3.2.1','83b89587607e3eb65c70d361f13bab43',url = "ftp://sourceware.org/pub/libffi/libffi-3.2.1.tar.gz") + #version('3.1', 'f5898b29bbfd70502831a212d9249d10',url = "ftp://sourceware.org/pub/libffi/libffi-3.1.tar.gz") # Has a bug $(lib64) instead of ${lib64} in libffi.pc def install(self, spec, prefix): configure("--prefix=%s" % prefix) From 1501786fd9be0a01abe7f6bb917e416fb3256aa6 Mon Sep 17 00:00:00 2001 From: karenyyng Date: Sat, 3 Oct 2015 13:04:37 -0700 Subject: [PATCH 002/481] adding python_recipe_parser as a submodule under the folder utils --- .gitmodules | 3 +++ utils/python_recipe_parser | 1 + 2 files changed, 4 insertions(+) create mode 100644 .gitmodules create mode 160000 utils/python_recipe_parser diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000000..a097fcd8cf --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "utils/python_recipe_parser"] + path = utils/python_recipe_parser + url = https://github.com/karenyyng/spack_python_package_parser diff --git a/utils/python_recipe_parser b/utils/python_recipe_parser new file mode 160000 index 0000000000..3158074a2d --- /dev/null +++ b/utils/python_recipe_parser @@ -0,0 +1 @@ +Subproject commit 3158074a2d74f99aa72289dfcf093b1b0ba4474e From 79808da760eabcb3e692c88e29ea3c7c558b131d Mon Sep 17 00:00:00 2001 From: karenyyng Date: Sat, 3 Oct 2015 18:47:36 -0700 Subject: [PATCH 003/481] added a bunch of packages and added submodule --- .gitignore | 1 + .gitmodules | 4 +- utils/python_recipe_parser | 1 - var/spack/packages/ImageMagick/package.py | 0 var/spack/packages/Mitos/package.py | 0 var/spack/packages/R/package.py | 0 var/spack/packages/SAMRAI/no-tool-build.patch | 0 var/spack/packages/SAMRAI/package.py | 0 var/spack/packages/adept-utils/package.py | 0 var/spack/packages/arpack/package.py | 0 var/spack/packages/asciidoc/package.py | 18 --------- var/spack/packages/atk/package.py | 0 var/spack/packages/atlas/package.py | 0 var/spack/packages/autoconf/package.py | 0 var/spack/packages/automaded/package.py | 0 var/spack/packages/automake/package.py | 0 var/spack/packages/bear/package.py | 0 var/spack/packages/bib2xhtml/package.py | 0 var/spack/packages/binutils/package.py | 0 var/spack/packages/bison/package.py | 0 var/spack/packages/boost/package.py | 4 -- var/spack/packages/bowtie2/bowtie2-2.5.patch | 16 -------- var/spack/packages/bowtie2/package.py | 24 ------------ var/spack/packages/boxlib/package.py | 0 var/spack/packages/bzip2/package.py | 0 var/spack/packages/cairo/package.py | 0 var/spack/packages/callpath/package.py | 0 var/spack/packages/cblas/package.py | 0 var/spack/packages/cgm/package.py | 0 var/spack/packages/clang/package.py | 0 var/spack/packages/cloog/package.py | 0 var/spack/packages/cmake/package.py | 0 var/spack/packages/coreutils/package.py | 0 var/spack/packages/cppcheck/package.py | 0 var/spack/packages/cram/package.py | 0 var/spack/packages/cscope/package.py | 17 -------- var/spack/packages/cube/package.py | 0 var/spack/packages/czmq/package.py | 19 --------- var/spack/packages/dbus/package.py | 0 var/spack/packages/docbook-xml/package.py | 19 --------- var/spack/packages/doxygen/package.py | 25 ------------ var/spack/packages/dri2proto/package.py | 0 var/spack/packages/dtcmp/package.py | 0 var/spack/packages/dyninst/package.py | 0 var/spack/packages/eigen/package.py | 17 ++++++++ var/spack/packages/elfutils/package.py | 26 ------------- var/spack/packages/extrae/package.py | 0 var/spack/packages/exuberant-ctags/package.py | 0 var/spack/packages/fish/package.py | 18 --------- var/spack/packages/flex/package.py | 0 var/spack/packages/flux/package.py | 36 ----------------- var/spack/packages/fontconfig/package.py | 0 var/spack/packages/freetype/package.py | 0 var/spack/packages/gasnet/package.py | 0 var/spack/packages/gcc/package.py | 0 var/spack/packages/gdk-pixbuf/package.py | 0 var/spack/packages/geos/package.py | 0 var/spack/packages/gflags/package.py | 21 ---------- var/spack/packages/ghostscript/package.py | 17 -------- var/spack/packages/git/package.py | 0 var/spack/packages/glib/package.py | 0 var/spack/packages/global/package.py | 0 var/spack/packages/glog/package.py | 15 ------- var/spack/packages/gmp/package.py | 0 var/spack/packages/gnutls/package.py | 0 var/spack/packages/gperf/package.py | 0 var/spack/packages/gperftools/package.py | 0 var/spack/packages/graphlib/package.py | 0 var/spack/packages/graphviz/package.py | 19 +++++---- var/spack/packages/gtkplus/package.py | 0 var/spack/packages/harfbuzz/package.py | 0 var/spack/packages/hdf5/package.py | 0 var/spack/packages/hwloc/package.py | 0 var/spack/packages/hypre/package.py | 0 var/spack/packages/icu/package.py | 0 var/spack/packages/icu4c/package.py | 0 var/spack/packages/isl/package.py | 0 var/spack/packages/jdk/package.py | 0 var/spack/packages/jpeg/package.py | 0 .../{netlib-lapack => lapack}/package.py | 18 +-------- var/spack/packages/launchmon/package.py | 0 .../packages/launchmon/patch.lmon_install_dir | 0 var/spack/packages/lcms/package.py | 0 var/spack/packages/leveldb/package.py | 29 -------------- var/spack/packages/libNBC/package.py | 0 var/spack/packages/libarchive/package.py | 0 var/spack/packages/libcircle/package.py | 0 var/spack/packages/libdrm/package.py | 5 ++- var/spack/packages/libdwarf/package.py | 0 var/spack/packages/libelf/package.py | 2 - var/spack/packages/libevent/package.py | 0 var/spack/packages/libffi/package.py | 6 +-- var/spack/packages/libgcrypt/package.py | 0 var/spack/packages/libgpg-error/package.py | 0 var/spack/packages/libjpeg-turbo/package.py | 0 var/spack/packages/libjson-c/package.py | 14 ------- var/spack/packages/libmng/package.py | 0 var/spack/packages/libmonitor/package.py | 0 var/spack/packages/libpciaccess/package.py | 17 +++----- var/spack/packages/libpng/package.py | 0 var/spack/packages/libsodium/package.py | 19 --------- var/spack/packages/libtiff/package.py | 0 var/spack/packages/libtool/package.py | 0 var/spack/packages/libunwind/package.py | 0 var/spack/packages/libuuid/package.py | 0 var/spack/packages/libxcb/package.py | 0 var/spack/packages/libxml2/package.py | 5 +-- var/spack/packages/libxshmfence/package.py | 0 var/spack/packages/libxslt/package.py | 0 var/spack/packages/llvm-lld/package.py | 0 var/spack/packages/llvm/package.py | 0 var/spack/packages/lmdb/package.py | 39 ------------------- var/spack/packages/lua/package.py | 26 ------------- var/spack/packages/lwgrp/package.py | 0 var/spack/packages/lwm2/package.py | 0 var/spack/packages/memaxes/package.py | 0 var/spack/packages/mesa/package.py | 5 +-- var/spack/packages/metis/package.py | 0 var/spack/packages/mpc/package.py | 0 var/spack/packages/mpe2/mpe2.patch | 0 var/spack/packages/mpe2/package.py | 0 var/spack/packages/mpfr/package.py | 0 var/spack/packages/mpibash/mpibash-4.3.patch | 0 var/spack/packages/mpibash/package.py | 0 var/spack/packages/mpich/package.py | 1 + var/spack/packages/mpileaks/package.py | 0 var/spack/packages/mrnet/package.py | 0 var/spack/packages/munge/package.py | 20 ---------- var/spack/packages/muster/package.py | 0 .../ad_lustre_rwcontig_open_source.patch | 0 var/spack/packages/mvapich2/package.py | 0 var/spack/packages/nasm/package.py | 0 var/spack/packages/ncurses/package.py | 8 ---- var/spack/packages/netcdf/package.py | 0 var/spack/packages/netgauge/package.py | 0 var/spack/packages/netlib-blas/package.py | 6 --- var/spack/packages/nettle/package.py | 0 var/spack/packages/ompss/package.py | 0 var/spack/packages/opari2/package.py | 0 .../ad_lustre_rwcontig_open_source.patch | 0 .../packages/openmpi/llnl-platforms.patch | 0 var/spack/packages/openmpi/package.py | 0 var/spack/packages/openssl/package.py | 0 var/spack/packages/otf/package.py | 0 var/spack/packages/otf2/package.py | 0 var/spack/packages/pango/package.py | 0 var/spack/packages/papi/package.py | 0 var/spack/packages/paraver/package.py | 0 var/spack/packages/parmetis/package.py | 0 var/spack/packages/parpack/package.py | 0 var/spack/packages/pcre/package.py | 0 var/spack/packages/petsc/package.py | 0 var/spack/packages/pidx/package.py | 21 ---------- var/spack/packages/pixman/package.py | 0 var/spack/packages/pkg-config/package.py | 17 -------- var/spack/packages/pmgr_collective/package.py | 0 var/spack/packages/postgresql/package.py | 0 var/spack/packages/ppl/package.py | 0 var/spack/packages/protobuf/package.py | 16 -------- var/spack/packages/py-Cython/package.py | 12 ++++++ var/spack/packages/py-Distutils2/package.py | 15 +++++++ var/spack/packages/py-astroML/package.py | 12 ++++++ var/spack/packages/py-astropy/package.py | 12 ++++++ var/spack/packages/py-autopep8/package.py | 12 ++++++ var/spack/packages/py-basemap/package.py | 0 var/spack/packages/py-biopython/package.py | 0 var/spack/packages/py-certifi/package.py | 12 ++++++ var/spack/packages/py-cffi/package.py | 0 var/spack/packages/py-cython/package.py | 0 var/spack/packages/py-dateutil/package.py | 0 var/spack/packages/py-decorator/package.py | 12 ++++++ var/spack/packages/py-emcee/package.py | 12 ++++++ var/spack/packages/py-epydoc/package.py | 0 var/spack/packages/py-fitsio/package.py | 13 +++++++ var/spack/packages/py-genders/package.py | 0 var/spack/packages/py-gnuplot/package.py | 0 var/spack/packages/py-gnureadline/package.py | 12 ++++++ var/spack/packages/py-h5py/package.py | 0 var/spack/packages/py-ipython/package.py | 21 ++++++++-- .../packages/py-ipython/package.py.backup | 27 +++++++++++++ .../packages/py-ipython_genutils/package.py | 12 ++++++ var/spack/packages/py-joblib/package.py | 12 ++++++ var/spack/packages/py-jsonschema/package.py | 16 ++++++++ var/spack/packages/py-libxml2/package.py | 0 var/spack/packages/py-lockfile/package.py | 0 var/spack/packages/py-mako/package.py | 0 var/spack/packages/py-matplotlib/package.py | 1 - var/spack/packages/py-mock/package.py | 17 +++----- var/spack/packages/py-mpi4py/package.py | 0 var/spack/packages/py-mx/package.py | 0 var/spack/packages/py-nose/package.py | 0 var/spack/packages/py-numexpr/package.py | 12 ++++++ var/spack/packages/py-numpy/package.py | 15 +------ var/spack/packages/py-pandas/package.py | 2 + var/spack/packages/py-pep8/package.py | 12 ++++++ var/spack/packages/py-pexpect/package.py | 0 var/spack/packages/py-pickleshare/package.py | 12 ++++++ var/spack/packages/py-pil/package.py | 0 var/spack/packages/py-pip/package.py | 12 ++++++ var/spack/packages/py-pmw/package.py | 0 var/spack/packages/py-py/package.py | 12 ++++++ var/spack/packages/py-py4j/package.py | 15 +++++++ var/spack/packages/py-pychecker/package.py | 0 var/spack/packages/py-pycparser/package.py | 0 var/spack/packages/py-pyelftools/package.py | 0 var/spack/packages/py-pygments/package.py | 0 var/spack/packages/py-pylint/package.py | 0 var/spack/packages/py-pypar/package.py | 0 var/spack/packages/py-pyparsing/package.py | 0 var/spack/packages/py-pyqt/package.py | 0 var/spack/packages/py-pyside/package.py | 5 +-- var/spack/packages/py-pytest/package.py | 12 ++++++ .../packages/py-python-daemon/package.py | 0 .../packages/py-python-dateutil/package.py | 12 ++++++ var/spack/packages/py-pytz/package.py | 2 + var/spack/packages/py-pyzmq/package.py | 16 ++++++++ var/spack/packages/py-rpy2/package.py | 0 .../packages/py-scientificpython/package.py | 0 var/spack/packages/py-scikit-learn/package.py | 3 ++ var/spack/packages/py-scipy/package.py | 4 +- var/spack/packages/py-seaborn/package.py | 12 ++++++ var/spack/packages/py-setuptools/package.py | 9 +++-- var/spack/packages/py-shiboken/package.py | 0 var/spack/packages/py-sip/package.py | 0 var/spack/packages/py-six/package.py | 2 + var/spack/packages/py-statsmodels/package.py | 12 ++++++ var/spack/packages/py-sympy/package.py | 0 var/spack/packages/py-tables/package.py | 12 ++++++ var/spack/packages/py-tornado/package.py | 17 ++++++++ var/spack/packages/py-traitlets/package.py | 14 +++++++ .../packages/py-triangle-plot/package.py | 12 ++++++ var/spack/packages/py-virtualenv/package.py | 0 var/spack/packages/py-yapf/package.py | 0 var/spack/packages/python/package.py | 0 var/spack/packages/qhull/package.py | 0 var/spack/packages/qt/package.py | 0 var/spack/packages/qthreads/package.py | 0 var/spack/packages/ravel/package.py | 0 var/spack/packages/readline/package.py | 0 .../rose/add_spack_compiler_recognition.patch | 0 var/spack/packages/rose/package.py | 0 var/spack/packages/ruby/package.py | 0 var/spack/packages/samtools/package.py | 18 --------- var/spack/packages/samtools/samtools1.2.patch | 20 ---------- var/spack/packages/scalasca/package.py | 0 var/spack/packages/scorep/package.py | 15 +++---- var/spack/packages/scotch/package.py | 0 var/spack/packages/scr/package.py | 0 var/spack/packages/silo/package.py | 0 var/spack/packages/snappy/package.py | 15 ------- var/spack/packages/spindle/package.py | 0 var/spack/packages/sqlite/package.py | 0 .../packages/stat/configure_mpicxx.patch | 0 var/spack/packages/stat/package.py | 11 +----- var/spack/packages/sundials/package.py | 0 var/spack/packages/swig/package.py | 0 var/spack/packages/task/package.py | 0 var/spack/packages/taskd/package.py | 0 var/spack/packages/tau/package.py | 0 var/spack/packages/tcl/package.py | 0 .../packages/the_silver_searcher/package.py | 0 var/spack/packages/thrift/package.py | 0 var/spack/packages/tk/package.py | 0 var/spack/packages/tmux/package.py | 0 var/spack/packages/tmuxinator/package.py | 0 var/spack/packages/uncrustify/package.py | 0 var/spack/packages/util-linux/package.py | 0 var/spack/packages/vim/package.py | 23 ++--------- var/spack/packages/vtk/package.py | 0 var/spack/packages/wget/package.py | 0 var/spack/packages/wx/package.py | 0 var/spack/packages/wxpropgrid/package.py | 0 var/spack/packages/xcb-proto/package.py | 0 var/spack/packages/xz/package.py | 0 var/spack/packages/yasm/package.py | 0 var/spack/packages/zeromq/package.py | 20 ---------- var/spack/packages/zlib/package.py | 0 var/spack/packages/zsh/package.py | 0 278 files changed, 491 insertions(+), 688 deletions(-) delete mode 160000 utils/python_recipe_parser mode change 100644 => 100755 var/spack/packages/ImageMagick/package.py mode change 100644 => 100755 var/spack/packages/Mitos/package.py mode change 100644 => 100755 var/spack/packages/R/package.py mode change 100644 => 100755 var/spack/packages/SAMRAI/no-tool-build.patch mode change 100644 => 100755 var/spack/packages/SAMRAI/package.py mode change 100644 => 100755 var/spack/packages/adept-utils/package.py mode change 100644 => 100755 var/spack/packages/arpack/package.py delete mode 100644 var/spack/packages/asciidoc/package.py mode change 100644 => 100755 var/spack/packages/atk/package.py mode change 100644 => 100755 var/spack/packages/atlas/package.py mode change 100644 => 100755 var/spack/packages/autoconf/package.py mode change 100644 => 100755 var/spack/packages/automaded/package.py mode change 100644 => 100755 var/spack/packages/automake/package.py mode change 100644 => 100755 var/spack/packages/bear/package.py mode change 100644 => 100755 var/spack/packages/bib2xhtml/package.py mode change 100644 => 100755 var/spack/packages/binutils/package.py mode change 100644 => 100755 var/spack/packages/bison/package.py mode change 100644 => 100755 var/spack/packages/boost/package.py delete mode 100644 var/spack/packages/bowtie2/bowtie2-2.5.patch delete mode 100644 var/spack/packages/bowtie2/package.py mode change 100644 => 100755 var/spack/packages/boxlib/package.py mode change 100644 => 100755 var/spack/packages/bzip2/package.py mode change 100644 => 100755 var/spack/packages/cairo/package.py mode change 100644 => 100755 var/spack/packages/callpath/package.py mode change 100644 => 100755 var/spack/packages/cblas/package.py mode change 100644 => 100755 var/spack/packages/cgm/package.py mode change 100644 => 100755 var/spack/packages/clang/package.py mode change 100644 => 100755 var/spack/packages/cloog/package.py mode change 100644 => 100755 var/spack/packages/cmake/package.py mode change 100644 => 100755 var/spack/packages/coreutils/package.py mode change 100644 => 100755 var/spack/packages/cppcheck/package.py mode change 100644 => 100755 var/spack/packages/cram/package.py delete mode 100644 var/spack/packages/cscope/package.py mode change 100644 => 100755 var/spack/packages/cube/package.py delete mode 100644 var/spack/packages/czmq/package.py mode change 100644 => 100755 var/spack/packages/dbus/package.py delete mode 100644 var/spack/packages/docbook-xml/package.py delete mode 100644 var/spack/packages/doxygen/package.py mode change 100644 => 100755 var/spack/packages/dri2proto/package.py mode change 100644 => 100755 var/spack/packages/dtcmp/package.py mode change 100644 => 100755 var/spack/packages/dyninst/package.py create mode 100644 var/spack/packages/eigen/package.py delete mode 100644 var/spack/packages/elfutils/package.py mode change 100644 => 100755 var/spack/packages/extrae/package.py mode change 100644 => 100755 var/spack/packages/exuberant-ctags/package.py delete mode 100644 var/spack/packages/fish/package.py mode change 100644 => 100755 var/spack/packages/flex/package.py delete mode 100644 var/spack/packages/flux/package.py mode change 100644 => 100755 var/spack/packages/fontconfig/package.py mode change 100644 => 100755 var/spack/packages/freetype/package.py mode change 100644 => 100755 var/spack/packages/gasnet/package.py mode change 100644 => 100755 var/spack/packages/gcc/package.py mode change 100644 => 100755 var/spack/packages/gdk-pixbuf/package.py mode change 100644 => 100755 var/spack/packages/geos/package.py delete mode 100644 var/spack/packages/gflags/package.py delete mode 100644 var/spack/packages/ghostscript/package.py mode change 100644 => 100755 var/spack/packages/git/package.py mode change 100644 => 100755 var/spack/packages/glib/package.py mode change 100644 => 100755 var/spack/packages/global/package.py delete mode 100644 var/spack/packages/glog/package.py mode change 100644 => 100755 var/spack/packages/gmp/package.py mode change 100644 => 100755 var/spack/packages/gnutls/package.py mode change 100644 => 100755 var/spack/packages/gperf/package.py mode change 100644 => 100755 var/spack/packages/gperftools/package.py mode change 100644 => 100755 var/spack/packages/graphlib/package.py mode change 100644 => 100755 var/spack/packages/gtkplus/package.py mode change 100644 => 100755 var/spack/packages/harfbuzz/package.py mode change 100644 => 100755 var/spack/packages/hdf5/package.py mode change 100644 => 100755 var/spack/packages/hwloc/package.py mode change 100644 => 100755 var/spack/packages/hypre/package.py mode change 100644 => 100755 var/spack/packages/icu/package.py mode change 100644 => 100755 var/spack/packages/icu4c/package.py mode change 100644 => 100755 var/spack/packages/isl/package.py mode change 100644 => 100755 var/spack/packages/jdk/package.py mode change 100644 => 100755 var/spack/packages/jpeg/package.py rename var/spack/packages/{netlib-lapack => lapack}/package.py (77%) mode change 100644 => 100755 mode change 100644 => 100755 var/spack/packages/launchmon/package.py mode change 100644 => 100755 var/spack/packages/launchmon/patch.lmon_install_dir mode change 100644 => 100755 var/spack/packages/lcms/package.py delete mode 100644 var/spack/packages/leveldb/package.py mode change 100644 => 100755 var/spack/packages/libNBC/package.py mode change 100644 => 100755 var/spack/packages/libarchive/package.py mode change 100644 => 100755 var/spack/packages/libcircle/package.py mode change 100644 => 100755 var/spack/packages/libdrm/package.py mode change 100644 => 100755 var/spack/packages/libdwarf/package.py mode change 100644 => 100755 var/spack/packages/libelf/package.py mode change 100644 => 100755 var/spack/packages/libevent/package.py mode change 100644 => 100755 var/spack/packages/libffi/package.py mode change 100644 => 100755 var/spack/packages/libgcrypt/package.py mode change 100644 => 100755 var/spack/packages/libgpg-error/package.py mode change 100644 => 100755 var/spack/packages/libjpeg-turbo/package.py delete mode 100644 var/spack/packages/libjson-c/package.py mode change 100644 => 100755 var/spack/packages/libmng/package.py mode change 100644 => 100755 var/spack/packages/libmonitor/package.py mode change 100644 => 100755 var/spack/packages/libpciaccess/package.py mode change 100644 => 100755 var/spack/packages/libpng/package.py delete mode 100644 var/spack/packages/libsodium/package.py mode change 100644 => 100755 var/spack/packages/libtiff/package.py mode change 100644 => 100755 var/spack/packages/libtool/package.py mode change 100644 => 100755 var/spack/packages/libunwind/package.py mode change 100644 => 100755 var/spack/packages/libuuid/package.py mode change 100644 => 100755 var/spack/packages/libxcb/package.py mode change 100644 => 100755 var/spack/packages/libxml2/package.py mode change 100644 => 100755 var/spack/packages/libxshmfence/package.py mode change 100644 => 100755 var/spack/packages/libxslt/package.py mode change 100644 => 100755 var/spack/packages/llvm-lld/package.py mode change 100644 => 100755 var/spack/packages/llvm/package.py delete mode 100644 var/spack/packages/lmdb/package.py delete mode 100644 var/spack/packages/lua/package.py mode change 100644 => 100755 var/spack/packages/lwgrp/package.py mode change 100644 => 100755 var/spack/packages/lwm2/package.py mode change 100644 => 100755 var/spack/packages/memaxes/package.py mode change 100644 => 100755 var/spack/packages/mesa/package.py mode change 100644 => 100755 var/spack/packages/metis/package.py mode change 100644 => 100755 var/spack/packages/mpc/package.py mode change 100644 => 100755 var/spack/packages/mpe2/mpe2.patch mode change 100644 => 100755 var/spack/packages/mpe2/package.py mode change 100644 => 100755 var/spack/packages/mpfr/package.py mode change 100644 => 100755 var/spack/packages/mpibash/mpibash-4.3.patch mode change 100644 => 100755 var/spack/packages/mpibash/package.py mode change 100644 => 100755 var/spack/packages/mpich/package.py mode change 100644 => 100755 var/spack/packages/mpileaks/package.py mode change 100644 => 100755 var/spack/packages/mrnet/package.py delete mode 100644 var/spack/packages/munge/package.py mode change 100644 => 100755 var/spack/packages/muster/package.py mode change 100644 => 100755 var/spack/packages/mvapich2/ad_lustre_rwcontig_open_source.patch mode change 100644 => 100755 var/spack/packages/mvapich2/package.py mode change 100644 => 100755 var/spack/packages/nasm/package.py mode change 100644 => 100755 var/spack/packages/ncurses/package.py mode change 100644 => 100755 var/spack/packages/netcdf/package.py mode change 100644 => 100755 var/spack/packages/netgauge/package.py mode change 100644 => 100755 var/spack/packages/netlib-blas/package.py mode change 100644 => 100755 var/spack/packages/nettle/package.py mode change 100644 => 100755 var/spack/packages/ompss/package.py mode change 100644 => 100755 var/spack/packages/opari2/package.py mode change 100644 => 100755 var/spack/packages/openmpi/ad_lustre_rwcontig_open_source.patch mode change 100644 => 100755 var/spack/packages/openmpi/llnl-platforms.patch mode change 100644 => 100755 var/spack/packages/openmpi/package.py mode change 100644 => 100755 var/spack/packages/openssl/package.py mode change 100644 => 100755 var/spack/packages/otf/package.py mode change 100644 => 100755 var/spack/packages/otf2/package.py mode change 100644 => 100755 var/spack/packages/pango/package.py mode change 100644 => 100755 var/spack/packages/papi/package.py mode change 100644 => 100755 var/spack/packages/paraver/package.py mode change 100644 => 100755 var/spack/packages/parmetis/package.py mode change 100644 => 100755 var/spack/packages/parpack/package.py mode change 100644 => 100755 var/spack/packages/pcre/package.py mode change 100644 => 100755 var/spack/packages/petsc/package.py delete mode 100644 var/spack/packages/pidx/package.py mode change 100644 => 100755 var/spack/packages/pixman/package.py delete mode 100644 var/spack/packages/pkg-config/package.py mode change 100644 => 100755 var/spack/packages/pmgr_collective/package.py mode change 100644 => 100755 var/spack/packages/postgresql/package.py mode change 100644 => 100755 var/spack/packages/ppl/package.py delete mode 100644 var/spack/packages/protobuf/package.py create mode 100644 var/spack/packages/py-Cython/package.py create mode 100644 var/spack/packages/py-Distutils2/package.py create mode 100644 var/spack/packages/py-astroML/package.py create mode 100644 var/spack/packages/py-astropy/package.py create mode 100644 var/spack/packages/py-autopep8/package.py mode change 100644 => 100755 var/spack/packages/py-basemap/package.py mode change 100644 => 100755 var/spack/packages/py-biopython/package.py create mode 100644 var/spack/packages/py-certifi/package.py mode change 100644 => 100755 var/spack/packages/py-cffi/package.py mode change 100644 => 100755 var/spack/packages/py-cython/package.py mode change 100644 => 100755 var/spack/packages/py-dateutil/package.py create mode 100644 var/spack/packages/py-decorator/package.py create mode 100644 var/spack/packages/py-emcee/package.py mode change 100644 => 100755 var/spack/packages/py-epydoc/package.py create mode 100644 var/spack/packages/py-fitsio/package.py mode change 100644 => 100755 var/spack/packages/py-genders/package.py mode change 100644 => 100755 var/spack/packages/py-gnuplot/package.py create mode 100644 var/spack/packages/py-gnureadline/package.py mode change 100644 => 100755 var/spack/packages/py-h5py/package.py mode change 100644 => 100755 var/spack/packages/py-ipython/package.py create mode 100755 var/spack/packages/py-ipython/package.py.backup create mode 100644 var/spack/packages/py-ipython_genutils/package.py create mode 100644 var/spack/packages/py-joblib/package.py create mode 100755 var/spack/packages/py-jsonschema/package.py mode change 100644 => 100755 var/spack/packages/py-libxml2/package.py mode change 100644 => 100755 var/spack/packages/py-lockfile/package.py mode change 100644 => 100755 var/spack/packages/py-mako/package.py mode change 100644 => 100755 var/spack/packages/py-matplotlib/package.py mode change 100644 => 100755 var/spack/packages/py-mpi4py/package.py mode change 100644 => 100755 var/spack/packages/py-mx/package.py mode change 100644 => 100755 var/spack/packages/py-nose/package.py create mode 100644 var/spack/packages/py-numexpr/package.py mode change 100644 => 100755 var/spack/packages/py-numpy/package.py mode change 100644 => 100755 var/spack/packages/py-pandas/package.py create mode 100644 var/spack/packages/py-pep8/package.py mode change 100644 => 100755 var/spack/packages/py-pexpect/package.py create mode 100644 var/spack/packages/py-pickleshare/package.py mode change 100644 => 100755 var/spack/packages/py-pil/package.py create mode 100644 var/spack/packages/py-pip/package.py mode change 100644 => 100755 var/spack/packages/py-pmw/package.py create mode 100644 var/spack/packages/py-py/package.py create mode 100755 var/spack/packages/py-py4j/package.py mode change 100644 => 100755 var/spack/packages/py-pychecker/package.py mode change 100644 => 100755 var/spack/packages/py-pycparser/package.py mode change 100644 => 100755 var/spack/packages/py-pyelftools/package.py mode change 100644 => 100755 var/spack/packages/py-pygments/package.py mode change 100644 => 100755 var/spack/packages/py-pylint/package.py mode change 100644 => 100755 var/spack/packages/py-pypar/package.py mode change 100644 => 100755 var/spack/packages/py-pyparsing/package.py mode change 100644 => 100755 var/spack/packages/py-pyqt/package.py mode change 100644 => 100755 var/spack/packages/py-pyside/package.py create mode 100644 var/spack/packages/py-pytest/package.py mode change 100644 => 100755 var/spack/packages/py-python-daemon/package.py create mode 100644 var/spack/packages/py-python-dateutil/package.py mode change 100644 => 100755 var/spack/packages/py-pytz/package.py create mode 100755 var/spack/packages/py-pyzmq/package.py mode change 100644 => 100755 var/spack/packages/py-rpy2/package.py mode change 100644 => 100755 var/spack/packages/py-scientificpython/package.py mode change 100644 => 100755 var/spack/packages/py-scikit-learn/package.py mode change 100644 => 100755 var/spack/packages/py-scipy/package.py create mode 100644 var/spack/packages/py-seaborn/package.py mode change 100644 => 100755 var/spack/packages/py-setuptools/package.py mode change 100644 => 100755 var/spack/packages/py-shiboken/package.py mode change 100644 => 100755 var/spack/packages/py-sip/package.py mode change 100644 => 100755 var/spack/packages/py-six/package.py create mode 100644 var/spack/packages/py-statsmodels/package.py mode change 100644 => 100755 var/spack/packages/py-sympy/package.py create mode 100644 var/spack/packages/py-tables/package.py create mode 100755 var/spack/packages/py-tornado/package.py create mode 100755 var/spack/packages/py-traitlets/package.py create mode 100644 var/spack/packages/py-triangle-plot/package.py mode change 100644 => 100755 var/spack/packages/py-virtualenv/package.py mode change 100644 => 100755 var/spack/packages/py-yapf/package.py mode change 100644 => 100755 var/spack/packages/python/package.py mode change 100644 => 100755 var/spack/packages/qhull/package.py mode change 100644 => 100755 var/spack/packages/qt/package.py mode change 100644 => 100755 var/spack/packages/qthreads/package.py mode change 100644 => 100755 var/spack/packages/ravel/package.py mode change 100644 => 100755 var/spack/packages/readline/package.py mode change 100644 => 100755 var/spack/packages/rose/add_spack_compiler_recognition.patch mode change 100644 => 100755 var/spack/packages/rose/package.py mode change 100644 => 100755 var/spack/packages/ruby/package.py delete mode 100644 var/spack/packages/samtools/package.py delete mode 100644 var/spack/packages/samtools/samtools1.2.patch mode change 100644 => 100755 var/spack/packages/scalasca/package.py mode change 100644 => 100755 var/spack/packages/scorep/package.py mode change 100644 => 100755 var/spack/packages/scotch/package.py mode change 100644 => 100755 var/spack/packages/scr/package.py mode change 100644 => 100755 var/spack/packages/silo/package.py delete mode 100644 var/spack/packages/snappy/package.py mode change 100644 => 100755 var/spack/packages/spindle/package.py mode change 100644 => 100755 var/spack/packages/sqlite/package.py mode change 100644 => 100755 var/spack/packages/stat/configure_mpicxx.patch mode change 100644 => 100755 var/spack/packages/stat/package.py mode change 100644 => 100755 var/spack/packages/sundials/package.py mode change 100644 => 100755 var/spack/packages/swig/package.py mode change 100644 => 100755 var/spack/packages/task/package.py mode change 100644 => 100755 var/spack/packages/taskd/package.py mode change 100644 => 100755 var/spack/packages/tau/package.py mode change 100644 => 100755 var/spack/packages/tcl/package.py mode change 100644 => 100755 var/spack/packages/the_silver_searcher/package.py mode change 100644 => 100755 var/spack/packages/thrift/package.py mode change 100644 => 100755 var/spack/packages/tk/package.py mode change 100644 => 100755 var/spack/packages/tmux/package.py mode change 100644 => 100755 var/spack/packages/tmuxinator/package.py mode change 100644 => 100755 var/spack/packages/uncrustify/package.py mode change 100644 => 100755 var/spack/packages/util-linux/package.py mode change 100644 => 100755 var/spack/packages/vim/package.py mode change 100644 => 100755 var/spack/packages/vtk/package.py mode change 100644 => 100755 var/spack/packages/wget/package.py mode change 100644 => 100755 var/spack/packages/wx/package.py mode change 100644 => 100755 var/spack/packages/wxpropgrid/package.py mode change 100644 => 100755 var/spack/packages/xcb-proto/package.py mode change 100644 => 100755 var/spack/packages/xz/package.py mode change 100644 => 100755 var/spack/packages/yasm/package.py delete mode 100644 var/spack/packages/zeromq/package.py mode change 100644 => 100755 var/spack/packages/zlib/package.py mode change 100644 => 100755 var/spack/packages/zsh/package.py diff --git a/.gitignore b/.gitignore index 1c6ca4c99e..8146921fd8 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ /etc/spackconfig /share/spack/dotkit /share/spack/modules +*.ropeproject diff --git a/.gitmodules b/.gitmodules index a097fcd8cf..1d770f415b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ -[submodule "utils/python_recipe_parser"] - path = utils/python_recipe_parser +[submodule "/lib/spack/spack/util/python_recipe_parser"] + path = /lib/spack/spack/util/python_recipe_parser url = https://github.com/karenyyng/spack_python_package_parser diff --git a/utils/python_recipe_parser b/utils/python_recipe_parser deleted file mode 160000 index 3158074a2d..0000000000 --- a/utils/python_recipe_parser +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 3158074a2d74f99aa72289dfcf093b1b0ba4474e diff --git a/var/spack/packages/ImageMagick/package.py b/var/spack/packages/ImageMagick/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/Mitos/package.py b/var/spack/packages/Mitos/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/R/package.py b/var/spack/packages/R/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/SAMRAI/no-tool-build.patch b/var/spack/packages/SAMRAI/no-tool-build.patch old mode 100644 new mode 100755 diff --git a/var/spack/packages/SAMRAI/package.py b/var/spack/packages/SAMRAI/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/adept-utils/package.py b/var/spack/packages/adept-utils/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/arpack/package.py b/var/spack/packages/arpack/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/asciidoc/package.py b/var/spack/packages/asciidoc/package.py deleted file mode 100644 index 828f3b3f4f..0000000000 --- a/var/spack/packages/asciidoc/package.py +++ /dev/null @@ -1,18 +0,0 @@ -from spack import * - -class Asciidoc(Package): - """ A presentable text document format for writing articles, UNIX man - pages and other small to medium sized documents.""" - homepage = "http://asciidoc.org" - url = "http://downloads.sourceforge.net/project/asciidoc/asciidoc/8.6.9/asciidoc-8.6.9.tar.gz" - - version('8.6.9', 'c59018f105be8d022714b826b0be130a') - - depends_on('libxml2') - depends_on('libxslt') - - def install(self, spec, prefix): - configure('--prefix=%s' % prefix) - - make() - make("install") diff --git a/var/spack/packages/atk/package.py b/var/spack/packages/atk/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/atlas/package.py b/var/spack/packages/atlas/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/autoconf/package.py b/var/spack/packages/autoconf/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/automaded/package.py b/var/spack/packages/automaded/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/automake/package.py b/var/spack/packages/automake/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/bear/package.py b/var/spack/packages/bear/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/bib2xhtml/package.py b/var/spack/packages/bib2xhtml/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/binutils/package.py b/var/spack/packages/binutils/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/bison/package.py b/var/spack/packages/bison/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/boost/package.py b/var/spack/packages/boost/package.py old mode 100644 new mode 100755 index 35824d53a2..254d1afda1 --- a/var/spack/packages/boost/package.py +++ b/var/spack/packages/boost/package.py @@ -14,10 +14,6 @@ class Boost(Package): list_url = "http://sourceforge.net/projects/boost/files/boost/" list_depth = 2 - version('1.59.0', '6aa9a5c6a4ca1016edd0ed1178e3cb87') - version('1.58.0', 'b8839650e61e9c1c0a89f371dd475546') - version('1.57.0', '1be49befbdd9a5ce9def2983ba3e7b76') - version('1.56.0', 'a744cf167b05d72335f27c88115f211d') version('1.55.0', 'd6eef4b4cacb2183f2bf265a5a03a354') version('1.54.0', '15cb8c0803064faef0c4ddf5bc5ca279') version('1.53.0', 'a00d22605d5dbcfb4c9936a9b35bc4c2') diff --git a/var/spack/packages/bowtie2/bowtie2-2.5.patch b/var/spack/packages/bowtie2/bowtie2-2.5.patch deleted file mode 100644 index 290be39c73..0000000000 --- a/var/spack/packages/bowtie2/bowtie2-2.5.patch +++ /dev/null @@ -1,16 +0,0 @@ ---- Makefile 2015-02-26 10:50:00.000000000 -0800 -+++ Makefile.new 2015-07-29 18:03:59.891357399 -0700 -@@ -22,10 +22,10 @@ - # - - INC = --GCC_PREFIX = $(shell dirname `which gcc`) -+GCC_PREFIX = - GCC_SUFFIX = --CC = $(GCC_PREFIX)/gcc$(GCC_SUFFIX) --CPP = $(GCC_PREFIX)/g++$(GCC_SUFFIX) -+CC = cc -+CPP = c++ - CXX = $(CPP) - HEADERS = $(wildcard *.h) - BOWTIE_MM = 1 diff --git a/var/spack/packages/bowtie2/package.py b/var/spack/packages/bowtie2/package.py deleted file mode 100644 index 339aab6598..0000000000 --- a/var/spack/packages/bowtie2/package.py +++ /dev/null @@ -1,24 +0,0 @@ -from spack import * -from glob import glob -class Bowtie2(Package): - """Description""" - homepage = "bowtie-bio.sourceforge.net/bowtie2/index.shtml" - version('2.2.5','51fa97a862d248d7ee660efc1147c75f', url = "http://downloads.sourceforge.net/project/bowtie-bio/bowtie2/2.2.5/bowtie2-2.2.5-source.zip") - - patch('bowtie2-2.5.patch',when='@2.2.5', level=0) - - def install(self, spec, prefix): - make() - mkdirp(prefix.bin) - for bow in glob("bowtie2*"): - install(bow, prefix.bin) - # install('bowtie2',prefix.bin) - # install('bowtie2-align-l',prefix.bin) - # install('bowtie2-align-s',prefix.bin) - # install('bowtie2-build',prefix.bin) - # install('bowtie2-build-l',prefix.bin) - # install('bowtie2-build-s',prefix.bin) - # install('bowtie2-inspect',prefix.bin) - # install('bowtie2-inspect-l',prefix.bin) - # install('bowtie2-inspect-s',prefix.bin) - diff --git a/var/spack/packages/boxlib/package.py b/var/spack/packages/boxlib/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/bzip2/package.py b/var/spack/packages/bzip2/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/cairo/package.py b/var/spack/packages/cairo/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/callpath/package.py b/var/spack/packages/callpath/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/cblas/package.py b/var/spack/packages/cblas/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/cgm/package.py b/var/spack/packages/cgm/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/clang/package.py b/var/spack/packages/clang/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/cloog/package.py b/var/spack/packages/cloog/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/cmake/package.py b/var/spack/packages/cmake/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/coreutils/package.py b/var/spack/packages/coreutils/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/cppcheck/package.py b/var/spack/packages/cppcheck/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/cram/package.py b/var/spack/packages/cram/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/cscope/package.py b/var/spack/packages/cscope/package.py deleted file mode 100644 index 9aac0f7304..0000000000 --- a/var/spack/packages/cscope/package.py +++ /dev/null @@ -1,17 +0,0 @@ -from spack import * - -class Cscope(Package): - """Cscope is a developer's tool for browsing source code.""" - homepage = "http://http://cscope.sourceforge.net/" - url = "http://downloads.sourceforge.net/project/cscope/cscope/15.8b/cscope-15.8b.tar.gz" - - version('15.8b', '8f9409a238ee313a96f9f87fe0f3b176') - - # Can be configured to use flex (not necessary) - # ./configure --with-flex - - def install(self, spec, prefix): - configure('--prefix=%s' % prefix) - - make() - make("install") diff --git a/var/spack/packages/cube/package.py b/var/spack/packages/cube/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/czmq/package.py b/var/spack/packages/czmq/package.py deleted file mode 100644 index a2f1947554..0000000000 --- a/var/spack/packages/czmq/package.py +++ /dev/null @@ -1,19 +0,0 @@ -from spack import * - -class Czmq(Package): - """ A C interface to the ZMQ library """ - homepage = "http://czmq.zeromq.org" - url = "https://github.com/zeromq/czmq/archive/v3.0.2.tar.gz" - - version('3.0.2', '23e9885f7ee3ce88d99d0425f52e9be1', url='https://github.com/zeromq/czmq/archive/v3.0.2.tar.gz') - - depends_on('zeromq') - - def install(self, spec, prefix): - bash = which("bash") - bash("./autogen.sh") - configure("--prefix=%s" % prefix) - - make() - make("install") - diff --git a/var/spack/packages/dbus/package.py b/var/spack/packages/dbus/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/docbook-xml/package.py b/var/spack/packages/docbook-xml/package.py deleted file mode 100644 index fce1de7deb..0000000000 --- a/var/spack/packages/docbook-xml/package.py +++ /dev/null @@ -1,19 +0,0 @@ -import os -import glob -from spack import * - - -class DocbookXml(Package): - """Docbook DTD XML files.""" - homepage = "http://www.oasis-open.org/docbook" - url = "http://www.oasis-open.org/docbook/xml/4.5/docbook-xml-4.5.zip" - - version('4.5', '03083e288e87a7e829e437358da7ef9e') - - def install(self, spec, prefix): - cp = which('cp') - - install_args = ['-a', '-t', prefix] - install_args.extend(glob.glob('*')) - - cp(*install_args) diff --git a/var/spack/packages/doxygen/package.py b/var/spack/packages/doxygen/package.py deleted file mode 100644 index 3d4a4e47a7..0000000000 --- a/var/spack/packages/doxygen/package.py +++ /dev/null @@ -1,25 +0,0 @@ -#------------------------------------------------------------------------------ -# Author: Justin Too -# Date: September 11, 2015 -#------------------------------------------------------------------------------ - -from spack import * - -class Doxygen(Package): - """Doxygen is the de facto standard tool for generating documentation - from annotated C++ sources, but it also supports other popular programming - languages such as C, Objective-C, C#, PHP, Java, Python, IDL (Corba, - Microsoft, and UNO/OpenOffice flavors), Fortran, VHDL, Tcl, and to some extent D.. - """ - homepage = "http://www.stack.nl/~dimitri/doxygen/" - url = "http://ftp.stack.nl/pub/users/dimitri/doxygen-1.8.10.src.tar.gz" - - version('1.8.10', '79767ccd986f12a0f949015efb5f058f') - - depends_on("cmake@2.8.12:") - - def install(self, spec, prefix): - cmake('.', *std_cmake_args) - - make() - make("install") diff --git a/var/spack/packages/dri2proto/package.py b/var/spack/packages/dri2proto/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/dtcmp/package.py b/var/spack/packages/dtcmp/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/dyninst/package.py b/var/spack/packages/dyninst/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/eigen/package.py b/var/spack/packages/eigen/package.py new file mode 100644 index 0000000000..b4d89ea3a0 --- /dev/null +++ b/var/spack/packages/eigen/package.py @@ -0,0 +1,17 @@ +from spack import * + +class Eigen(Package): + """Eigen is a C++ template library for linear algebra: matrices, vectors, + numerical solvers, and related algorithms..""" + homepage = "http://eigen.tuxfamily.org/" + url = "http://bitbucket.org/eigen/eigen/get/3.2.5.tar.gz" + + version('3.2.5', '8cc513ac6ec687117acadddfcacf551b') + + def install(self, spec, prefix): + import os + os.system("mkdir ./build_dir && cd ./build_dir") + cmake('../', *std_cmake_args) + + make() + make("install") diff --git a/var/spack/packages/elfutils/package.py b/var/spack/packages/elfutils/package.py deleted file mode 100644 index 926d234584..0000000000 --- a/var/spack/packages/elfutils/package.py +++ /dev/null @@ -1,26 +0,0 @@ -from spack import * - -class Elfutils(Package): - """elfutils is a collection of various binary tools such as - eu-objdump, eu-readelf, and other utilities that allow you to - inspect and manipulate ELF files. Refer to Table 5.Tools Included - in elfutils for Red Hat Developer for a complete list of binary - tools that are distributed with the Red Hat Developer Toolset - version of elfutils.""" - - homepage = "https://fedorahosted.org/elfutils/" - - version('0.163', - git='git://git.fedorahosted.org/git/elfutils.git', - tag='elfutils-0.163') - - provides('elf') - - def install(self, spec, prefix): - autoreconf = which('autoreconf') - autoreconf('-if') - - configure('--prefix=%s' % prefix, '--enable-maintainer-mode') - make() - make("install") - diff --git a/var/spack/packages/extrae/package.py b/var/spack/packages/extrae/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/exuberant-ctags/package.py b/var/spack/packages/exuberant-ctags/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/fish/package.py b/var/spack/packages/fish/package.py deleted file mode 100644 index 1225558705..0000000000 --- a/var/spack/packages/fish/package.py +++ /dev/null @@ -1,18 +0,0 @@ -from spack import * - -class Fish(Package): - """fish is a smart and user-friendly command line shell for OS X, Linux, and - the rest of the family. - """ - - homepage = "http://fishshell.com/" - url = "http://fishshell.com/files/2.2.0/fish-2.2.0.tar.gz" - list_url = homepage - - version('2.2.0', 'a76339fd14ce2ec229283c53e805faac48c3e99d9e3ede9d82c0554acfc7b77a') - - def install(self, spec, prefix): - configure('--prefix=%s' % prefix) - - make() - make("install") diff --git a/var/spack/packages/flex/package.py b/var/spack/packages/flex/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/flux/package.py b/var/spack/packages/flux/package.py deleted file mode 100644 index c128f46be8..0000000000 --- a/var/spack/packages/flux/package.py +++ /dev/null @@ -1,36 +0,0 @@ -from spack import * -import os - -class Flux(Package): - """ A next-generation resource manager (pre-alpha) """ - - homepage = "https://github.com/flux-framework/flux-core" - url = "https://github.com/flux-framework/flux-core" - - version('master', branch='master', git='https://github.com/flux-framework/flux-core') - - # Also needs autotools, but should use the system version if available - depends_on("zeromq@4.0.4:") - depends_on("czmq@2.2:") - depends_on("lua@5.1:5.1.99") - depends_on("munge") - depends_on("libjson-c") - depends_on("libxslt") - # TODO: This provides a catalog, hacked with environment below for now - depends_on("docbook-xml") - depends_on("asciidoc") - depends_on("python") - depends_on("py-cffi") - - def install(self, spec, prefix): - # Bootstrap with autotools - bash = which('bash') - bash('./autogen.sh') - - # Fix asciidoc dependency on xml style sheets and whatnot - os.environ['XML_CATALOG_FILES'] = os.path.join(spec['docbook-xml'].prefix, - 'catalog.xml') - # Configure, compile & install - configure("--prefix=" + prefix) - make("install", "V=1") - diff --git a/var/spack/packages/fontconfig/package.py b/var/spack/packages/fontconfig/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/freetype/package.py b/var/spack/packages/freetype/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/gasnet/package.py b/var/spack/packages/gasnet/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/gcc/package.py b/var/spack/packages/gcc/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/gdk-pixbuf/package.py b/var/spack/packages/gdk-pixbuf/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/geos/package.py b/var/spack/packages/geos/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/gflags/package.py b/var/spack/packages/gflags/package.py deleted file mode 100644 index 62dd80a094..0000000000 --- a/var/spack/packages/gflags/package.py +++ /dev/null @@ -1,21 +0,0 @@ -import os -from spack import * - -class Gflags(Package): - """The gflags package contains a C++ library that implements - commandline flags processing. It includes built-in support for - standard types such as string and the ability to define flags - in the source file in which they are used. Online documentation - available at: https://gflags.github.io/gflags/""" - - homepage = "https://gflags.github.io/gflags" - url = "https://github.com/gflags/gflags/archive/v2.1.2.tar.gz" - - version('2.1.2', 'ac432de923f9de1e9780b5254884599f') - - def install(self, spec, prefix): - cmake("-DCMAKE_INSTALL_PREFIX=" + prefix, - "-DBUILD_SHARED_LIBS=ON") - make() - make("test") - make("install") diff --git a/var/spack/packages/ghostscript/package.py b/var/spack/packages/ghostscript/package.py deleted file mode 100644 index 0ab49d425f..0000000000 --- a/var/spack/packages/ghostscript/package.py +++ /dev/null @@ -1,17 +0,0 @@ -from spack import * - -class Ghostscript(Package): - """an interpreter for the PostScript language and for PDF. """ - homepage = "http://ghostscript.com/" - url = "http://downloads.ghostscript.com/public/ghostscript-9.16.tar.gz" - - version('9.16', '829319325bbdb83f5c81379a8f86f38f') - - parallel = False - - def install(self, spec, prefix): - configure("--prefix=%s" %prefix, "--enable-shared") - - make() - make("install") - diff --git a/var/spack/packages/git/package.py b/var/spack/packages/git/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/glib/package.py b/var/spack/packages/glib/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/global/package.py b/var/spack/packages/global/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/glog/package.py b/var/spack/packages/glog/package.py deleted file mode 100644 index d73386b394..0000000000 --- a/var/spack/packages/glog/package.py +++ /dev/null @@ -1,15 +0,0 @@ -import os -from spack import * - -class Glog(Package): - """C++ implementation of the Google logging module.""" - - homepage = "https://github.com/google/glog" - url = "https://github.com/google/glog/archive/v0.3.3.tar.gz" - - version('0.3.3', 'c1f86af27bd9c73186730aa957607ed0') - - def install(self, spec, prefix): - configure("--prefix=" + prefix) - make() - make("install") diff --git a/var/spack/packages/gmp/package.py b/var/spack/packages/gmp/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/gnutls/package.py b/var/spack/packages/gnutls/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/gperf/package.py b/var/spack/packages/gperf/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/gperftools/package.py b/var/spack/packages/gperftools/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/graphlib/package.py b/var/spack/packages/graphlib/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/graphviz/package.py b/var/spack/packages/graphviz/package.py index 7af7da1881..8244feccfd 100644 --- a/var/spack/packages/graphviz/package.py +++ b/var/spack/packages/graphviz/package.py @@ -1,21 +1,24 @@ from spack import * class Graphviz(Package): - """Graph Visualization Software""" + """graph visualization software.""" homepage = "http://www.graphviz.org" url = "http://www.graphviz.org/pub/graphviz/stable/SOURCES/graphviz-2.38.0.tar.gz" version('2.38.0', '5b6a829b2ac94efcd5fa3c223ed6d3ae') + version('2.36.0', '1f41664dba0c93109ac8b71216bf2b57') - parallel = False - - depends_on("swig") - depends_on("python") - depends_on("ghostscript") + depends_on("cairo@1.1.10") + depends_on("freetype@2.1.10") + depends_on("fontconfig") + depends_on("zlib@1.2.3") + # depends_on("libpng@1.2.10") + # depends_on("expat@2.0.0") + # depends_on("gd@2.0.34") def install(self, spec, prefix): - configure("--prefix=%s" %prefix) + configure('--prefix=%s' % prefix) + # FIXME: Add logic to build and install here make() make("install") - diff --git a/var/spack/packages/gtkplus/package.py b/var/spack/packages/gtkplus/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/harfbuzz/package.py b/var/spack/packages/harfbuzz/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/hdf5/package.py b/var/spack/packages/hdf5/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/hwloc/package.py b/var/spack/packages/hwloc/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/hypre/package.py b/var/spack/packages/hypre/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/icu/package.py b/var/spack/packages/icu/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/icu4c/package.py b/var/spack/packages/icu4c/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/isl/package.py b/var/spack/packages/isl/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/jdk/package.py b/var/spack/packages/jdk/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/jpeg/package.py b/var/spack/packages/jpeg/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/netlib-lapack/package.py b/var/spack/packages/lapack/package.py old mode 100644 new mode 100755 similarity index 77% rename from var/spack/packages/netlib-lapack/package.py rename to var/spack/packages/lapack/package.py index fb6b99e27c..d9d37e3e4a --- a/var/spack/packages/netlib-lapack/package.py +++ b/var/spack/packages/lapack/package.py @@ -1,6 +1,6 @@ from spack import * -class NetlibLapack(Package): +class Lapack(Package): """ LAPACK version 3.X is a comprehensive FORTRAN library that does linear algebra operations including matrix inversions, least @@ -18,16 +18,9 @@ class NetlibLapack(Package): version('3.4.0', '02d5706ec03ba885fc246e5fa10d8c70') version('3.3.1', 'd0d533ec9a5b74933c2a1e84eedc58b4') - variant('shared', default=False, description="Build shared library version") - - # virtual dependency - provides('lapack') - # blas is a virtual dependency. depends_on('blas') - depends_on('cmake') - # Doesn't always build correctly in parallel parallel = False @@ -46,14 +39,7 @@ def get_blas_libs(self): def install(self, spec, prefix): blas_libs = ";".join(self.get_blas_libs()) - cmake_args = [".", '-DBLAS_LIBRARIES=' + blas_libs] - - if '+shared' in spec: - cmake_args.append('-DBUILD_SHARED_LIBS=ON') - - cmake_args += std_cmake_args - - cmake(*cmake_args) + cmake(".", '-DBLAS_LIBRARIES=' + blas_libs, *std_cmake_args) make() make("install") diff --git a/var/spack/packages/launchmon/package.py b/var/spack/packages/launchmon/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/launchmon/patch.lmon_install_dir b/var/spack/packages/launchmon/patch.lmon_install_dir old mode 100644 new mode 100755 diff --git a/var/spack/packages/lcms/package.py b/var/spack/packages/lcms/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/leveldb/package.py b/var/spack/packages/leveldb/package.py deleted file mode 100644 index da68a9cbcb..0000000000 --- a/var/spack/packages/leveldb/package.py +++ /dev/null @@ -1,29 +0,0 @@ -import os -import glob -from spack import * - -class Leveldb(Package): - """LevelDB is a fast key-value storage library written at Google - that provides an ordered mapping from string keys to string values.""" - - homepage = "https://github.com/google/leveldb" - url = "https://github.com/google/leveldb/archive/v1.18.tar.gz" - - version('1.18', '73770de34a2a5ab34498d2e05b2b7fa0') - - depends_on("snappy") - - def install(self, spec, prefix): - make() - - mkdirp(prefix.include) - mkdirp(prefix.lib) - - cp = which('cp') - - # cp --preserve=links libleveldb.* prefix/lib - args = glob.glob('libleveldb.*') - args.append(prefix + '/lib') - cp('--preserve=links', *args) - - cp('-r', 'include/leveldb', prefix + '/include') diff --git a/var/spack/packages/libNBC/package.py b/var/spack/packages/libNBC/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/libarchive/package.py b/var/spack/packages/libarchive/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/libcircle/package.py b/var/spack/packages/libcircle/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/libdrm/package.py b/var/spack/packages/libdrm/package.py old mode 100644 new mode 100755 index 00736b7811..1cab5372be --- a/var/spack/packages/libdrm/package.py +++ b/var/spack/packages/libdrm/package.py @@ -2,7 +2,7 @@ class Libdrm(Package): """A userspace library for accessing the DRM, direct - rendering manager, on Linux, BSD and other operating + rendering manager, on Linux, BSD and other operating systems that support the ioctl interface.""" homepage = "http://dri.freedesktop.org/libdrm/" # no real website... @@ -11,8 +11,11 @@ class Libdrm(Package): version('2.4.59', '105ac7af1afcd742d402ca7b4eb168b6') version('2.4.33', '86e4e3debe7087d5404461e0032231c8') + depends_on("libpciaccess") + def install(self, spec, prefix): configure("--prefix=%s" % prefix) + make("clean") make() make("install") diff --git a/var/spack/packages/libdwarf/package.py b/var/spack/packages/libdwarf/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/libelf/package.py b/var/spack/packages/libelf/package.py old mode 100644 new mode 100755 index 9338b8f393..bf2fefabd5 --- a/var/spack/packages/libelf/package.py +++ b/var/spack/packages/libelf/package.py @@ -36,8 +36,6 @@ class Libelf(Package): version('0.8.13', '4136d7b4c04df68b686570afa26988ac') version('0.8.12', 'e21f8273d9f5f6d43a59878dc274fec7') - provides('elf') - def install(self, spec, prefix): configure("--prefix=" + prefix, "--enable-shared", diff --git a/var/spack/packages/libevent/package.py b/var/spack/packages/libevent/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/libffi/package.py b/var/spack/packages/libffi/package.py old mode 100644 new mode 100755 index 18acabb00c..2c1c4eed4d --- a/var/spack/packages/libffi/package.py +++ b/var/spack/packages/libffi/package.py @@ -6,9 +6,9 @@ class Libffi(Package): to call any function specified by a call interface description at run time.""" homepage = "https://sourceware.org/libffi/" - - version('3.2.1','83b89587607e3eb65c70d361f13bab43',url = "ftp://sourceware.org/pub/libffi/libffi-3.2.1.tar.gz") - #version('3.1', 'f5898b29bbfd70502831a212d9249d10',url = "ftp://sourceware.org/pub/libffi/libffi-3.1.tar.gz") # Has a bug $(lib64) instead of ${lib64} in libffi.pc + url = "ftp://sourceware.org/pub/libffi/libffi-3.1.tar.gz" + + version('3.1', 'f5898b29bbfd70502831a212d9249d10') def install(self, spec, prefix): configure("--prefix=%s" % prefix) diff --git a/var/spack/packages/libgcrypt/package.py b/var/spack/packages/libgcrypt/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/libgpg-error/package.py b/var/spack/packages/libgpg-error/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/libjpeg-turbo/package.py b/var/spack/packages/libjpeg-turbo/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/libjson-c/package.py b/var/spack/packages/libjson-c/package.py deleted file mode 100644 index c0801cce9c..0000000000 --- a/var/spack/packages/libjson-c/package.py +++ /dev/null @@ -1,14 +0,0 @@ -from spack import * - -class LibjsonC(Package): - """ A JSON implementation in C """ - homepage = "https://github.com/json-c/json-c/wiki" - url = "https://s3.amazonaws.com/json-c_releases/releases/json-c-0.11.tar.gz" - - version('0.11', 'aa02367d2f7a830bf1e3376f77881e98') - - def install(self, spec, prefix): - configure('--prefix=%s' % prefix) - - make() - make("install") diff --git a/var/spack/packages/libmng/package.py b/var/spack/packages/libmng/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/libmonitor/package.py b/var/spack/packages/libmonitor/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/libpciaccess/package.py b/var/spack/packages/libpciaccess/package.py old mode 100644 new mode 100755 index 6022fc34a3..371fbb6685 --- a/var/spack/packages/libpciaccess/package.py +++ b/var/spack/packages/libpciaccess/package.py @@ -1,21 +1,16 @@ from spack import * class Libpciaccess(Package): - """Generic PCI access library.""" - + """Generic PCI access library""" homepage = "http://cgit.freedesktop.org/xorg/lib/libpciaccess/" - url = "http://cgit.freedesktop.org/xorg/lib/libpciaccess/" + url = "http://pkgs.fedoraproject.org/repo/pkgs/libpciaccess/libpciaccess-0.13.2.tar.bz2/b7c0d3afce14eedca57312a3141ec13a/libpciaccess-0.13.2.tar.bz2" - version('0.13.4', git='http://anongit.freedesktop.org/git/xorg/lib/libpciaccess.git', - tag='libpciaccess-0.13.4') - - depends_on('autoconf') - depends_on('libtool') + version('0.13.2', 'b7c0d3afce14eedca57312a3141ec13a') def install(self, spec, prefix): - from subprocess import call - call(["./autogen.sh"]) - configure("--prefix=%s" % prefix) + import os + os.system("autoconf") + configure('--prefix=%s' % prefix) make() make("install") diff --git a/var/spack/packages/libpng/package.py b/var/spack/packages/libpng/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/libsodium/package.py b/var/spack/packages/libsodium/package.py deleted file mode 100644 index 1c8a16d998..0000000000 --- a/var/spack/packages/libsodium/package.py +++ /dev/null @@ -1,19 +0,0 @@ -from spack import * - -class Libsodium(Package): - """Sodium is a modern, easy-to-use software library for encryption, - decryption, signatures, password hashing and more.""" - homepage = "https://download.libsodium.org/doc/" - url = "https://download.libsodium.org/libsodium/releases/libsodium-1.0.3.tar.gz" - - version('1.0.3', 'b3bcc98e34d3250f55ae196822307fab') - version('1.0.2', 'dc40eb23e293448c6fc908757738003f') - version('1.0.1', '9a221b49fba7281ceaaf5e278d0f4430') - version('1.0.0', '3093dabe4e038d09f0d150cef064b2f7') - version('0.7.1', 'c224fe3923d1dcfe418c65c8a7246316') - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - - make() - make("install") diff --git a/var/spack/packages/libtiff/package.py b/var/spack/packages/libtiff/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/libtool/package.py b/var/spack/packages/libtool/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/libunwind/package.py b/var/spack/packages/libunwind/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/libuuid/package.py b/var/spack/packages/libuuid/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/libxcb/package.py b/var/spack/packages/libxcb/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/libxml2/package.py b/var/spack/packages/libxml2/package.py old mode 100644 new mode 100755 index 3a0af6b368..72199d8def --- a/var/spack/packages/libxml2/package.py +++ b/var/spack/packages/libxml2/package.py @@ -9,12 +9,11 @@ class Libxml2(Package): version('2.9.2', '9e6a9aca9d155737868b3dc5fd82f788') - extends('python') depends_on('zlib') depends_on('xz') def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - + configure("--prefix=%s" % prefix, + "--without-python") make() make("install") diff --git a/var/spack/packages/libxshmfence/package.py b/var/spack/packages/libxshmfence/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/libxslt/package.py b/var/spack/packages/libxslt/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/llvm-lld/package.py b/var/spack/packages/llvm-lld/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/llvm/package.py b/var/spack/packages/llvm/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/lmdb/package.py b/var/spack/packages/lmdb/package.py deleted file mode 100644 index 875b8100c5..0000000000 --- a/var/spack/packages/lmdb/package.py +++ /dev/null @@ -1,39 +0,0 @@ -import os -from spack import * - -class Lmdb(Package): - """Read-only mirror of official repo on openldap.org. Issues and - pull requests here are ignored. Use OpenLDAP ITS for issues. - http://www.openldap.org/software/repo.html""" - - - homepage = "http://www.openldap.org/software/repo.html" - url = "https://github.com/LMDB/lmdb/archive/LMDB_0.9.16.tar.gz" - - version('0.9.16', '0de89730b8f3f5711c2b3a4ba517b648') - - def install(self, spec, prefix): - os.chdir('libraries/liblmdb') - - make() - - mkdirp(prefix.bin) - mkdirp(prefix + '/man/man1') - mkdirp(prefix.lib) - mkdirp(prefix.include) - - bins = ['mdb_stat', 'mdb_copy', 'mdb_dump', 'mdb_load'] - for f in bins: - install(f, prefix.bin) - - mans = ['mdb_stat.1', 'mdb_copy.1', 'mdb_dump.1', 'mdb_load.1'] - for f in mans: - install(f, prefix + '/man/man1') - - libs = ['liblmdb.a', 'liblmdb.so'] - for f in libs: - install(f, prefix.lib) - - includes = ['lmdb.h'] - for f in includes: - install(f, prefix.include) diff --git a/var/spack/packages/lua/package.py b/var/spack/packages/lua/package.py deleted file mode 100644 index 57c443cc2d..0000000000 --- a/var/spack/packages/lua/package.py +++ /dev/null @@ -1,26 +0,0 @@ -from spack import * -import os - -class Lua(Package): - """ The Lua programming language interpreter and library """ - homepage = "http://www.lua.org" - url = "http://www.lua.org/ftp/lua-5.1.5.tar.gz" - - version('5.3.1', '797adacada8d85761c079390ff1d9961') - version('5.3.0', 'a1b0a7e92d0c85bbff7a8d27bf29f8af') - version('5.2.4', '913fdb32207046b273fdb17aad70be13') - version('5.2.3', 'dc7f94ec6ff15c985d2d6ad0f1b35654') - version('5.2.2', 'efbb645e897eae37cad4344ce8b0a614') - version('5.2.1', 'ae08f641b45d737d12d30291a5e5f6e3') - version('5.2.0', 'f1ea831f397214bae8a265995ab1a93e') - version('5.1.5', '2e115fe26e435e33b0d5c022e4490567') - version('5.1.4', 'd0870f2de55d59c1c8419f36e8fac150') - version('5.1.3', 'a70a8dfaa150e047866dc01a46272599') - - depends_on('ncurses') - - def install(self, spec, prefix): - make('INSTALL_TOP=%s' % prefix, - 'MYLDFLAGS=-L%s/lib' % spec['ncurses'].prefix, - 'linux', - 'install') diff --git a/var/spack/packages/lwgrp/package.py b/var/spack/packages/lwgrp/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/lwm2/package.py b/var/spack/packages/lwm2/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/memaxes/package.py b/var/spack/packages/memaxes/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/mesa/package.py b/var/spack/packages/mesa/package.py old mode 100644 new mode 100755 index 2a04a8fd51..2dba878a77 --- a/var/spack/packages/mesa/package.py +++ b/var/spack/packages/mesa/package.py @@ -1,7 +1,7 @@ from spack import * class Mesa(Package): - """Mesa is an open-source implementation of the OpenGL + """Mesa is an open-source implementation of the OpenGL specification - a system for rendering interactive 3D graphics.""" homepage = "http://www.mesa3d.org" @@ -11,10 +11,9 @@ class Mesa(Package): # version('10.4.4', '8d863a3c209bf5116b2babfccccc68ce') version('8.0.5', 'cda5d101f43b8784fa60bdeaca4056f2') - # mesa 7.x, 8.x, 9.x + # mesa 7.x, 8.x, 9.x depends_on("libdrm@2.4.33") depends_on("llvm@3.0") - depends_on("libxml2") # patch("llvm-fixes.patch") # using newer llvm diff --git a/var/spack/packages/metis/package.py b/var/spack/packages/metis/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/mpc/package.py b/var/spack/packages/mpc/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/mpe2/mpe2.patch b/var/spack/packages/mpe2/mpe2.patch old mode 100644 new mode 100755 diff --git a/var/spack/packages/mpe2/package.py b/var/spack/packages/mpe2/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/mpfr/package.py b/var/spack/packages/mpfr/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/mpibash/mpibash-4.3.patch b/var/spack/packages/mpibash/mpibash-4.3.patch old mode 100644 new mode 100755 diff --git a/var/spack/packages/mpibash/package.py b/var/spack/packages/mpibash/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/mpich/package.py b/var/spack/packages/mpich/package.py old mode 100644 new mode 100755 index b6b2dfde21..6aa8f2eca4 --- a/var/spack/packages/mpich/package.py +++ b/var/spack/packages/mpich/package.py @@ -38,6 +38,7 @@ class Mpich(Package): provides('mpi@:3', when='@3:') provides('mpi@:1', when='@1:') + def setup_dependent_environment(self, module, spec, dep_spec): """For dependencies, make mpicc's use spack wrapper.""" os.environ['MPICH_CC'] = 'cc' diff --git a/var/spack/packages/mpileaks/package.py b/var/spack/packages/mpileaks/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/mrnet/package.py b/var/spack/packages/mrnet/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/munge/package.py b/var/spack/packages/munge/package.py deleted file mode 100644 index c737ca0354..0000000000 --- a/var/spack/packages/munge/package.py +++ /dev/null @@ -1,20 +0,0 @@ -from spack import * -import os - -class Munge(Package): - """ MUNGE Uid 'N' Gid Emporium """ - homepage = "https://code.google.com/p/munge/" - url = "https://github.com/dun/munge/releases/download/munge-0.5.11/munge-0.5.11.tar.bz2" - - version('0.5.11', 'bd8fca8d5f4c1fcbef1816482d49ee01', url='https://github.com/dun/munge/releases/download/munge-0.5.11/munge-0.5.11.tar.bz2') - - depends_on('openssl') - depends_on('libgcrypt') - - def install(self, spec, prefix): - os.makedirs(os.path.join(prefix, "lib/systemd/system")) - configure("--prefix=%s" % prefix) - - make() - make("install") - diff --git a/var/spack/packages/muster/package.py b/var/spack/packages/muster/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/mvapich2/ad_lustre_rwcontig_open_source.patch b/var/spack/packages/mvapich2/ad_lustre_rwcontig_open_source.patch old mode 100644 new mode 100755 diff --git a/var/spack/packages/mvapich2/package.py b/var/spack/packages/mvapich2/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/nasm/package.py b/var/spack/packages/nasm/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/ncurses/package.py b/var/spack/packages/ncurses/package.py old mode 100644 new mode 100755 index 8f5763bfdd..a22e83503f --- a/var/spack/packages/ncurses/package.py +++ b/var/spack/packages/ncurses/package.py @@ -21,11 +21,3 @@ def install(self, spec, prefix): make() make("install") - configure("--prefix=%s" % prefix, - "--with-shared", - "--disable-widec", - "--disable-pc-files", - "--without-ada") - make() - make("install") - diff --git a/var/spack/packages/netcdf/package.py b/var/spack/packages/netcdf/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/netgauge/package.py b/var/spack/packages/netgauge/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/netlib-blas/package.py b/var/spack/packages/netlib-blas/package.py old mode 100644 new mode 100755 index 85e97323d3..0a6cdb0442 --- a/var/spack/packages/netlib-blas/package.py +++ b/var/spack/packages/netlib-blas/package.py @@ -9,8 +9,6 @@ class NetlibBlas(Package): version('3.5.0', 'b1d3e3e425b2e44a06760ff173104bdf') - variant('fpic', default=False, description="Build with -fpic compiler option") - # virtual dependency provides('blas') @@ -25,10 +23,6 @@ def patch(self): mf.filter('^LOADER.*', 'LOADER = f90') mf.filter('^CC =.*', 'CC = cc') - if '+fpic' in self.spec: - mf.filter('^OPTS.*=.*', 'OPTS = -O2 -frecursive -fpic') - mf.filter('^CFLAGS =.*', 'CFLAGS = -O3 -fpic') - def install(self, spec, prefix): make('blaslib') diff --git a/var/spack/packages/nettle/package.py b/var/spack/packages/nettle/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/ompss/package.py b/var/spack/packages/ompss/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/opari2/package.py b/var/spack/packages/opari2/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/openmpi/ad_lustre_rwcontig_open_source.patch b/var/spack/packages/openmpi/ad_lustre_rwcontig_open_source.patch old mode 100644 new mode 100755 diff --git a/var/spack/packages/openmpi/llnl-platforms.patch b/var/spack/packages/openmpi/llnl-platforms.patch old mode 100644 new mode 100755 diff --git a/var/spack/packages/openmpi/package.py b/var/spack/packages/openmpi/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/openssl/package.py b/var/spack/packages/openssl/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/otf/package.py b/var/spack/packages/otf/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/otf2/package.py b/var/spack/packages/otf2/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/pango/package.py b/var/spack/packages/pango/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/papi/package.py b/var/spack/packages/papi/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/paraver/package.py b/var/spack/packages/paraver/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/parmetis/package.py b/var/spack/packages/parmetis/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/parpack/package.py b/var/spack/packages/parpack/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/pcre/package.py b/var/spack/packages/pcre/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/petsc/package.py b/var/spack/packages/petsc/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/pidx/package.py b/var/spack/packages/pidx/package.py deleted file mode 100644 index 81aed62fb1..0000000000 --- a/var/spack/packages/pidx/package.py +++ /dev/null @@ -1,21 +0,0 @@ -from spack import * - -class Pidx(Package): - """PIDX Parallel I/O Library. - - PIDX is an efficient parallel I/O library that reads and writes - multiresolution IDX data files. - """ - - homepage = "http://www.cedmav.com/pidx" - - version('1.0', git='https://github.com/sci-visus/PIDX.git', - commit='6afa1cf71d1c41263296dc049c8fabaf73c296da') - - depends_on("mpi") - - def install(self, spec, prefix): - with working_dir('spack-build', create=True): - cmake('..', *std_cmake_args) - make() - make("install") diff --git a/var/spack/packages/pixman/package.py b/var/spack/packages/pixman/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/pkg-config/package.py b/var/spack/packages/pkg-config/package.py deleted file mode 100644 index 9964c6ce34..0000000000 --- a/var/spack/packages/pkg-config/package.py +++ /dev/null @@ -1,17 +0,0 @@ -from spack import * - -class PkgConfig(Package): - """pkg-config is a helper tool used when compiling applications and libraries""" - homepage = "http://www.freedesktop.org/wiki/Software/pkg-config/" - url = "http://pkgconfig.freedesktop.org/releases/pkg-config-0.28.tar.gz" - - version('0.28', 'aa3c86e67551adc3ac865160e34a2a0d') - - parallel = False - - def install(self, spec, prefix): - configure("--prefix=%s" %prefix, "--enable-shared") - - make() - make("install") - diff --git a/var/spack/packages/pmgr_collective/package.py b/var/spack/packages/pmgr_collective/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/postgresql/package.py b/var/spack/packages/postgresql/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/ppl/package.py b/var/spack/packages/ppl/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/protobuf/package.py b/var/spack/packages/protobuf/package.py deleted file mode 100644 index 34085c7ce9..0000000000 --- a/var/spack/packages/protobuf/package.py +++ /dev/null @@ -1,16 +0,0 @@ -import os -from spack import * - -class Protobuf(Package): - """Google's data interchange format.""" - - homepage = "https://developers.google.com/protocol-buffers" - url = "https://github.com/google/protobuf/releases/download/v2.5.0/protobuf-2.5.0.tar.bz2" - - version('2.5.0', 'a72001a9067a4c2c4e0e836d0f92ece4') - - def install(self, spec, prefix): - configure("--prefix=" + prefix) - make() - make("check") - make("install") diff --git a/var/spack/packages/py-Cython/package.py b/var/spack/packages/py-Cython/package.py new file mode 100644 index 0000000000..c250139114 --- /dev/null +++ b/var/spack/packages/py-Cython/package.py @@ -0,0 +1,12 @@ +from spack import * + +class PyCython(Package): + """The Cython compiler for writing C extensions for the Python language.""" + homepage = "http://www.cython.org" + version("0.21.2", "d21adb870c75680dc857cd05d41046a4", + url="https://pypi.python.org/packages/source/C/Cython/Cython-0.21.2.tar.gz") + + extends("python") + + def install(self, spec, prefix): + python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-Distutils2/package.py b/var/spack/packages/py-Distutils2/package.py new file mode 100644 index 0000000000..8690f569fe --- /dev/null +++ b/var/spack/packages/py-Distutils2/package.py @@ -0,0 +1,15 @@ +from spack import * + +class PyDistutils2(Package): + """Python Packaging Library.""" + + homepage = "https://hg.python.org/distutils2" + url = "https://pypi.python.org/packages/source/D/Distutils2/Distutils2-1.0a4.tar.gz" + + version('1.0a4', '52bc9dffb394970c27e02853ae3a3241') + + depends_on("python") + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) + diff --git a/var/spack/packages/py-astroML/package.py b/var/spack/packages/py-astroML/package.py new file mode 100644 index 0000000000..1583e249d2 --- /dev/null +++ b/var/spack/packages/py-astroML/package.py @@ -0,0 +1,12 @@ +from spack import * + +class PyAstroml(Package): + """tools for machine learning and data mining in Astronomy""" + homepage = "http://astroML.github.com" + version("0.2", "85f558368546660564f20b30efafb024", + url="https://pypi.python.org/packages/source/a/astroML/astroML-0.2.tar.gz") + + extends("python") + + def install(self, spec, prefix): + python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-astropy/package.py b/var/spack/packages/py-astropy/package.py new file mode 100644 index 0000000000..5d2d0619cc --- /dev/null +++ b/var/spack/packages/py-astropy/package.py @@ -0,0 +1,12 @@ +from spack import * + +class PyAstropy(Package): + """Community-developed python astronomy tools""" + homepage = "http://astropy.org" + version("0.4.2", "6dc4f643cde37ba0a8b4967dc8becee8", + url="https://pypi.python.org/packages/source/a/astropy/astropy-0.4.2.tar.gz") + + extends("python") + + def install(self, spec, prefix): + python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-autopep8/package.py b/var/spack/packages/py-autopep8/package.py new file mode 100644 index 0000000000..39a2288ea0 --- /dev/null +++ b/var/spack/packages/py-autopep8/package.py @@ -0,0 +1,12 @@ +from spack import * + +class PyAutopep8(Package): + """A tool that automatically formats Python code to conform to the PEP 8 style guide""" + homepage = "https://github.com/hhatto/autopep8" + version("1.1", "7998358d8f0efd77dcb2cc8e34e3cb5c", + url="https://pypi.python.org/packages/source/a/autopep8/autopep8-1.1.tar.gz") + + extends("python") + + def install(self, spec, prefix): + python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-basemap/package.py b/var/spack/packages/py-basemap/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-biopython/package.py b/var/spack/packages/py-biopython/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-certifi/package.py b/var/spack/packages/py-certifi/package.py new file mode 100644 index 0000000000..442903e6a0 --- /dev/null +++ b/var/spack/packages/py-certifi/package.py @@ -0,0 +1,12 @@ +from spack import * + +class PyCertifi(Package): + """Python package for providing Mozilla's CA Bundle.""" + homepage = "http://python-requests.org" + version("14.05.14", "315ea4e50673a16ab047099f816fd32a", + url="https://pypi.python.org/packages/source/c/certifi/certifi-14.05.14.tar.gz") + + extends("python") + + def install(self, spec, prefix): + python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-cffi/package.py b/var/spack/packages/py-cffi/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-cython/package.py b/var/spack/packages/py-cython/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-dateutil/package.py b/var/spack/packages/py-dateutil/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-decorator/package.py b/var/spack/packages/py-decorator/package.py new file mode 100644 index 0000000000..00c1c541d4 --- /dev/null +++ b/var/spack/packages/py-decorator/package.py @@ -0,0 +1,12 @@ +from spack import * + +class PyDecorator(Package): + """Better living through Python with decorators""" + homepage = "https://github.com/micheles/decorator" + version("4.0.4", "dd3a0669e1e6f09699eefa2c7fbd9756", + url="https://pypi.python.org/packages/source/d/decorator/decorator-4.0.4.tar.gz") + + extends("python") + + def install(self, spec, prefix): + python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-emcee/package.py b/var/spack/packages/py-emcee/package.py new file mode 100644 index 0000000000..544066b92d --- /dev/null +++ b/var/spack/packages/py-emcee/package.py @@ -0,0 +1,12 @@ +from spack import * + +class PyEmcee(Package): + """Kick ass affine-invariant ensemble MCMC sampling""" + homepage = "http://dan.iel.fm/emcee/" + version("2.1.0", "c6b6fad05c824d40671d4a4fc58dfff7", + url="https://pypi.python.org/packages/source/e/emcee/emcee-2.1.0.tar.gz") + + extends("python") + + def install(self, spec, prefix): + python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-epydoc/package.py b/var/spack/packages/py-epydoc/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-fitsio/package.py b/var/spack/packages/py-fitsio/package.py new file mode 100644 index 0000000000..046bc386d5 --- /dev/null +++ b/var/spack/packages/py-fitsio/package.py @@ -0,0 +1,13 @@ +from spack import * + +class PyFitsio(Package): + """A full featured python library to read from and write to FITS files.""" + homepage = "https://github.com/esheldon/fitsio" + version("0.9.7", "75fa05f999ae8f55c8290bd78ada49e7", + url="https://pypi.python.org/packages/source/f/fitsio/fitsio-0.9.7.tar.gz") + + extends("python") + depends_on("py-numpy") + + def install(self, spec, prefix): + python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-genders/package.py b/var/spack/packages/py-genders/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-gnuplot/package.py b/var/spack/packages/py-gnuplot/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-gnureadline/package.py b/var/spack/packages/py-gnureadline/package.py new file mode 100644 index 0000000000..51d1c6227e --- /dev/null +++ b/var/spack/packages/py-gnureadline/package.py @@ -0,0 +1,12 @@ +from spack import * + +class PyGnureadline(Package): + """The standard Python readline extension statically linked against the GNU readline library.""" + homepage = "http://github.com/ludwigschwardt/python-gnureadline" + version("6.3.3", "c4af83c9a3fbeac8f2da9b5a7c60e51c", + url="https://pypi.python.org/packages/source/g/gnureadline/gnureadline-6.3.3.tar.gz") + + extends("python") + + def install(self, spec, prefix): + python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-h5py/package.py b/var/spack/packages/py-h5py/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-ipython/package.py b/var/spack/packages/py-ipython/package.py old mode 100644 new mode 100755 index 8d0e64a07f..3b94b71f29 --- a/var/spack/packages/py-ipython/package.py +++ b/var/spack/packages/py-ipython/package.py @@ -3,14 +3,29 @@ class PyIpython(Package): """IPython provides a rich toolkit to help you make the most out of using Python interactively.""" homepage = "https://pypi.python.org/pypi/ipython" - url = "https://pypi.python.org/packages/source/i/ipython/ipython-2.3.1.tar.gz" - version('2.3.1', '2b7085525dac11190bfb45bb8ec8dcbf') + version('3.0.0','b3f00f3c0be036fafef3b0b9d663f27e', + url='https://pypi.python.org/packages/source/i/ipython/ipython-3.0.0.tar.gz') + version('4.0.0','c2fecbcf1c0fbdc82625c77a50733dd6', + url='https://pypi.python.org/packages/source/i/ipython/ipython-4.0.0.tar.gz') + version('2.3.1', '2b7085525dac11190bfb45bb8ec8dcbf', + url="https://pypi.python.org/packages/source/i/ipython/ipython-2.3.1.tar.gz") version('3.1.0', 'a749d90c16068687b0ec45a27e72ef8f') + version('3.2.1', 'f4c93d67ac4b2d4fc69df693b6f3c9e0', + url='https://github.com/ipython/ipython/archive/rel-3.2.1.tar.gz') extends('python') depends_on('py-pygments') - depends_on('py-setuptools') + depends_on('py-setuptools@18.1') + depends_on('py-traitlets') + depends_on('py-pyzmq') + depends_on('py-pexpect') + depends_on('py-tornado') + depends_on('py-jinja2') def install(self, spec, prefix): + import os + # os.system("spack activate py-setuptools@18.1") + # os.system("easy_install pip") + # os.system("pip install ipython[notebook]") python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-ipython/package.py.backup b/var/spack/packages/py-ipython/package.py.backup new file mode 100755 index 0000000000..1ed8b57e60 --- /dev/null +++ b/var/spack/packages/py-ipython/package.py.backup @@ -0,0 +1,27 @@ +from spack import * + +class PyIpython(Package): + """IPython provides a rich toolkit to help you make the most out of using Python interactively.""" + homepage = "https://pypi.python.org/pypi/ipython" + + version('2.3.1', '2b7085525dac11190bfb45bb8ec8dcbf', + url="https://pypi.python.org/packages/source/i/ipython/ipython-2.3.1.tar.gz") + version('3.1.0', 'a749d90c16068687b0ec45a27e72ef8f') + version('3.2.1', 'f4c93d67ac4b2d4fc69df693b6f3c9e0', + url='https://github.com/ipython/ipython/archive/rel-3.2.1.tar.gz') + + extends('python') + depends_on('py-pygments') + depends_on('py-setuptools@18.1') + depends_on('py-traitlets') + depends_on('py-pyzmq') + depends_on('py-pexpect') + depends_on('py-tornado') + depends_on('py-Jinja2') + + def install(self, spec, prefix): + import os + # os.system("spack activate py-setuptools@18.1") + # os.system("easy_install pip") + # os.system("pip install ipython[notebook]") + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-ipython_genutils/package.py b/var/spack/packages/py-ipython_genutils/package.py new file mode 100644 index 0000000000..39b15eec73 --- /dev/null +++ b/var/spack/packages/py-ipython_genutils/package.py @@ -0,0 +1,12 @@ +from spack import * + +class PyIpythonGenutils(Package): + """Vestigial utilities from IPython""" + homepage = "http://ipython.org" + version("0.1.0", "9a8afbe0978adbcbfcb3b35b2d015a56", + url="https://pypi.python.org/packages/source/i/ipython_genutils/ipython_genutils-0.1.0.tar.gz") + + extends("python") + + def install(self, spec, prefix): + python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-joblib/package.py b/var/spack/packages/py-joblib/package.py new file mode 100644 index 0000000000..10672314de --- /dev/null +++ b/var/spack/packages/py-joblib/package.py @@ -0,0 +1,12 @@ +from spack import * + +class PyJoblib(Package): + """Lightweight pipelining: using Python functions as pipeline jobs.""" + homepage = "http://packages.python.org/joblib/" + version("0.8.4", "90a1c25cc4dc4a8e3536093dbc35cff3", + url="https://pypi.python.org/packages/source/j/joblib/joblib-0.8.4.tar.gz") + + extends("python") + + def install(self, spec, prefix): + python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-jsonschema/package.py b/var/spack/packages/py-jsonschema/package.py new file mode 100755 index 0000000000..9a7fc21862 --- /dev/null +++ b/var/spack/packages/py-jsonschema/package.py @@ -0,0 +1,16 @@ +from spack import * + +class PyJsonschema(Package): + """implementation of JSON Schema validation for Python""" + homepage = "https://github.com/Julian/jsonschema" + url = "https://pypi.python.org/packages/source/j/jsonschema/jsonschema-2.5.1.tar.gz" + + version('2.4.0','661f85c3d23094afbb9ac3c0673840bf', + url='https://pypi.python.org/packages/source/j/jsonschema/jsonschema-2.4.0.tar.gz') + version('2.5.1', '374e848fdb69a3ce8b7e778b47c30640') + + extends("python") + depends_on("py-setuptools@18.1") + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-libxml2/package.py b/var/spack/packages/py-libxml2/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-lockfile/package.py b/var/spack/packages/py-lockfile/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-mako/package.py b/var/spack/packages/py-mako/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-matplotlib/package.py b/var/spack/packages/py-matplotlib/package.py old mode 100644 new mode 100755 index e7ce3dfd24..b9244daf56 --- a/var/spack/packages/py-matplotlib/package.py +++ b/var/spack/packages/py-matplotlib/package.py @@ -12,7 +12,6 @@ class PyMatplotlib(Package): extends('python', ignore=r'bin/nosetests.*$') depends_on('py-pyside') - depends_on('py-ipython') depends_on('py-pyparsing') depends_on('py-six') depends_on('py-dateutil') diff --git a/var/spack/packages/py-mock/package.py b/var/spack/packages/py-mock/package.py index 3b08428ba0..c43842c187 100644 --- a/var/spack/packages/py-mock/package.py +++ b/var/spack/packages/py-mock/package.py @@ -1,17 +1,12 @@ from spack import * class PyMock(Package): - """mock is a library for testing in Python. It allows you to replace parts - of your system under test with mock objects and make assertions about how - they have been used.""" + """A Python Mocking and Patching Library for Testing""" + homepage = "http://www.voidspace.org.uk/python/mock/" + version("1.0.1", "c3971991738caa55ec7c356bbc154ee2", + url="https://pypi.python.org/packages/source/m/mock/mock-1.0.1.tar.gz") - homepage = "https://github.com/testing-cabal/mock" - url = "https://pypi.python.org/packages/source/m/mock/mock-1.3.0.tar.gz" - - version('1.3.0', '73ee8a4afb3ff4da1b4afa287f39fdeb') - - extends('python') - depends_on('py-setuptools@17.1:') + extends("python") def install(self, spec, prefix): - python('setup.py', 'install', '--prefix=%s' % prefix) + python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-mpi4py/package.py b/var/spack/packages/py-mpi4py/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-mx/package.py b/var/spack/packages/py-mx/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-nose/package.py b/var/spack/packages/py-nose/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-numexpr/package.py b/var/spack/packages/py-numexpr/package.py new file mode 100644 index 0000000000..283ea18772 --- /dev/null +++ b/var/spack/packages/py-numexpr/package.py @@ -0,0 +1,12 @@ +from spack import * + +class PyNumexpr(Package): + """Fast numerical expression evaluator for NumPy""" + homepage = "https://github.com/pydata/numexpr" + version("2.4", "df7e8d9e9dbb145b56d43c465c2bf854", + url="https://pypi.python.org/packages/source/n/numexpr/numexpr-2.4.tar.gz") + + extends("python") + + def install(self, spec, prefix): + python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-numpy/package.py b/var/spack/packages/py-numpy/package.py old mode 100644 new mode 100755 index efa109a3e9..6534e063d2 --- a/var/spack/packages/py-numpy/package.py +++ b/var/spack/packages/py-numpy/package.py @@ -7,22 +7,9 @@ class PyNumpy(Package): version('1.9.1', '78842b73560ec378142665e712ae4ad9') version('1.9.2', 'a1ed53432dbcd256398898d35bc8e645') - + extends('python') depends_on('py-nose') - depends_on('netlib-blas+fpic') - depends_on('netlib-lapack+shared') - - def patch(self): - filter_file( - "possible_executables = \['(gfortran|g77|ifort|efl)", - "possible_executables = ['fc", - "numpy/distutils/fcompiler/gnu.py", - "numpy/distutils/fcompiler/intel.py") def install(self, spec, prefix): - with open('site.cfg', 'w') as f: - f.write('[DEFAULT]\n') - f.write('libraries=lapack,blas\n') - f.write('library_dirs=%s/lib:%s/lib\n' % (spec['blas'].prefix, spec['lapack'].prefix)) python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-pandas/package.py b/var/spack/packages/py-pandas/package.py old mode 100644 new mode 100755 index 5b9997faa9..07ee12782f --- a/var/spack/packages/py-pandas/package.py +++ b/var/spack/packages/py-pandas/package.py @@ -6,6 +6,8 @@ class PyPandas(Package): homepage = "http://pandas.pydata.org/" url = "https://pypi.python.org/packages/source/p/pandas/pandas-0.16.0.tar.gz#md5=bfe311f05dc0c351f8955fbd1e296e73" + version('0.15.1','f6fd8d49fc801f6f3fa48f9117a6290b', + url='https://pypi.python.org/packages/source/p/pandas/pandas-0.15.1.tar.gz') version('0.16.0', 'bfe311f05dc0c351f8955fbd1e296e73') version('0.16.1', 'fac4f25748f9610a3e00e765474bdea8') diff --git a/var/spack/packages/py-pep8/package.py b/var/spack/packages/py-pep8/package.py new file mode 100644 index 0000000000..0bcf416922 --- /dev/null +++ b/var/spack/packages/py-pep8/package.py @@ -0,0 +1,12 @@ +from spack import * + +class PyPep8(Package): + """Python style guide checker""" + homepage = "http://pep8.readthedocs.org/" + version("1.6.1", "76cf60b245f8549cb458ffcd85710738", + url="https://pypi.python.org/packages/source/p/pep8/pep8-1.6.1.tar.gz") + + extends("python") + + def install(self, spec, prefix): + python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-pexpect/package.py b/var/spack/packages/py-pexpect/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-pickleshare/package.py b/var/spack/packages/py-pickleshare/package.py new file mode 100644 index 0000000000..cf9ddb7db8 --- /dev/null +++ b/var/spack/packages/py-pickleshare/package.py @@ -0,0 +1,12 @@ +from spack import * + +class PyPickleshare(Package): + """Tiny 'shelve'-like database with concurrency support""" + homepage = "https://github.com/vivainio/pickleshare" + version("0.5", "25337740507cb855ad58bfcf60f7710e", + url="https://pypi.python.org/packages/source/p/pickleshare/pickleshare-0.5.tar.gz") + + extends("python") + + def install(self, spec, prefix): + python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-pil/package.py b/var/spack/packages/py-pil/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-pip/package.py b/var/spack/packages/py-pip/package.py new file mode 100644 index 0000000000..89b8570fe4 --- /dev/null +++ b/var/spack/packages/py-pip/package.py @@ -0,0 +1,12 @@ +from spack import * + +class PyPip(Package): + """The PyPA recommended tool for installing Python packages.""" + homepage = "https://pip.pypa.io/" + version("7.1.2", "3823d2343d9f3aaab21cf9c917710196", + url="https://pypi.python.org/packages/source/p/pip/pip-7.1.2.tar.gz") + + extends("python") + + def install(self, spec, prefix): + python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-pmw/package.py b/var/spack/packages/py-pmw/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-py/package.py b/var/spack/packages/py-py/package.py new file mode 100644 index 0000000000..0ef0403124 --- /dev/null +++ b/var/spack/packages/py-py/package.py @@ -0,0 +1,12 @@ +from spack import * + +class PyPy(Package): + """library with cross-python path, ini-parsing, io, code, log facilities""" + homepage = "http://pylib.readthedocs.org/" + version("1.4.26", "30c3fd92a53f1a5ed6f3591c1fe75c0e", + url="https://pypi.python.org/packages/source/p/py/py-1.4.26.tar.gz") + + extends("python") + + def install(self, spec, prefix): + python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-py4j/package.py b/var/spack/packages/py-py4j/package.py new file mode 100755 index 0000000000..4c31f57298 --- /dev/null +++ b/var/spack/packages/py-py4j/package.py @@ -0,0 +1,15 @@ +from spack import * + + +class PyPy4j(Package): + """Enables Python programs to dynamically access arbitrary Java objects""" + homepage = "https://www.py4j.org/" + url = "https://pypi.python.org/packages/source/p/py4j/py4j-0.9.tar.gz" + + version('0.9', 'b6fed5faef81a5368e3d50a91a5c9a60') + + extends("python") + depends_on("py-setuptools@18.1") + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-pychecker/package.py b/var/spack/packages/py-pychecker/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-pycparser/package.py b/var/spack/packages/py-pycparser/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-pyelftools/package.py b/var/spack/packages/py-pyelftools/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-pygments/package.py b/var/spack/packages/py-pygments/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-pylint/package.py b/var/spack/packages/py-pylint/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-pypar/package.py b/var/spack/packages/py-pypar/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-pyparsing/package.py b/var/spack/packages/py-pyparsing/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-pyqt/package.py b/var/spack/packages/py-pyqt/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-pyside/package.py b/var/spack/packages/py-pyside/package.py old mode 100644 new mode 100755 index bb5da44d02..8d093efd2c --- a/var/spack/packages/py-pyside/package.py +++ b/var/spack/packages/py-pyside/package.py @@ -8,12 +8,9 @@ class PyPyside(Package): version('1.2.2', 'c45bc400c8a86d6b35f34c29e379e44d') - # TODO: make build dependency - # depends_on("cmake") - extends('python') depends_on('py-setuptools') - depends_on('qt@:4') + depends_on('qt@5.4.0') def patch(self): """Undo PySide RPATH handling and add Spack RPATH.""" diff --git a/var/spack/packages/py-pytest/package.py b/var/spack/packages/py-pytest/package.py new file mode 100644 index 0000000000..3611b68862 --- /dev/null +++ b/var/spack/packages/py-pytest/package.py @@ -0,0 +1,12 @@ +from spack import * + +class PyPytest(Package): + """pytest: simple powerful testing with Python""" + homepage = "http://pytest.org" + version("2.6.4", "14341e122f7e9031a0948eb6b01a2640", + url="https://pypi.python.org/packages/source/p/pytest/pytest-2.6.4.tar.gz") + + extends("python") + + def install(self, spec, prefix): + python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-python-daemon/package.py b/var/spack/packages/py-python-daemon/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-python-dateutil/package.py b/var/spack/packages/py-python-dateutil/package.py new file mode 100644 index 0000000000..8efa114db1 --- /dev/null +++ b/var/spack/packages/py-python-dateutil/package.py @@ -0,0 +1,12 @@ +from spack import * + +class PyPythonDateutil(Package): + """Extensions to the standard Python datetime module""" + homepage = "http://labix.org/python-dateutil" + version("2.2", "c1f654d0ff7e33999380a8ba9783fd5c", + url="https://pypi.python.org/packages/source/p/python-dateutil/python-dateutil-2.2.tar.gz") + + extends("python") + + def install(self, spec, prefix): + python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-pytz/package.py b/var/spack/packages/py-pytz/package.py old mode 100644 new mode 100755 index da6311a784..f67bb42854 --- a/var/spack/packages/py-pytz/package.py +++ b/var/spack/packages/py-pytz/package.py @@ -5,6 +5,8 @@ class PyPytz(Package): homepage = "https://pypi.python.org/pypi/pytz" url = "https://pypi.python.org/packages/source/p/pytz/pytz-2014.10.tar.gz" + version('2014.9','d42bda2f4c1e873e02fbd1e4acfd1b8c', + url='https://pypi.python.org/packages/source/p/pytz/pytz-2014.9.tar.gz') version('2014.10', 'eb1cb941a20c5b751352c52486aa1dd7') version('2015.4', '417a47b1c432d90333e42084a605d3d8') diff --git a/var/spack/packages/py-pyzmq/package.py b/var/spack/packages/py-pyzmq/package.py new file mode 100755 index 0000000000..f5551182b3 --- /dev/null +++ b/var/spack/packages/py-pyzmq/package.py @@ -0,0 +1,16 @@ +from spack import * + +class PyPyzmq(Package): + """official Python binding for the ZeroMQ Messaging Library""" + homepage = "https://github.com/zeromq/pyzmq" + url = "https://pypi.python.org/packages/source/p/pyzmq/pyzmq-14.7.0.tar.gz" + + version('14.4.1','a6a7fb6950b1bf5333c6795f3edf196c', + url='https://pypi.python.org/packages/source/p/pyzmq/pyzmq-14.4.1.tar.gz') + version('14.7.0', '87e3abb33af5794db5ae85c667bbf324') + + extends('python') + depends_on('py-setuptools@18.1') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-rpy2/package.py b/var/spack/packages/py-rpy2/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-scientificpython/package.py b/var/spack/packages/py-scientificpython/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-scikit-learn/package.py b/var/spack/packages/py-scikit-learn/package.py old mode 100644 new mode 100755 index 5b078ce901..744b37e7e0 --- a/var/spack/packages/py-scikit-learn/package.py +++ b/var/spack/packages/py-scikit-learn/package.py @@ -9,6 +9,9 @@ class PyScikitLearn(Package): version('0.16.1', '363ddda501e3b6b61726aa40b8dbdb7e') extends('python') + depends_on('python@2.7.10') + depends_on('py-numpy@1.9.2') + depends_on('py-scipy@0.15.1') def install(self, spec, prefix): python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-scipy/package.py b/var/spack/packages/py-scipy/package.py old mode 100644 new mode 100755 index 3a1124cc15..bcbdb2100f --- a/var/spack/packages/py-scipy/package.py +++ b/var/spack/packages/py-scipy/package.py @@ -5,14 +5,14 @@ class PyScipy(Package): homepage = "https://pypi.python.org/pypi/scipy" url = "https://pypi.python.org/packages/source/s/scipy/scipy-0.15.0.tar.gz" + version('0.14.0','d7c7f4ccf8b07b08d6fe49d5cd51f85d', + url='https://pypi.python.org/packages/source/s/scipy/scipy-0.14.0.tar.gz') version('0.15.0', '639112f077f0aeb6d80718dc5019dc7a') version('0.15.1', 'be56cd8e60591d6332aac792a5880110') extends('python') depends_on('py-nose') depends_on('py-numpy') - depends_on('blas') - depends_on('lapack') def install(self, spec, prefix): python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-seaborn/package.py b/var/spack/packages/py-seaborn/package.py new file mode 100644 index 0000000000..1e228b0622 --- /dev/null +++ b/var/spack/packages/py-seaborn/package.py @@ -0,0 +1,12 @@ +from spack import * + +class PySeaborn(Package): + """Seaborn: statistical data visualization""" + homepage = "http://stanford.edu/~mwaskom/software/seaborn/" + version("0.5.1", "2ce6ea7d3c67858c0b1f5793fa2043b8", + url="https://pypi.python.org/packages/source/s/seaborn/seaborn-0.5.1.tar.gz") + + extends("python") + + def install(self, spec, prefix): + python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-setuptools/package.py b/var/spack/packages/py-setuptools/package.py old mode 100644 new mode 100755 index 760ad4d6db..053fe1ba80 --- a/var/spack/packages/py-setuptools/package.py +++ b/var/spack/packages/py-setuptools/package.py @@ -3,11 +3,12 @@ class PySetuptools(Package): """Easily download, build, install, upgrade, and uninstall Python packages.""" homepage = "https://pypi.python.org/pypi/setuptools" - url = "https://pypi.python.org/packages/source/s/setuptools/setuptools-11.3.tar.gz" - version('11.3.1', '01f69212e019a2420c1693fb43593930') - version('16.0', '0ace0b96233516fc5f7c857d086aa3ad') - version('18.1', 'f72e87f34fbf07f299f6cb46256a0b06') + version('18.1', 'f72e87f34fbf07f299f6cb46256a0b06', + url='https://pypi.python.org/packages/source/s/setuptools/setuptools-18.1.tar.gz') + version('11.3.1', '01f69212e019a2420c1693fb43593930', + url="https://pypi.python.org/packages/source/s/setuptools/setuptools-11.3.tar.gz") + version('16.0', '0ace0b96233516fc5f7c857d086aa3ad',) extends('python') diff --git a/var/spack/packages/py-shiboken/package.py b/var/spack/packages/py-shiboken/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-sip/package.py b/var/spack/packages/py-sip/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-six/package.py b/var/spack/packages/py-six/package.py old mode 100644 new mode 100755 index 05c5bd00a9..e9946a1e56 --- a/var/spack/packages/py-six/package.py +++ b/var/spack/packages/py-six/package.py @@ -5,6 +5,8 @@ class PySix(Package): homepage = "https://pypi.python.org/pypi/six" url = "https://pypi.python.org/packages/source/s/six/six-1.9.0.tar.gz" + version('1.8.0','1626eb24cc889110c38f7e786ec69885', + url='https://pypi.python.org/packages/source/s/six/six-1.8.0.tar.gz') version('1.9.0', '476881ef4012262dfc8adc645ee786c4') extends('python') diff --git a/var/spack/packages/py-statsmodels/package.py b/var/spack/packages/py-statsmodels/package.py new file mode 100644 index 0000000000..393c772738 --- /dev/null +++ b/var/spack/packages/py-statsmodels/package.py @@ -0,0 +1,12 @@ +from spack import * + +class PyStatsmodels(Package): + """Statistical computations and models for use with SciPy""" + homepage = "http://statsmodels.sourceforge.net/" + version("0.6.1", "f7580ebf7d2a2c9b87abfad190dcb9a3", + url="https://pypi.python.org/packages/source/s/statsmodels/statsmodels-0.6.1.tar.gz") + + extends("python") + + def install(self, spec, prefix): + python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-sympy/package.py b/var/spack/packages/py-sympy/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-tables/package.py b/var/spack/packages/py-tables/package.py new file mode 100644 index 0000000000..637ccf34cd --- /dev/null +++ b/var/spack/packages/py-tables/package.py @@ -0,0 +1,12 @@ +from spack import * + +class PyTables(Package): + """Hierarchical datasets for Python""" + homepage = "http://www.pytables.org/" + version("3.1.1", "38d917f0c6dfb0bc28ce9ea0c3492524", + url="https://pypi.python.org/packages/source/t/tables/tables-3.1.1.tar.gz") + + extends("python") + + def install(self, spec, prefix): + python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-tornado/package.py b/var/spack/packages/py-tornado/package.py new file mode 100755 index 0000000000..4fc2d42c7b --- /dev/null +++ b/var/spack/packages/py-tornado/package.py @@ -0,0 +1,17 @@ +from spack import * + +class PyTornado(Package): + """Python web framework and asynchronous networking library""" + homepage = "http://www.tornadoweb.org/" + url = "https://pypi.python.org/packages/source/t/tornado/tornado-4.2.1.tar.gz" + + version('4.0.2','985c0e704b765c33a6193d49d1935588', + url='https://pypi.python.org/packages/source/t/tornado/tornado-4.0.2.tar.gz') + version('4.2.1', 'd523204389cfb70121bb69709f551b20') + + extends('python') + depends_on('py-certifi') + depends_on('py-setuptools@18.1') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-traitlets/package.py b/var/spack/packages/py-traitlets/package.py new file mode 100755 index 0000000000..7b7f5c64cc --- /dev/null +++ b/var/spack/packages/py-traitlets/package.py @@ -0,0 +1,14 @@ +from spack import * + +class PyTraitlets(Package): + """A lightweight Traits like module""" + homepage = "http://traitlets.readthedocs.org/" + url = "https://github.com/ipython/traitlets/archive/4.0.0.tar.gz" + + version('4.0.0', 'b5b95ea5941fd9619b4704dfd8201568') + + extends('python') + depends_on("py-setuptools@18.1") + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-triangle-plot/package.py b/var/spack/packages/py-triangle-plot/package.py new file mode 100644 index 0000000000..29c5f92857 --- /dev/null +++ b/var/spack/packages/py-triangle-plot/package.py @@ -0,0 +1,12 @@ +from spack import * + +class PyTrianglePlot(Package): + """Make some beautiful corner plots of samples.""" + homepage = "https://github.com/dfm/triangle.py" + version("0.0.6", "970a35a9bde6002d673e3188fe39f0ed", + url="https://pypi.python.org/packages/source/t/triangle_plot/triangle_plot-0.0.6.tar.gz") + + extends("python") + + def install(self, spec, prefix): + python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-virtualenv/package.py b/var/spack/packages/py-virtualenv/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-yapf/package.py b/var/spack/packages/py-yapf/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/python/package.py b/var/spack/packages/python/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/qhull/package.py b/var/spack/packages/qhull/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/qt/package.py b/var/spack/packages/qt/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/qthreads/package.py b/var/spack/packages/qthreads/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/ravel/package.py b/var/spack/packages/ravel/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/readline/package.py b/var/spack/packages/readline/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/rose/add_spack_compiler_recognition.patch b/var/spack/packages/rose/add_spack_compiler_recognition.patch old mode 100644 new mode 100755 diff --git a/var/spack/packages/rose/package.py b/var/spack/packages/rose/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/ruby/package.py b/var/spack/packages/ruby/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/samtools/package.py b/var/spack/packages/samtools/package.py deleted file mode 100644 index 72900398d8..0000000000 --- a/var/spack/packages/samtools/package.py +++ /dev/null @@ -1,18 +0,0 @@ -from spack import * - -class Samtools(Package): - """SAM Tools provide various utilities for manipulating alignments in the SAM format, - including sorting, merging, indexing and generating - alignments in a per-position format""" - - homepage = "www.htslib.org" - version('1.2','988ec4c3058a6ceda36503eebecd4122',url = "https://github.com/samtools/samtools/releases/download/1.2/samtools-1.2.tar.bz2") - - depends_on("zlib") - depends_on("mpc") - parallel=False - patch("samtools1.2.patch",level=0) - - def install(self, spec, prefix): - make("prefix=%s" % prefix, "install") - diff --git a/var/spack/packages/samtools/samtools1.2.patch b/var/spack/packages/samtools/samtools1.2.patch deleted file mode 100644 index ead3ab4e2c..0000000000 --- a/var/spack/packages/samtools/samtools1.2.patch +++ /dev/null @@ -1,20 +0,0 @@ ---- Makefile 2015-02-03 08:27:34.000000000 -0800 -+++ Makefile.new 2015-07-21 10:38:27.881406892 -0700 -@@ -26,7 +26,7 @@ - CFLAGS = -g -Wall -O2 - LDFLAGS = - LDLIBS = --DFLAGS= -D_FILE_OFFSET_BITS=64 -D_LARGEFILE64_SOURCE -D_CURSES_LIB=1 -+DFLAGS= -D_FILE_OFFSET_BITS=64 -D_LARGEFILE64_SOURCE -D_CURSES_LIB=0 - LOBJS= bam_aux.o bam.o bam_import.o sam.o \ - sam_header.o bam_plbuf.o - AOBJS= bam_index.o bam_plcmd.o sam_view.o \ -@@ -37,7 +37,7 @@ - faidx.o stats.o stats_isize.o bam_flags.o bam_split.o \ - bam_tview.o bam_tview_curses.o bam_tview_html.o bam_lpileup.o - INCLUDES= -I. -I$(HTSDIR) --LIBCURSES= -lcurses # -lXCurses -+#LIBCURSES= -lcurses # -lXCurses - - prefix = /usr/local - exec_prefix = $(prefix) diff --git a/var/spack/packages/scalasca/package.py b/var/spack/packages/scalasca/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/scorep/package.py b/var/spack/packages/scorep/package.py old mode 100644 new mode 100755 index f013bd1cbb..32a772e3db --- a/var/spack/packages/scorep/package.py +++ b/var/spack/packages/scorep/package.py @@ -1,10 +1,11 @@ # FIXME: Add copyright statement from spack import * +from contextlib import closing class Scorep(Package): - """The Score-P measurement infrastructure is a highly scalable and - easy-to-use tool suite for profiling, event tracing, and online + """The Score-P measurement infrastructure is a highly scalable and + easy-to-use tool suite for profiling, event tracing, and online analysis of HPC applications.""" # FIXME: add a proper url for your package's homepage here. @@ -19,7 +20,7 @@ class Scorep(Package): depends_on("mpi") depends_on("papi") # depends_on("otf2@1.2:1.2.1") # only Score-P 1.2.x - depends_on("otf2") + depends_on("otf2") depends_on("opari2") depends_on("cube@4.2:4.2.3") @@ -52,12 +53,12 @@ def install(self, spec, prefix): # Use a custom compiler configuration, otherwise the score-p # build system messes with spack's compiler settings. # Create these three files in the build directory - with open("platform-backend-user-provided", "w") as backend_file: + with closing(open("platform-backend-user-provided", "w")) as backend_file: backend_file.write(self.backend_user_provided) - with open("platform-frontend-user-provided", "w") as frontend_file: + with closing(open("platform-frontend-user-provided", "w")) as frontend_file: frontend_file.write(self.frontend_user_provided) - with open("platform-mpi-user-provided", "w") as mpi_file: - mpi_file.write(self.mpi_user_provided) + with closing(open("platform-mpi-user-provided", "w")) as mpi_file: + mpi_file.write(self.mpi_user_provided) configure_args = ["--prefix=%s" % prefix, "--with-custom-compilers", diff --git a/var/spack/packages/scotch/package.py b/var/spack/packages/scotch/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/scr/package.py b/var/spack/packages/scr/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/silo/package.py b/var/spack/packages/silo/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/snappy/package.py b/var/spack/packages/snappy/package.py deleted file mode 100644 index c8f9ceef7d..0000000000 --- a/var/spack/packages/snappy/package.py +++ /dev/null @@ -1,15 +0,0 @@ -import os -from spack import * - -class Snappy(Package): - """A fast compressor/decompressor: https://code.google.com/p/snappy""" - - homepage = "https://code.google.com/p/snappy" - url = "https://github.com/google/snappy/releases/download/1.1.3/snappy-1.1.3.tar.gz" - - version('1.1.3', '7358c82f133dc77798e4c2062a749b73') - - def install(self, spec, prefix): - configure("--prefix=" + prefix) - make() - make("install") diff --git a/var/spack/packages/spindle/package.py b/var/spack/packages/spindle/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/sqlite/package.py b/var/spack/packages/sqlite/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/stat/configure_mpicxx.patch b/var/spack/packages/stat/configure_mpicxx.patch old mode 100644 new mode 100755 diff --git a/var/spack/packages/stat/package.py b/var/spack/packages/stat/package.py old mode 100644 new mode 100755 index 5d81e62731..c40d65ae78 --- a/var/spack/packages/stat/package.py +++ b/var/spack/packages/stat/package.py @@ -9,20 +9,17 @@ class Stat(Package): version('2.1.0', 'ece26beaf057aa9134d62adcdda1ba91') version('2.0.0', 'c7494210b0ba26b577171b92838e1a9b') - variant('dysect', default=False, description="enable DySectAPI") - depends_on('libelf') depends_on('libdwarf') depends_on('dyninst') depends_on('graphlib') - depends_on('graphviz') depends_on('launchmon') depends_on('mrnet') patch('configure_mpicxx.patch', when='@2.1.0') def install(self, spec, prefix): - configure_args = [ + configure( "--enable-gui", "--prefix=%s" % prefix, "--disable-examples", # Examples require MPI: avoid this dependency. @@ -30,11 +27,7 @@ def install(self, spec, prefix): "--with-mrnet=%s" % spec['mrnet'].prefix, "--with-graphlib=%s" % spec['graphlib'].prefix, "--with-stackwalker=%s" % spec['dyninst'].prefix, - "--with-libdwarf=%s" % spec['libdwarf'].prefix - ] - if '+dysect' in spec: - configure_args.append('--enable-dysectapi') - configure(*configure_args) + "--with-libdwarf=%s" % spec['libdwarf'].prefix) make(parallel=False) make("install") diff --git a/var/spack/packages/sundials/package.py b/var/spack/packages/sundials/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/swig/package.py b/var/spack/packages/swig/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/task/package.py b/var/spack/packages/task/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/taskd/package.py b/var/spack/packages/taskd/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/tau/package.py b/var/spack/packages/tau/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/tcl/package.py b/var/spack/packages/tcl/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/the_silver_searcher/package.py b/var/spack/packages/the_silver_searcher/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/thrift/package.py b/var/spack/packages/thrift/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/tk/package.py b/var/spack/packages/tk/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/tmux/package.py b/var/spack/packages/tmux/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/tmuxinator/package.py b/var/spack/packages/tmuxinator/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/uncrustify/package.py b/var/spack/packages/uncrustify/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/util-linux/package.py b/var/spack/packages/util-linux/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/vim/package.py b/var/spack/packages/vim/package.py old mode 100644 new mode 100755 index 4099b3257f..d0cf80b898 --- a/var/spack/packages/vim/package.py +++ b/var/spack/packages/vim/package.py @@ -35,26 +35,15 @@ class Vim(Package): variant('ruby', default=False, description="build with Ruby") depends_on('ruby', when='+ruby') - variant('cscope', default=False, description="build with cscope support") - depends_on('cscope', when='+cscope') - - variant('gui', default=False, description="build with gui (gvim)") - # virtual dependency? - def install(self, spec, prefix): feature_set = None for fs in self.feature_sets: if "+" + fs in spec: if feature_set is not None: - tty.error("Only one feature set allowed, both %s and %s specified" - % (feature_set, fs)) + tty.error("Only one feature set allowed, both {} and {} specified".format( + feature_set, + fs)) feature_set = fs - if '+gui' in spec: - if feature_set is not None: - if feature_set is not 'huge': - tty.error("+gui variant requires 'huge' feature set, %s was specified" - % feature_set) - feature_set = 'huge' if feature_set is None: feature_set = 'normal' @@ -71,12 +60,6 @@ def install(self, spec, prefix): else: configure_args.append("--enable-rubyinterp=dynamic") - if '+gui' in spec: - configure_args.append("--enable-gui=auto") - - if '+cscope' in spec: - configure_args.append("--enable-cscope") - configure("--prefix=%s" % prefix, *configure_args) make() diff --git a/var/spack/packages/vtk/package.py b/var/spack/packages/vtk/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/wget/package.py b/var/spack/packages/wget/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/wx/package.py b/var/spack/packages/wx/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/wxpropgrid/package.py b/var/spack/packages/wxpropgrid/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/xcb-proto/package.py b/var/spack/packages/xcb-proto/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/xz/package.py b/var/spack/packages/xz/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/yasm/package.py b/var/spack/packages/yasm/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/zeromq/package.py b/var/spack/packages/zeromq/package.py deleted file mode 100644 index b5a1e3d4cd..0000000000 --- a/var/spack/packages/zeromq/package.py +++ /dev/null @@ -1,20 +0,0 @@ -from spack import * - -class Zeromq(Package): - """ The ZMQ networking/concurrency library and core API """ - homepage = "http://zguide.zeromq.org/" - url = "http://download.zeromq.org/zeromq-4.1.2.tar.gz" - - version('4.1.2', '159c0c56a895472f02668e692d122685') - version('4.1.1', '0a4b44aa085644f25c177f79dc13f253') - version('4.0.7', '9b46f7e7b0704b83638ef0d461fd59ab') - version('4.0.6', 'd47dd09ed7ae6e7fd6f9a816d7f5fdf6') - version('4.0.5', '73c39f5eb01b9d7eaf74a5d899f1d03d') - - depends_on("libsodium") - - def install(self, spec, prefix): - configure("--with-libsodium","--prefix=%s" % prefix) - - make() - make("install") diff --git a/var/spack/packages/zlib/package.py b/var/spack/packages/zlib/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/zsh/package.py b/var/spack/packages/zsh/package.py old mode 100644 new mode 100755 From e9f7d033ff04138905c99cb427e2071cc55cdc16 Mon Sep 17 00:00:00 2001 From: karenyyng Date: Sat, 3 Oct 2015 19:13:08 -0700 Subject: [PATCH 004/481] make sure submodule is added in the correct location --- .gitmodules | 6 +++--- lib/spack/spack/util/python_recipe_parser | 1 + var/spack/packages/py-astropy/package.py | 1 + var/spack/packages/py-jsonschema/package.py | 2 ++ 4 files changed, 7 insertions(+), 3 deletions(-) create mode 160000 lib/spack/spack/util/python_recipe_parser diff --git a/.gitmodules b/.gitmodules index 1d770f415b..d5995e8e4b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ -[submodule "/lib/spack/spack/util/python_recipe_parser"] - path = /lib/spack/spack/util/python_recipe_parser - url = https://github.com/karenyyng/spack_python_package_parser +[submodule "lib/spack/spack/util/python_recipe_parser"] + path = lib/spack/spack/util/python_recipe_parser + url = git@github.com:karenyyng/spack_python_package_parser.git diff --git a/lib/spack/spack/util/python_recipe_parser b/lib/spack/spack/util/python_recipe_parser new file mode 160000 index 0000000000..6777dd98e7 --- /dev/null +++ b/lib/spack/spack/util/python_recipe_parser @@ -0,0 +1 @@ +Subproject commit 6777dd98e75229856d94da6671676b2a66e75986 diff --git a/var/spack/packages/py-astropy/package.py b/var/spack/packages/py-astropy/package.py index 5d2d0619cc..9fda37519e 100644 --- a/var/spack/packages/py-astropy/package.py +++ b/var/spack/packages/py-astropy/package.py @@ -7,6 +7,7 @@ class PyAstropy(Package): url="https://pypi.python.org/packages/source/a/astropy/astropy-0.4.2.tar.gz") extends("python") + depends_on("py-numpy") def install(self, spec, prefix): python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-jsonschema/package.py b/var/spack/packages/py-jsonschema/package.py index 9a7fc21862..fa233635ce 100755 --- a/var/spack/packages/py-jsonschema/package.py +++ b/var/spack/packages/py-jsonschema/package.py @@ -11,6 +11,8 @@ class PyJsonschema(Package): extends("python") depends_on("py-setuptools@18.1") + depends_on("py-vcversioner") + depends_on("py-functools32") def install(self, spec, prefix): python('setup.py', 'install', '--prefix=%s' % prefix) From f3254ff02d589c9e177aab58276f5c1971866018 Mon Sep 17 00:00:00 2001 From: karenyyng Date: Sat, 3 Oct 2015 19:22:53 -0700 Subject: [PATCH 005/481] removed package for python packages with problematic / duplicate names --- var/spack/packages/py-Cython/package.py | 12 ------------ var/spack/packages/py-Distutils2/package.py | 15 --------------- 2 files changed, 27 deletions(-) delete mode 100644 var/spack/packages/py-Cython/package.py delete mode 100644 var/spack/packages/py-Distutils2/package.py diff --git a/var/spack/packages/py-Cython/package.py b/var/spack/packages/py-Cython/package.py deleted file mode 100644 index c250139114..0000000000 --- a/var/spack/packages/py-Cython/package.py +++ /dev/null @@ -1,12 +0,0 @@ -from spack import * - -class PyCython(Package): - """The Cython compiler for writing C extensions for the Python language.""" - homepage = "http://www.cython.org" - version("0.21.2", "d21adb870c75680dc857cd05d41046a4", - url="https://pypi.python.org/packages/source/C/Cython/Cython-0.21.2.tar.gz") - - extends("python") - - def install(self, spec, prefix): - python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-Distutils2/package.py b/var/spack/packages/py-Distutils2/package.py deleted file mode 100644 index 8690f569fe..0000000000 --- a/var/spack/packages/py-Distutils2/package.py +++ /dev/null @@ -1,15 +0,0 @@ -from spack import * - -class PyDistutils2(Package): - """Python Packaging Library.""" - - homepage = "https://hg.python.org/distutils2" - url = "https://pypi.python.org/packages/source/D/Distutils2/Distutils2-1.0a4.tar.gz" - - version('1.0a4', '52bc9dffb394970c27e02853ae3a3241') - - depends_on("python") - - def install(self, spec, prefix): - python('setup.py', 'install', '--prefix=%s' % prefix) - From c809cc92735c7ff57dab2aca02bc7fa40c901a43 Mon Sep 17 00:00:00 2001 From: karenyyng Date: Sat, 3 Oct 2015 19:24:15 -0700 Subject: [PATCH 006/481] clean up package.py file and remove duplicate file --- var/spack/packages/py-ipython/package.py | 4 --- .../packages/py-ipython/package.py.backup | 27 ------------------- 2 files changed, 31 deletions(-) delete mode 100755 var/spack/packages/py-ipython/package.py.backup diff --git a/var/spack/packages/py-ipython/package.py b/var/spack/packages/py-ipython/package.py index 3b94b71f29..225366552a 100755 --- a/var/spack/packages/py-ipython/package.py +++ b/var/spack/packages/py-ipython/package.py @@ -24,8 +24,4 @@ class PyIpython(Package): depends_on('py-jinja2') def install(self, spec, prefix): - import os - # os.system("spack activate py-setuptools@18.1") - # os.system("easy_install pip") - # os.system("pip install ipython[notebook]") python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-ipython/package.py.backup b/var/spack/packages/py-ipython/package.py.backup deleted file mode 100755 index 1ed8b57e60..0000000000 --- a/var/spack/packages/py-ipython/package.py.backup +++ /dev/null @@ -1,27 +0,0 @@ -from spack import * - -class PyIpython(Package): - """IPython provides a rich toolkit to help you make the most out of using Python interactively.""" - homepage = "https://pypi.python.org/pypi/ipython" - - version('2.3.1', '2b7085525dac11190bfb45bb8ec8dcbf', - url="https://pypi.python.org/packages/source/i/ipython/ipython-2.3.1.tar.gz") - version('3.1.0', 'a749d90c16068687b0ec45a27e72ef8f') - version('3.2.1', 'f4c93d67ac4b2d4fc69df693b6f3c9e0', - url='https://github.com/ipython/ipython/archive/rel-3.2.1.tar.gz') - - extends('python') - depends_on('py-pygments') - depends_on('py-setuptools@18.1') - depends_on('py-traitlets') - depends_on('py-pyzmq') - depends_on('py-pexpect') - depends_on('py-tornado') - depends_on('py-Jinja2') - - def install(self, spec, prefix): - import os - # os.system("spack activate py-setuptools@18.1") - # os.system("easy_install pip") - # os.system("pip install ipython[notebook]") - python('setup.py', 'install', '--prefix=%s' % prefix) From d00314c621bd427b142fecafc11c25006f3b5279 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 5 Oct 2015 01:30:25 -0700 Subject: [PATCH 007/481] Protptype cray compiler detection and support --- lib/spack/env/cc | 17 +++++++ lib/spack/spack/build_environment.py | 9 ++++ lib/spack/spack/compiler.py | 67 ++++++++++++++++++++++++++- lib/spack/spack/compilers/__init__.py | 9 ++++ lib/spack/spack/compilers/clang.py | 2 +- lib/spack/spack/compilers/gcc.py | 7 ++- lib/spack/spack/compilers/intel.py | 3 ++ lib/spack/spack/compilers/pgi.py | 3 ++ lib/spack/spack/compilers/xl.py | 4 +- 9 files changed, 116 insertions(+), 5 deletions(-) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index fa85bb595e..b6c6e03e42 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -70,6 +70,23 @@ for param in $parameters; do fi done + +# +# Cray module environment-related stuff. +# +if [ ! -z "$SPACK_CRAYPE" ]; then + cur_pe=$(module list 2>&1 | grep PrgEnv | grep -o 'PrgEnv-[^/]*') + if [ ! -z "$cur_pe" ]; then + module swap $cur_pe $SPACK_CRAYPE + else + module load $SPACK_CRAYPE + fi +fi + +if [ ! -z "$SPACK_COMP_MODULE" ]; then + module load $SPACK_COMP_MODULE +fi + # # Figure out the type of compiler, the language, and the mode so that # the compiler script knows what to do. diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index a133faa629..68388958f5 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -57,6 +57,9 @@ SPACK_SHORT_SPEC = 'SPACK_SHORT_SPEC' SPACK_DEBUG_LOG_DIR = 'SPACK_DEBUG_LOG_DIR' +SPACK_CRAYPE = 'SPACK_CRAYPE' +SPACK_COMP_MODULE = 'SPACK_COMP_MODULE' + class MakeExecutable(Executable): """Special callable executable object for make so the user can @@ -105,6 +108,12 @@ def set_compiler_environment_variables(pkg): os.environ['SPACK_COMPILER_SPEC'] = str(pkg.spec.compiler) + if compiler.PrgEnv: + os.environ['SPACK_CRAYPE'] = compiler.PrgEnv + os.environ['SPACK_COMP_MODULE'] = compiler.module + + + def set_build_environment_variables(pkg): """This ensures a clean install environment when we build packages. diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index 1e800a8979..e7d450ee8b 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -25,6 +25,7 @@ import os import re import itertools +import subprocess from datetime import datetime import llnl.util.tty as tty @@ -98,7 +99,14 @@ class Compiler(object): cxx11_flag = "-std=c++11" - def __init__(self, cspec, cc, cxx, f77, fc): + # Cray PrgEnv name that can be used to load this compiler + PrgEnv = None + + # Name of module used to switch versions of this compiler + PrgEnv_compiler = None + + + def __init__(self, cspec, cc, cxx, f77, fc, module=None): def check(exe): if exe is None: return None @@ -111,6 +119,8 @@ def check(exe): self.fc = check(fc) self.spec = cspec + self.module = module + @property @@ -255,6 +265,61 @@ def find(cls, *path): return list(compilers.values()) + @classmethod + def find_in_modules(cls): + compilers = [] + + if cls.PrgEnv: + if not cls.PrgEnv_compiler: + tty.die('Must supply PrgEnv_compiler with PrgEnv') + + output = _shell('module avail %s' % cls.PrgEnv_compiler) + matches = re.findall(r'(%s)/([^\s(]*)' % cls.PrgEnv_compiler, output) + + for name, version in matches: + v = version + '-craype' + comp = cls(spack.spec.CompilerSpec(name + '@' + v), + 'cc', 'CC', 'ftn', 'ftn', name +'/' + v) + + compilers.append(comp) + + return compilers + + +def _cur_prgenv(): + out, err = subprocess.Popen( + ['module list'], shell=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() + matches = re.findall(r'(PrgEnv-[^/]*)/', err) + return matches[0] + + +def _module_shell(module, *args): + cmd = 'module swap %s %s;' % (_cur_prgenv(), module) + cmd += 'module load %s;' % compiler + cmd += 'module unload cray-libsci;' + +# + +# 'module load craype-network-gemini;' + +# 'module load %s;' % module + +# 'module swap gcc/4.6.1;' + +# 'module load eswrap; ' + +# 'module load craype-mc12; ' + +# 'module load cray-shmem; ' + +# 'module load cray-mpich; ') + cmd += ' '.join(args) + out, err = subprocess.Popen([cmd + ' '.join(args)], shell=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() + return out + + +def _shell(*args): + return subprocess.Popen([' '.join(args)], shell=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[1] + + + + def __repr__(self): """Return a string represntation of the compiler toolchain.""" return self.__str__() diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index b7b021a1ac..8c4cb38926 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -44,6 +44,7 @@ _imported_compilers_module = 'spack.compilers' _required_instance_vars = ['cc', 'cxx', 'f77', 'fc'] +_optional_instance_vars = ['module'] _default_order = ['gcc', 'intel', 'pgi', 'clang', 'xlc'] @@ -127,11 +128,18 @@ def add_compilers_to_config(scope, *compilers): compiler_config_tree = {} for compiler in compilers: compiler_entry = {} + for c in _required_instance_vars: val = getattr(compiler, c) if not val: val = "None" compiler_entry[c] = val + + for c in _optional_instance_vars: + val = getattr(compiler, c) + if val: + compiler_entry[c] = val + compiler_config_tree[str(compiler.spec)] = compiler_entry spack.config.add_to_compiler_config(compiler_config_tree, scope) @@ -220,6 +228,7 @@ def class_for_compiler_name(compiler_name): def all_compiler_types(): +# return [class_for_compiler_name(c) for c in ['gcc']] return [class_for_compiler_name(c) for c in supported_compilers()] diff --git a/lib/spack/spack/compilers/clang.py b/lib/spack/spack/compilers/clang.py index 790901c86e..8d32608ff4 100644 --- a/lib/spack/spack/compilers/clang.py +++ b/lib/spack/spack/compilers/clang.py @@ -39,7 +39,7 @@ class Clang(Compiler): @classmethod - def default_version(self, comp): + def default_version(cls, comp): """The '--version' option works for clang compilers. Output looks like this:: diff --git a/lib/spack/spack/compilers/gcc.py b/lib/spack/spack/compilers/gcc.py index f0d27d590e..ff0b8889a8 100644 --- a/lib/spack/spack/compilers/gcc.py +++ b/lib/spack/spack/compilers/gcc.py @@ -42,6 +42,9 @@ class Gcc(Compiler): # MacPorts builds gcc versions with prefixes and -mp-X.Y suffixes. suffixes = [r'-mp-\d\.\d'] + PrgEnv = 'gnu' + PrgEnv_compiler = 'gcc' + @property def cxx11_flag(self): if self.version < ver('4.3'): @@ -56,9 +59,9 @@ def fc_version(cls, fc): return get_compiler_version( fc, '-dumpversion', # older gfortran versions don't have simple dumpversion output. - r'(?:GNU Fortran \(GCC\))?(\d+\.\d+(?:\.\d+)?)') + r'(?:GNU Fortran \(GCC\))?(\d+\.\d+(?:\.\d+)?)', module) @classmethod def f77_version(cls, f77): - return cls.fc_version(f77) + return cls.fc_version(f77, module) diff --git a/lib/spack/spack/compilers/intel.py b/lib/spack/spack/compilers/intel.py index 2a72c4eaea..7c485fe69d 100644 --- a/lib/spack/spack/compilers/intel.py +++ b/lib/spack/spack/compilers/intel.py @@ -37,6 +37,9 @@ class Intel(Compiler): # Subclasses use possible names of Fortran 90 compiler fc_names = ['ifort'] + PrgEnv = 'intel' + PrgEnv_compiler = 'intel' + @property def cxx11_flag(self): if self.version < ver('11.1'): diff --git a/lib/spack/spack/compilers/pgi.py b/lib/spack/spack/compilers/pgi.py index d97f24c12e..8f1ed28825 100644 --- a/lib/spack/spack/compilers/pgi.py +++ b/lib/spack/spack/compilers/pgi.py @@ -37,6 +37,9 @@ class Pgi(Compiler): # Subclasses use possible names of Fortran 90 compiler fc_names = ['pgf95', 'pgf90'] + PrgEnv = 'pgi' + PrgEnv_compiler = 'pgi' + @classmethod def default_version(cls, comp): """The '-V' option works for all the PGI compilers. diff --git a/lib/spack/spack/compilers/xl.py b/lib/spack/spack/compilers/xl.py index 562186b865..179b720918 100644 --- a/lib/spack/spack/compilers/xl.py +++ b/lib/spack/spack/compilers/xl.py @@ -45,8 +45,9 @@ def cxx11_flag(self): else: return "-qlanglvl=extended0x" + @classmethod - def default_version(self, comp): + def default_version(cls, comp): """The '-qversion' is the standard option fo XL compilers. Output looks like this:: @@ -72,6 +73,7 @@ def default_version(self, comp): return get_compiler_version( comp, '-qversion',r'([0-9]?[0-9]\.[0-9])') + @classmethod def fc_version(cls, fc): """The fortran and C/C++ versions of the XL compiler are always two units apart. From f4e72f33c8ba988507aba667e318d1861c7b2b20 Mon Sep 17 00:00:00 2001 From: karenyyng Date: Mon, 5 Oct 2015 08:24:33 -0700 Subject: [PATCH 008/481] added dependencies for py-h5py --- lib/spack/spack/util/python_recipe_parser | 2 +- var/spack/packages/py-h5py/package.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/util/python_recipe_parser b/lib/spack/spack/util/python_recipe_parser index 6777dd98e7..437a62abb3 160000 --- a/lib/spack/spack/util/python_recipe_parser +++ b/lib/spack/spack/util/python_recipe_parser @@ -1 +1 @@ -Subproject commit 6777dd98e75229856d94da6671676b2a66e75986 +Subproject commit 437a62abb3df7212e3ee20269c0089a0a9766fe0 diff --git a/var/spack/packages/py-h5py/package.py b/var/spack/packages/py-h5py/package.py index 6293da5407..04072ca76b 100755 --- a/var/spack/packages/py-h5py/package.py +++ b/var/spack/packages/py-h5py/package.py @@ -13,6 +13,8 @@ class PyH5py(Package): depends_on('hdf5') depends_on('py-numpy') depends_on('py-cython') + depends_on('py-six') + depends_on('py-pkgconfig') def install(self, spec, prefix): python('setup.py', 'configure', '--hdf5=%s' % spec['hdf5'].prefix) From b6d2a12ceb090692aa2be363b46d7f25486c0245 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 7 Oct 2015 15:57:29 -0700 Subject: [PATCH 009/481] Started changing the find in modules method written by Todd --- lib/spack/spack/compiler.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index e7d450ee8b..cbe6d8cd7d 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -272,9 +272,11 @@ def find_in_modules(cls): if cls.PrgEnv: if not cls.PrgEnv_compiler: tty.die('Must supply PrgEnv_compiler with PrgEnv') - - output = _shell('module avail %s' % cls.PrgEnv_compiler) - matches = re.findall(r'(%s)/([^\s(]*)' % cls.PrgEnv_compiler, output) + + loaded_modules = os.environ["LOADEDMODULES"].split(":") + #output = _shell('module avail %s' % cls.PrgEnv_compiler) + for module in loaded_modules: + match = re.findall(r'(%s)/([^\s(]*)' % cls.PrgEnv_compiler, module) for name, version in matches: v = version + '-craype' From 4f21344e87da327b4166fff1fd2ce32afaa07dbc Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 14 Oct 2015 19:41:07 -0700 Subject: [PATCH 010/481] Started created the Architecture class for Spack to use --- lib/spack/spack/architecture.py | 71 ++++++++++++++++++++++++++------- 1 file changed, 56 insertions(+), 15 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 0c4b605e91..829aaa4c1c 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -30,7 +30,7 @@ import spack import spack.error as serr from spack.version import Version - +from external import yaml class InvalidSysTypeError(serr.SpackError): def __init__(self, sys_type): @@ -43,41 +43,81 @@ def __init__(self): super(NoSysTypeError, self).__init__( "Could not determine sys_type for this machine.") +class Architecture(object): + def __init__(self, *arch_name): + + """ Constructor for the architecture class. Should return a dictionary of name (grabbed from uname) and a strategy for + searching for that architecture's compiler. The target passed to it should be a dictionary of names and strategies. + """ + self.arch_dict = {} + self.arch_name = arch_name + + def add_arch_strategy(self): + """ Create a dictionary using the tuples of arch_names""" + for n in self.arch_name: + if 'cray' in n.lower(): + self.arch_dict[n] = "MODULES" + if 'linux' in n.lower() or 'x86_64' in n.lower(): + self.arch_dict[n] = "PATH" + else: + self.arch_dict[n] = None def get_sys_type_from_spack_globals(): - """Return the SYS_TYPE from spack globals, or None if it isn't set.""" + """Return the SYS_TYPE from spack globals, or None if it isn't set. Front-end""" if not hasattr(spack, "sys_type"): - return None + return None elif hasattr(spack.sys_type, "__call__"): - return spack.sys_type() + return Architecture(spack.sys_type()) else: - return spack.sys_type - - -def get_sys_type_from_environment(): - """Return $SYS_TYPE or None if it's not defined.""" - return os.environ.get('SYS_TYPE') + return Architecture(spack.sys_type) +# This is livermore dependent. Hard coded for livermore +#def get_sys_type_from_environment(): +# """Return $SYS_TYPE or None if it's not defined.""" +# return os.environ.get('SYS_TYPE') def get_mac_sys_type(): - """Return a Mac OS SYS_TYPE or None if this isn't a mac.""" + """Return a Mac OS SYS_TYPE or None if this isn't a mac. + Front-end config + """ + mac_ver = py_platform.mac_ver()[0] if not mac_ver: return None - return "macosx_%s_%s" % ( - Version(mac_ver).up_to(2), py_platform.machine()) + return Architecture("macosx_%s_%s" % (Version(mac_ver).up_to(2), py_platform.machine())) +def get_sys_type_from_uname(): + """ Returns a sys_type from the uname argument + Front-end config + """ + return Architecture(os.uname()[0] + " " + os.uname()[-1]) + +def get_sys_type_from_config_file(): + """ Should read in a sys_type from the config yaml file. This should be the first thing looked at since + The user can specify that the architecture is a cray-xc40 + """ + + home_dir = os.environ["HOME"] + yaml_file = os.path.join(home_dir, ".spack/architecture.yaml") + if os.path.isfile(yaml_file): + with open(yaml_file) as config: + config_dict = config['architecture'] + front_end = config_dict['front'] + back_end = config_dict['back'] + return Architecture(front_end) @memoized def sys_type(): - """Returns a SysType for the current machine.""" + """Returns a SysType for the current machine. Should return output to an + Architecture class + """ methods = [get_sys_type_from_spack_globals, get_sys_type_from_environment, get_mac_sys_type] # search for a method that doesn't return None - sys_type = None + sys_type = (None,None) for method in methods: sys_type = method() if sys_type: break @@ -90,3 +130,4 @@ def sys_type(): raise InvalidSysTypeError(sys_type) return sys_type + From 29e03ac851f807e9fb5aefb687902e703d208615 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 15 Oct 2015 12:46:44 -0700 Subject: [PATCH 011/481] Added __eq__ testing method. Created tests for it in test/ folder --- lib/spack/spack/architecture.py | 56 +++++++++++++++++++++++---------- 1 file changed, 40 insertions(+), 16 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 829aaa4c1c..97c2cdbd6d 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -57,10 +57,20 @@ def add_arch_strategy(self): for n in self.arch_name: if 'cray' in n.lower(): self.arch_dict[n] = "MODULES" - if 'linux' in n.lower() or 'x86_64' in n.lower(): + elif 'linux' in n.lower() or 'x86_64' in n.lower(): self.arch_dict[n] = "PATH" else: self.arch_dict[n] = None + + def get_arch_dict(self): + """ Grab the dictionary from the Architecture class, rather than access the internal Architecture attributes """ + return self.arch_dict + + def __eq__(self, other): + if self.arch_dict != {} and other.arch_dict != {}: + return self.arch_dict == other.arch_dict + else: + return self.arch_name == self.arch_name def get_sys_type_from_spack_globals(): """Return the SYS_TYPE from spack globals, or None if it isn't set. Front-end""" @@ -95,29 +105,43 @@ def get_sys_type_from_uname(): def get_sys_type_from_config_file(): """ Should read in a sys_type from the config yaml file. This should be the first thing looked at since - The user can specify that the architecture is a cray-xc40 + The user can specify that the architecture is a cray-xc40. A template yaml should be created when spack + is installed. Similar to .spackconfig """ - home_dir = os.environ["HOME"] - yaml_file = os.path.join(home_dir, ".spack/architecture.yaml") - if os.path.isfile(yaml_file): - with open(yaml_file) as config: - config_dict = config['architecture'] - front_end = config_dict['front'] - back_end = config_dict['back'] - return Architecture(front_end) + spack_home_dir = os.environ["HOME"] + "/.spack" + yaml_file = os.path.join(spack_home_dir, "architecture.yaml") + + try: + config_dict = yaml.load(open(yaml_file)) # Fix this to have yaml.load() + arch = config_dict['architecture'] + front = arch['front'] + back = arch['back'] + + except: + print "No architecture.yaml config file found" + + return Architecture(front,back) @memoized def sys_type(): - """Returns a SysType for the current machine. Should return output to an - Architecture class + """Priority of gathering sys-type. + 1. YAML file that the user specifies the name of the architecture. e.g Cray-XC40 or Cray-XC30 + 2. UNAME + 3. GLOBALS + 4. MAC OSX + Yaml should be a priority here because we want the user to be able to specify the type of architecture to use. + If there is no yaml present then it should move on to the next function and stop immediately once it gets a + arch name + """ - methods = [get_sys_type_from_spack_globals, - get_sys_type_from_environment, - get_mac_sys_type] + methods = [get_sys_type_from_config_file, + get_sys_type_from_uname, + get_sys_type_from_spack_globals, + get_mac_sys_type] # search for a method that doesn't return None - sys_type = (None,None) + sys_type = None for method in methods: sys_type = method() if sys_type: break From d328f4c3b631e59ed14f52cf38f3d05774100c15 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 15 Oct 2015 12:47:26 -0700 Subject: [PATCH 012/481] Test suite for architecture class and functions --- lib/spack/spack/test/architecture.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 lib/spack/spack/test/architecture.py diff --git a/lib/spack/spack/test/architecture.py b/lib/spack/spack/test/architecture.py new file mode 100644 index 0000000000..6ff22aaa59 --- /dev/null +++ b/lib/spack/spack/test/architecture.py @@ -0,0 +1,19 @@ +""" Test checks if the architecture class is created correctly and also that + the functions are looking for the correct architecture name +""" +import unittest +import spack +from spack.architecture import * + +class ArchitectureTest(unittest.TestCase): + + def test_Architecture_class(self): + a = Architecture('Cray-XC40') + a.add_arch_strategy() + self.assertEquals(a.get_arch_dict(), {'Cray-XC40': 'MODULES'}) + + def test_get_sys_type_from_config_file(self): + output_arch_class = get_sys_type_from_config_file() + my_arch_class = Architecture('Linux x86_64','Cray-xc40') + + self.assertEqual(output_arch_class, my_arch_class) From ccdf1057592b48e13ba3b98df9972324e91d0be8 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 15 Oct 2015 12:48:12 -0700 Subject: [PATCH 013/481] Commented out a long list of tests to just include my arch test --- lib/spack/spack/test/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py index 6b3715be6f..ed51fac33a 100644 --- a/lib/spack/spack/test/__init__.py +++ b/lib/spack/spack/test/__init__.py @@ -31,7 +31,8 @@ import spack """Names of tests to be included in Spack's test suite""" -test_names = ['versions', +"""test_names = ['architecture', + 'versions', 'url_parse', 'url_substitution', 'packages', @@ -57,7 +58,8 @@ 'optional_deps', 'make_executable', 'configure_guess'] - +""" +test_names = ['architecture'] def list_tests(): """Return names of all tests that can be run for Spack.""" From fec197ccac94af485745b6b40d8150b4ae030e99 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 15 Oct 2015 15:25:13 -0700 Subject: [PATCH 014/481] Fixed the output of sys_type(), might need to add back the error handling part --- lib/spack/spack/architecture.py | 47 ++++++++++++++++----------------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 97c2cdbd6d..8f74fbc2e1 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -60,7 +60,7 @@ def add_arch_strategy(self): elif 'linux' in n.lower() or 'x86_64' in n.lower(): self.arch_dict[n] = "PATH" else: - self.arch_dict[n] = None + self.arch_dict[n] = "" def get_arch_dict(self): """ Grab the dictionary from the Architecture class, rather than access the internal Architecture attributes """ @@ -72,6 +72,7 @@ def __eq__(self, other): else: return self.arch_name == self.arch_name + def get_sys_type_from_spack_globals(): """Return the SYS_TYPE from spack globals, or None if it isn't set. Front-end""" if not hasattr(spack, "sys_type"): @@ -86,6 +87,7 @@ def get_sys_type_from_spack_globals(): # """Return $SYS_TYPE or None if it's not defined.""" # return os.environ.get('SYS_TYPE') + def get_mac_sys_type(): """Return a Mac OS SYS_TYPE or None if this isn't a mac. Front-end config @@ -97,12 +99,14 @@ def get_mac_sys_type(): return Architecture("macosx_%s_%s" % (Version(mac_ver).up_to(2), py_platform.machine())) + def get_sys_type_from_uname(): """ Returns a sys_type from the uname argument Front-end config """ return Architecture(os.uname()[0] + " " + os.uname()[-1]) + def get_sys_type_from_config_file(): """ Should read in a sys_type from the config yaml file. This should be the first thing looked at since The user can specify that the architecture is a cray-xc40. A template yaml should be created when spack @@ -110,21 +114,22 @@ def get_sys_type_from_config_file(): """ spack_home_dir = os.environ["HOME"] + "/.spack" - yaml_file = os.path.join(spack_home_dir, "architecture.yaml") + yaml_file = os.path.join(spack_home_dir, 'architecture.yaml') try: config_dict = yaml.load(open(yaml_file)) # Fix this to have yaml.load() arch = config_dict['architecture'] front = arch['front'] back = arch['back'] + return Architecture(front,back) except: print "No architecture.yaml config file found" - - return Architecture(front,back) + return None + @memoized -def sys_type(): +def sys_type(): # This function is going to give me issues isn't it?? """Priority of gathering sys-type. 1. YAML file that the user specifies the name of the architecture. e.g Cray-XC40 or Cray-XC30 2. UNAME @@ -133,25 +138,19 @@ def sys_type(): Yaml should be a priority here because we want the user to be able to specify the type of architecture to use. If there is no yaml present then it should move on to the next function and stop immediately once it gets a arch name - """ - methods = [get_sys_type_from_config_file, - get_sys_type_from_uname, - get_sys_type_from_spack_globals, - get_mac_sys_type] - - # search for a method that doesn't return None - sys_type = None - for method in methods: - sys_type = method() - if sys_type: break - - # Couldn't determine the sys_type for this machine. - if sys_type is None: - return "unknown_arch" - - if not isinstance(sys_type, basestring): - raise InvalidSysTypeError(sys_type) - + # Try to create an architecture object using the config file FIRST + functions = [get_sys_type_from_config_file, + get_sys_type_from_uname, + get_sys_type_from_spack_globals, + get_mac_sys_type] + + # TODO: Test for mac OSX system type but I'm sure it will be okay + for func in functions: + sys_type = None + sys_type = func() + if sys_type: + break + return sys_type From 3ba2842b53752e9a3ec8766f3a1350c6cc7577a2 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Tue, 20 Oct 2015 13:33:21 -0700 Subject: [PATCH 015/481] Got Architecture class working the way i wanted to. Next to write tests --- lib/spack/spack/architecture.py | 68 +++++++++++++++++---------------- 1 file changed, 36 insertions(+), 32 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 8f74fbc2e1..1a9f9de2cb 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -34,47 +34,50 @@ class InvalidSysTypeError(serr.SpackError): def __init__(self, sys_type): - super(InvalidSysTypeError, self).__init__( - "Invalid sys_type value for Spack: " + sys_type) + super(InvalidSysTypeError, self).__init__("Invalid sys_type value for Spack: " + sys_type) class NoSysTypeError(serr.SpackError): def __init__(self): - super(NoSysTypeError, self).__init__( - "Could not determine sys_type for this machine.") + super(NoSysTypeError, self).__init__("Could not determine sys_type for this machine.") + class Architecture(object): - def __init__(self, *arch_name): + """ Architecture class that contains a dictionary of architecture name and compiler search strategy methods. + The idea is to create an object that Spack can interact with and know how to search for the compiler + If it is on a Cray architecture it should look in modules. If it is anything else search $PATH. + """ + + def __init__(self, front=None, back=None): - """ Constructor for the architecture class. Should return a dictionary of name (grabbed from uname) and a strategy for - searching for that architecture's compiler. The target passed to it should be a dictionary of names and strategies. + """ Constructor for the architecture class. Should return a dictionary of name (grabbed from uname) + and a strategy for searching for that architecture's compiler. + The target passed to it should be a dictionary of names and strategies. """ - self.arch_dict = {} - self.arch_name = arch_name + names = [] + names.append(front) + names.append(back) - def add_arch_strategy(self): - """ Create a dictionary using the tuples of arch_names""" - for n in self.arch_name: - if 'cray' in n.lower(): - self.arch_dict[n] = "MODULES" - elif 'linux' in n.lower() or 'x86_64' in n.lower(): - self.arch_dict[n] = "PATH" - else: - self.arch_dict[n] = "" - - def get_arch_dict(self): - """ Grab the dictionary from the Architecture class, rather than access the internal Architecture attributes """ - return self.arch_dict - - def __eq__(self, other): - if self.arch_dict != {} and other.arch_dict != {}: - return self.arch_dict == other.arch_dict - else: - return self.arch_name == self.arch_name + def add_compiler_strategy(names): + """ Create a dictionary of {'arch-name': 'strategy'} + This will tell Spack whether to look in the $PATH + or $MODULES location for compilers + """ + d = {} + for n in names: + if n: + if 'cray' in n.lower(): + d[n] = "MODULES" + elif 'linux' in n.lower(): + d[n] = "PATH" + else: + d[n] = 'No Strategy' + return d + + self.arch_dict = add_compiler_strategy(names) - -def get_sys_type_from_spack_globals(): - """Return the SYS_TYPE from spack globals, or None if it isn't set. Front-end""" +def get_sys_type_from_spack_globals(): #TODO: Figure out how this function works + """Return the SYS_TYPE from spack globals, or None if it isn't set.""" if not hasattr(spack, "sys_type"): return None elif hasattr(spack.sys_type, "__call__"): @@ -104,7 +107,7 @@ def get_sys_type_from_uname(): """ Returns a sys_type from the uname argument Front-end config """ - return Architecture(os.uname()[0] + " " + os.uname()[-1]) + return Architecture(os.uname()[0]) def get_sys_type_from_config_file(): @@ -149,6 +152,7 @@ def sys_type(): # This function is going to give me issues isn't it?? for func in functions: sys_type = None sys_type = func() + if sys_type: break From ca3cc5b23e94b8168ba768ad343822b39aac4a25 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Tue, 20 Oct 2015 13:37:06 -0700 Subject: [PATCH 016/481] Refactored architecture class. Now it will automagically create a dict upon instantiation. --- lib/spack/spack/architecture.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 1a9f9de2cb..760c9cddd1 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -49,7 +49,6 @@ class Architecture(object): """ def __init__(self, front=None, back=None): - """ Constructor for the architecture class. Should return a dictionary of name (grabbed from uname) and a strategy for searching for that architecture's compiler. The target passed to it should be a dictionary of names and strategies. @@ -63,6 +62,7 @@ def add_compiler_strategy(names): This will tell Spack whether to look in the $PATH or $MODULES location for compilers """ + #TODO: Look for other strategies d = {} for n in names: if n: @@ -76,21 +76,20 @@ def add_compiler_strategy(names): self.arch_dict = add_compiler_strategy(names) -def get_sys_type_from_spack_globals(): #TODO: Figure out how this function works +def get_sys_type_from_spack_globals(): """Return the SYS_TYPE from spack globals, or None if it isn't set.""" if not hasattr(spack, "sys_type"): return None elif hasattr(spack.sys_type, "__call__"): - return Architecture(spack.sys_type()) + return Architecture(spack.sys_type()) #If in __init__.py there is a sys_type() then call that else: - return Architecture(spack.sys_type) + return Architecture(spack.sys_type) # Else use the attributed which defaults to None # This is livermore dependent. Hard coded for livermore #def get_sys_type_from_environment(): # """Return $SYS_TYPE or None if it's not defined.""" # return os.environ.get('SYS_TYPE') - def get_mac_sys_type(): """Return a Mac OS SYS_TYPE or None if this isn't a mac. Front-end config From c31da9bc8f10c3a74b3bf26c2dbac9a13c073eba Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Tue, 20 Oct 2015 14:10:54 -0700 Subject: [PATCH 017/481] Made sure architecture works with yaml file --- lib/spack/spack/architecture.py | 48 +++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 20 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 760c9cddd1..8460b0de1e 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -49,18 +49,24 @@ class Architecture(object): """ def __init__(self, front=None, back=None): - """ Constructor for the architecture class. Should return a dictionary of name (grabbed from uname) - and a strategy for searching for that architecture's compiler. - The target passed to it should be a dictionary of names and strategies. - """ - names = [] - names.append(front) - names.append(back) + """ Constructor for the architecture class. It will create a list from the given arguments and iterate + through that list. It will then create a dictionary of {arch_name : strategy} + Takes in two parameters: - def add_compiler_strategy(names): + front = None defaults to None. Should be the front-end architecture of the machine + back = None defaults to None. Should be the back-end architecture of the machine + + If no arguments are given it will return an empty dictionary + """ + _names = [] + _names.append(front) + _names.append(back) + + def _add_compiler_strategy(names): """ Create a dictionary of {'arch-name': 'strategy'} This will tell Spack whether to look in the $PATH or $MODULES location for compilers + Else it will return No Strategy """ #TODO: Look for other strategies d = {} @@ -74,7 +80,7 @@ def add_compiler_strategy(names): d[n] = 'No Strategy' return d - self.arch_dict = add_compiler_strategy(names) + self.arch_dict = _add_compiler_strategy(_names) def get_sys_type_from_spack_globals(): """Return the SYS_TYPE from spack globals, or None if it isn't set.""" @@ -85,20 +91,20 @@ def get_sys_type_from_spack_globals(): else: return Architecture(spack.sys_type) # Else use the attributed which defaults to None + # This is livermore dependent. Hard coded for livermore #def get_sys_type_from_environment(): # """Return $SYS_TYPE or None if it's not defined.""" # return os.environ.get('SYS_TYPE') + def get_mac_sys_type(): """Return a Mac OS SYS_TYPE or None if this isn't a mac. Front-end config """ - mac_ver = py_platform.mac_ver()[0] if not mac_ver: return None - return Architecture("macosx_%s_%s" % (Version(mac_ver).up_to(2), py_platform.machine())) @@ -113,11 +119,9 @@ def get_sys_type_from_config_file(): """ Should read in a sys_type from the config yaml file. This should be the first thing looked at since The user can specify that the architecture is a cray-xc40. A template yaml should be created when spack is installed. Similar to .spackconfig - """ - + """ spack_home_dir = os.environ["HOME"] + "/.spack" yaml_file = os.path.join(spack_home_dir, 'architecture.yaml') - try: config_dict = yaml.load(open(yaml_file)) # Fix this to have yaml.load() arch = config_dict['architecture'] @@ -131,7 +135,7 @@ def get_sys_type_from_config_file(): @memoized -def sys_type(): # This function is going to give me issues isn't it?? +def sys_type(): """Priority of gathering sys-type. 1. YAML file that the user specifies the name of the architecture. e.g Cray-XC40 or Cray-XC30 2. UNAME @@ -146,14 +150,18 @@ def sys_type(): # This function is going to give me issues isn't it?? get_sys_type_from_uname, get_sys_type_from_spack_globals, get_mac_sys_type] - - # TODO: Test for mac OSX system type but I'm sure it will be okay + + sys_type = None for func in functions: - sys_type = None sys_type = func() - if sys_type: - break + break + if sys_type is None: + return Architecture("unknown_arch") + + if not isinstance(sys_type, Architecture): + raise InvalidSysTypeError(sys_type) + return sys_type From 51b69ef00b947f4cf1801358b1a5da413d0b21bc Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 21 Oct 2015 09:27:57 -0700 Subject: [PATCH 018/481] initial fetch on crayport --- lib/spack/spack/compiler.py | 2 +- lib/spack/spack/directives.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index e7d450ee8b..4ce6e3f11d 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -119,7 +119,7 @@ def check(exe): self.fc = check(fc) self.spec = cspec - self.module = module + self.modules = modules.split() diff --git a/lib/spack/spack/directives.py b/lib/spack/spack/directives.py index 9297d6dac3..4bdc8772b8 100644 --- a/lib/spack/spack/directives.py +++ b/lib/spack/spack/directives.py @@ -252,7 +252,7 @@ def variant(pkg, name, default=False, description=""): """Define a variant for the package. Packager can specify a default value (on or off) as well as a text description.""" - default = bool(default) + default = default description = str(description).strip() if not re.match(spack.spec.identifier_re, name): From 9b387e7682c22b74661e3363442adcfb569d8680 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 21 Oct 2015 11:09:05 -0700 Subject: [PATCH 019/481] Added strategy method in init --- lib/spack/spack/architecture.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 8460b0de1e..3968b82124 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -57,18 +57,13 @@ def __init__(self, front=None, back=None): back = None defaults to None. Should be the back-end architecture of the machine If no arguments are given it will return an empty dictionary + Uses the _add_compiler_strategy(front, back) to create the dictionary """ - _names = [] - _names.append(front) - _names.append(back) - - def _add_compiler_strategy(names): - """ Create a dictionary of {'arch-name': 'strategy'} - This will tell Spack whether to look in the $PATH - or $MODULES location for compilers - Else it will return No Strategy - """ - #TODO: Look for other strategies + + def _add_compiler_strategy(front,back): + names = [] + names.append(front) + names.append(back) d = {} for n in names: if n: @@ -80,7 +75,8 @@ def _add_compiler_strategy(names): d[n] = 'No Strategy' return d - self.arch_dict = _add_compiler_strategy(_names) + self.arch_dict = _add_compiler_strategy(front, back) + def get_sys_type_from_spack_globals(): """Return the SYS_TYPE from spack globals, or None if it isn't set.""" From a89abb435f288c2864d9a21089b20f6eaac5d98b Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 21 Oct 2015 11:32:59 -0700 Subject: [PATCH 020/481] Changed structure of class, add compiler strategy is a method and can create a dict --- lib/spack/spack/architecture.py | 35 +++++++++++++++++---------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 3968b82124..7c1bdfb20f 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -48,6 +48,21 @@ class Architecture(object): If it is on a Cray architecture it should look in modules. If it is anything else search $PATH. """ + def add_compiler_strategy(self, front,back): + names = [] + names.append(front) + names.append(back) + d = {} + for n in names: + if n: + if 'cray' in n.lower(): + d[n] = "MODULES" + elif 'linux' in n.lower(): + d[n] = "PATH" + else: + d[n] = 'No Strategy' + return d + def __init__(self, front=None, back=None): """ Constructor for the architecture class. It will create a list from the given arguments and iterate through that list. It will then create a dictionary of {arch_name : strategy} @@ -59,23 +74,9 @@ def __init__(self, front=None, back=None): If no arguments are given it will return an empty dictionary Uses the _add_compiler_strategy(front, back) to create the dictionary """ - - def _add_compiler_strategy(front,back): - names = [] - names.append(front) - names.append(back) - d = {} - for n in names: - if n: - if 'cray' in n.lower(): - d[n] = "MODULES" - elif 'linux' in n.lower(): - d[n] = "PATH" - else: - d[n] = 'No Strategy' - return d - - self.arch_dict = _add_compiler_strategy(front, back) + self.front = front + self.back = back + self.arch_dict = self.add_compiler_strategy(front, back) def get_sys_type_from_spack_globals(): From 38508c5a3f294f730b31e209dfa90545e1920588 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 22 Oct 2015 12:02:26 -0700 Subject: [PATCH 021/481] Created a control flow logic that will loop through the strategies and find compilers using that. TODO: Need to find a way to locate their executables --- lib/spack/spack/cmd/compiler.py | 47 ++++++++++++++++++++------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/lib/spack/spack/cmd/compiler.py b/lib/spack/spack/cmd/compiler.py index 2a64dc914e..cbe3aba864 100644 --- a/lib/spack/spack/cmd/compiler.py +++ b/lib/spack/spack/cmd/compiler.py @@ -29,6 +29,8 @@ from llnl.util.tty.colify import colify from llnl.util.lang import index_by +import spack.architecture +import spack.compiler import spack.compilers import spack.spec import spack.config @@ -36,13 +38,12 @@ from spack.spec import CompilerSpec description = "Manage compilers" +ARCHITECTURE = spack.architecture.sys_type() def setup_parser(subparser): - sp = subparser.add_subparsers( - metavar='SUBCOMMAND', dest='compiler_command') + sp = subparser.add_subparsers(metavar='SUBCOMMAND', dest='compiler_command') - update_parser = sp.add_parser( - 'add', help='Add compilers to the Spack configuration.') + update_parser = sp.add_parser('add', help='Add compilers to the Spack configuration.') update_parser.add_argument('add_paths', nargs=argparse.REMAINDER) remove_parser = sp.add_parser('remove', help='remove compiler') @@ -55,23 +56,33 @@ def setup_parser(subparser): def compiler_add(args): - """Search either $PATH or a list of paths for compilers and add them + """Search either $PATH or a list of paths OR MODULES for compilers and add them to Spack's configuration.""" - paths = args.add_paths - if not paths: - paths = get_path('PATH') - compilers = [c for c in spack.compilers.find_compilers(*args.add_paths) - if c.spec not in spack.compilers.all_compilers()] + strategies = ARCHITECTURE.strategy() + + for strategy in strategies: + if strategy == 'PATH': + paths = args.add_paths # This might be a parser method. Parsing method to add_paths + if not paths: + paths = get_path('PATH') + + compilers = [c for c in spack.compilers.find_compilers(*args.add_paths) + if c.spec not in spack.compilers.all_compilers()] + + elif strategy == "MODULES": + from spack.compilers.cray import Cray + compilers = Cray.find_in_modules() + #TODO: Find a way to locate the executables - if compilers: - spack.compilers.add_compilers_to_config('user', *compilers) - n = len(compilers) - tty.msg("Added %d new compiler%s to %s" % ( - n, 's' if n > 1 else '', spack.config.get_config_scope_filename('user', 'compilers'))) - colify(reversed(sorted(c.spec for c in compilers)), indent=4) - else: - tty.msg("Found no new compilers") + if compilers: + spack.compilers.add_compilers_to_config('user', *compilers) + n = len(compilers) + tty.msg("Added %d new compiler%s to %s" % ( + n, 's' if n > 1 else '', spack.config.get_config_scope_filename('user', 'compilers'))) + colify(reversed(sorted(c.spec for c in compilers)), indent=4) + else: + tty.msg("Found no new compilers") def compiler_remove(args): From 09597fe8dccbd7d49acf1b3198f24ab928874cdb Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 30 Oct 2015 09:44:28 -0700 Subject: [PATCH 022/481] updated the executible to return stderr when specified. Added load_module to build_environment.py, loads target --- lib/spack/spack/build_environment.py | 36 ++++++++++++++++++++++------ lib/spack/spack/util/executable.py | 21 ++++++++++++---- 2 files changed, 45 insertions(+), 12 deletions(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 68388958f5..191e858735 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -86,6 +86,28 @@ def __call__(self, *args, **kwargs): return super(MakeExecutable, self).__call__(*args, **kwargs) +def load_module(mod): + """Takes a module name and removes modules until it is possible to + load that module. It then loads the provided module. Depends on the + modulecmd implementation of modules used in cray and lmod. + """ + #Create an executable of the module command that will output python code + modulecmd = which('modulecmd') + modulecmd.add_default_arg('python') + + # Read the module and remove any conflicting modules + # We do this without checking that they are already installed + # for ease of programming because unloading a module that is not + # loaded does nothing. + text = modulecmd('show', mod, return_oe=True).split() + for i, word in enumerate(text): + if word == 'conflict': + exec(compile(modulecmd('unload', text[i+1], return_oe=True), '', 'exec')) + # Load the module now that there are no conflicts + load = modulecmd('load', mod, return_oe=True) + exec(compile(load, '', 'exec')) + + def set_compiler_environment_variables(pkg): assert(pkg.spec.concrete) compiler = pkg.compiler @@ -108,11 +130,9 @@ def set_compiler_environment_variables(pkg): os.environ['SPACK_COMPILER_SPEC'] = str(pkg.spec.compiler) - if compiler.PrgEnv: - os.environ['SPACK_CRAYPE'] = compiler.PrgEnv - os.environ['SPACK_COMP_MODULE'] = compiler.module - - + if compiler.modules: + for mod in compiler.modules: + load_module(mod) def set_build_environment_variables(pkg): @@ -163,8 +183,10 @@ def set_build_environment_variables(pkg): pcdir = join_path(p, libdir, 'pkgconfig') if os.path.isdir(pcdir): pkg_config_dirs.append(pcdir) - path_set("PKG_CONFIG_PATH", pkg_config_dirs) + path_put_first("PKG_CONFIG_PATH", pkg_config_dirs) + if pkg.spec.architecture.compiler_strategy.lower() == 'module': + load_module(pkg.spec.architecture.module_name) def set_module_variables_for_package(pkg): """Populate the module scope of install() with some useful functions. @@ -239,8 +261,8 @@ def get_rpaths(pkg): def setup_package(pkg): """Execute all environment setup routines.""" - set_compiler_environment_variables(pkg) set_build_environment_variables(pkg) + set_compiler_environment_variables(pkg) set_module_variables_for_package(pkg) # Allow dependencies to set up environment as well. diff --git a/lib/spack/spack/util/executable.py b/lib/spack/spack/util/executable.py index d1dfb62ffb..15354089ac 100644 --- a/lib/spack/spack/util/executable.py +++ b/lib/spack/spack/util/executable.py @@ -56,7 +56,11 @@ def command(self): def __call__(self, *args, **kwargs): """Run the executable with subprocess.check_output, return output.""" - return_output = kwargs.get("return_output", False) + # Return oe returns a combined stream, setting both output and error + # without setting return oe returns them concatenated by a double line break + return_oe = kwargs.get("return_oe", False) + return_output = True if return_oe else kwargs.get("return_output", False) + return_error = True if return_oe else kwargs.get("return_error", False) fail_on_error = kwargs.get("fail_on_error", True) ignore_errors = kwargs.get("ignore_errors", ()) @@ -95,8 +99,8 @@ def streamify(arg, mode): proc = subprocess.Popen( cmd, stdin=input, - stderr=error, - stdout=subprocess.PIPE if return_output else output) + stdout=subprocess.PIPE if return_output else output, + stderr=subprocess.STDOUT if return_oe else (subprocess.PIPE if return_error else error)) out, err = proc.communicate() self.returncode = proc.returncode @@ -104,8 +108,15 @@ def streamify(arg, mode): if fail_on_error and rc != 0 and (rc not in ignore_errors): raise ProcessError("Command exited with status %d:" % proc.returncode, cmd_line) - if return_output: - return out + # Return out or error if specified. Return combined stream if requested, + # otherwise return them concatenated by double line break if both requested. + if return_output or return_error: + if return_oe or not return_error: + return out + elif return_output: + return out+'\n\n'+err + else: + return err except OSError, e: raise ProcessError( From 382d8478bf325099582b4814e0d71ff8522696aa Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 30 Oct 2015 10:23:33 -0700 Subject: [PATCH 023/481] prototype of new architecture concretization saved as new_concretize_architecture in concretize.py --- lib/spack/spack/concretize.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 66002492cb..c5041d67be 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -122,6 +122,34 @@ def concretize_architecture(self, spec): return True # changed + def new_concretize_architecture(self, spec): + """If the spec already has an architecture and it is a an architecture type, + return. Otherwise, if it has an architecture that is a string type, generate an + architecture based on that type. If it has no architecture and the root of the + DAG has an architecture, then use that. Otherwise, take the system's default + architecture. + """ + if spec.architecture is not None: + if isinstance(spec.architecture,spack.architecture.Target): + return False + else: + arch = spack.architecture.sys_type() + spec.architecture = arch.target(spec.architecture) + return True #changed + + if spec.root.architecture: + if isinstance(spec.root.architecture,spack.architecture.Target): + spec.architecture = spec.root.architecture + else: + arch = spack.architecture.sys_type() + spec.architecture = arch.target(spec.root.architecture) + else: + arch = spack.architecture.sys_type() + spec.architecture = arch.target('default') + + return True #changed + + def concretize_variants(self, spec): """If the spec already has variants filled in, return. Otherwise, add the default variants from the package specification. From 7ab921ff02a7d0a05287e4a49738ec9c51361ef0 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 30 Oct 2015 14:46:26 -0700 Subject: [PATCH 024/481] Changed architecture class and added class Target --- lib/spack/spack/architecture.py | 141 +++++++++++++++++++------------- 1 file changed, 85 insertions(+), 56 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 7c1bdfb20f..ea1f98e06e 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -23,11 +23,16 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os +import imp import platform as py_platform +import inspect -from llnl.util.lang import memoized +from llnl.util.lang import memoized, list_modules +from llnl.util.filesystem import join_path +import llnl.util.tty as tty import spack +from spack.util.naming import mod_to_class import spack.error as serr from spack.version import Version from external import yaml @@ -42,41 +47,55 @@ def __init__(self): super(NoSysTypeError, self).__init__("Could not determine sys_type for this machine.") -class Architecture(object): - """ Architecture class that contains a dictionary of architecture name and compiler search strategy methods. - The idea is to create an object that Spack can interact with and know how to search for the compiler - If it is on a Cray architecture it should look in modules. If it is anything else search $PATH. - """ - - def add_compiler_strategy(self, front,back): - names = [] - names.append(front) - names.append(back) - d = {} - for n in names: - if n: - if 'cray' in n.lower(): - d[n] = "MODULES" - elif 'linux' in n.lower(): - d[n] = "PATH" - else: - d[n] = 'No Strategy' - return d - - def __init__(self, front=None, back=None): - """ Constructor for the architecture class. It will create a list from the given arguments and iterate - through that list. It will then create a dictionary of {arch_name : strategy} - Takes in two parameters: +class Target(object): + """ This is the processor type e.g. cray-ivybridge """ + # Front end or back end target. Target needs to know which one this is + # Should autodetect from the machine + # features of a target + # - a module name + # -a compiler finding strategy + # -a name + # architecture classes handling the aliasing for front-end, back-end and default - front = None defaults to None. Should be the front-end architecture of the machine - back = None defaults to None. Should be the back-end architecture of the machine + def __init__(self,name, module_name=None): + self.name = name # case of cray "ivybridge but if it's x86_64 + self.module_name = module_name # craype-ivybridge + + def compiler_strategy(self): + if self.module_name: # If there is a module_name given then use MODULES + return "MODULES" + else: + return "PATH" + +class Architecture(object): + """ Abstract class that each type of Architecture will subclass. Will return a instance of it once it + is returned + """ + + priority = None # Subclass needs to set this number. This controls order in which arch is detected. + front = None + back = None + default_front = None # The default front end target. On cray sandybridge + default_back = None # The default back end target. On cray ivybridge + + def __init__(self, name): + self.targets = {} + self.name = name + + def add_target(self, name, target): + self.targets[name] = target - If no arguments are given it will return an empty dictionary - Uses the _add_compiler_strategy(front, back) to create the dictionary + + @classmethod + def detect(self): + """ Subclass is responsible for implementing this method. + Returns True if the architecture detects if it is the current architecture + and False if it's not. """ - self.front = front - self.back = back - self.arch_dict = self.add_compiler_strategy(front, back) + raise NotImplementedError() + + def __str__(self): + return self.name def get_sys_type_from_spack_globals(): @@ -84,9 +103,9 @@ def get_sys_type_from_spack_globals(): if not hasattr(spack, "sys_type"): return None elif hasattr(spack.sys_type, "__call__"): - return Architecture(spack.sys_type()) #If in __init__.py there is a sys_type() then call that + return spack.sys_type() #If in __init__.py there is a sys_type() then call that else: - return Architecture(spack.sys_type) # Else use the attributed which defaults to None + return spack.sys_type # Else use the attributed which defaults to None # This is livermore dependent. Hard coded for livermore @@ -102,15 +121,19 @@ def get_mac_sys_type(): mac_ver = py_platform.mac_ver()[0] if not mac_ver: return None - return Architecture("macosx_%s_%s" % (Version(mac_ver).up_to(2), py_platform.machine())) + return "macosx_%s_%s" % (Version(mac_ver).up_to(2), py_platform.machine()) def get_sys_type_from_uname(): """ Returns a sys_type from the uname argument Front-end config """ - return Architecture(os.uname()[0]) - + try: + arch_proc = subprocess.Popen(['uname', '-i'], stdout = subprocess.PIPE) + arch, _ = arch_proc.communicate() + return arch.strip() + except: + return None def get_sys_type_from_config_file(): """ Should read in a sys_type from the config yaml file. This should be the first thing looked at since @@ -131,34 +154,40 @@ def get_sys_type_from_config_file(): return None +@memoized +def all_architectures(): + modules = [] + for name in list_modules(spack.arch_path): + mod_name = 'spack.architectures.' + name + path = join_path(spack.arch_path, name) + ".py" + mod = imp.load_source(mod_name, path) + class_name = mod_to_class(name) + if not hasattr(mod, class_name): + tty.die('No class %s defined in %s' % (class_name, mod_name)) + cls = getattr(mod, class_name) + if not inspect.isclass(cls): + tty.die('%s.%s is not a class' % (mod_name, class_name)) + + modules.append(cls) + + return modules + @memoized def sys_type(): """Priority of gathering sys-type. 1. YAML file that the user specifies the name of the architecture. e.g Cray-XC40 or Cray-XC30 2. UNAME 3. GLOBALS - 4. MAC OSX + 4. MAC OSX Yaml should be a priority here because we want the user to be able to specify the type of architecture to use. If there is no yaml present then it should move on to the next function and stop immediately once it gets a arch name """ # Try to create an architecture object using the config file FIRST - functions = [get_sys_type_from_config_file, - get_sys_type_from_uname, - get_sys_type_from_spack_globals, - get_mac_sys_type] - - sys_type = None - for func in functions: - sys_type = func() - if sys_type: - break + architecture_list = all_architectures() + architecture_list.sort(key = lambda a: a.priority) - if sys_type is None: - return Architecture("unknown_arch") - - if not isinstance(sys_type, Architecture): - raise InvalidSysTypeError(sys_type) - - return sys_type + for arch in architecture_list: + if arch.detect(): + return arch() From d177184777591d21cc6a59771ed6f817960394f5 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 30 Oct 2015 15:12:29 -0700 Subject: [PATCH 025/481] added target method to architecture.py and minor cleanup --- lib/spack/spack/architecture.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index ea1f98e06e..f04fda2ee9 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -58,9 +58,10 @@ class Target(object): # architecture classes handling the aliasing for front-end, back-end and default def __init__(self,name, module_name=None): - self.name = name # case of cray "ivybridge but if it's x86_64 + self.name = name # case of cray "ivybridge" but if it's x86_64 self.module_name = module_name # craype-ivybridge + @property def compiler_strategy(self): if self.module_name: # If there is a module_name given then use MODULES return "MODULES" @@ -73,10 +74,9 @@ class Architecture(object): """ priority = None # Subclass needs to set this number. This controls order in which arch is detected. - front = None - back = None - default_front = None # The default front end target. On cray sandybridge - default_back = None # The default back end target. On cray ivybridge + front-end = None + back-end = None + default = None # The default back end target. On cray ivybridge def __init__(self, name): self.targets = {} @@ -85,7 +85,19 @@ def __init__(self, name): def add_target(self, name, target): self.targets[name] = target - + def target(self, name): + """This is a getter method for the target dictionary that handles defaulting based + on the values provided by default, front-end, and back-end. This can be overwritten + by a subclass for which we want to provide further aliasing options. + """ + if name == 'default': + name = default + elif name == 'front_end': + name = front-end + elif name == 'back_end': + name = back-end + return self.targets[name] + @classmethod def detect(self): """ Subclass is responsible for implementing this method. From 6e560703862794632176733a1d063550215dc573 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 30 Oct 2015 15:15:36 -0700 Subject: [PATCH 026/481] improved aliasing in target method in architecture.py --- lib/spack/spack/architecture.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index f04fda2ee9..3f38540c0e 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -92,9 +92,9 @@ def target(self, name): """ if name == 'default': name = default - elif name == 'front_end': + elif name == 'front_end' or name == 'fe': name = front-end - elif name == 'back_end': + elif name == 'back_end' or name == 'be': name = back-end return self.targets[name] From db0695e46a6c6fd4d2ac10dba04f7feb18c25a39 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 30 Oct 2015 15:57:00 -0700 Subject: [PATCH 027/481] architectures folder where the Architecture subclasses reside --- lib/spack/spack/architectures/__init__.py | 0 lib/spack/spack/architectures/cray.py | 17 +++++++++++++++++ 2 files changed, 17 insertions(+) create mode 100644 lib/spack/spack/architectures/__init__.py create mode 100644 lib/spack/spack/architectures/cray.py diff --git a/lib/spack/spack/architectures/__init__.py b/lib/spack/spack/architectures/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/lib/spack/spack/architectures/cray.py b/lib/spack/spack/architectures/cray.py new file mode 100644 index 0000000000..e6d6d84922 --- /dev/null +++ b/lib/spack/spack/architectures/cray.py @@ -0,0 +1,17 @@ +import os + +from spack.architecture import Architecture + +class Cray(Architecture): + priority = 20 + front_end = None + back_end = None + default = None + + def __init__(self): + super(Cray, self).__init__('cray') + + @classmethod + def detect(self): + return os.path.exists('/opt/cray/craype') + From 2d87bb92edb95cf1c9ec7d58f5d8a34e44bd11a4 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 30 Oct 2015 15:58:20 -0700 Subject: [PATCH 028/481] Changed front-end and back-end to front_end and back_end to avoid error --- lib/spack/spack/architecture.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 3f38540c0e..f488b65cc6 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -74,8 +74,8 @@ class Architecture(object): """ priority = None # Subclass needs to set this number. This controls order in which arch is detected. - front-end = None - back-end = None + front_end = None + back_end = None default = None # The default back end target. On cray ivybridge def __init__(self, name): From e39586c81da152e8d02fd26bf48c66e00e26519a Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 30 Oct 2015 15:58:49 -0700 Subject: [PATCH 029/481] Reverted back all architecture way of finding modules --- lib/spack/spack/cmd/compiler.py | 38 +++++++++++++-------------------- 1 file changed, 15 insertions(+), 23 deletions(-) diff --git a/lib/spack/spack/cmd/compiler.py b/lib/spack/spack/cmd/compiler.py index cbe3aba864..3e86928977 100644 --- a/lib/spack/spack/cmd/compiler.py +++ b/lib/spack/spack/cmd/compiler.py @@ -38,7 +38,6 @@ from spack.spec import CompilerSpec description = "Manage compilers" -ARCHITECTURE = spack.architecture.sys_type() def setup_parser(subparser): sp = subparser.add_subparsers(metavar='SUBCOMMAND', dest='compiler_command') @@ -59,30 +58,23 @@ def compiler_add(args): """Search either $PATH or a list of paths OR MODULES for compilers and add them to Spack's configuration.""" - strategies = ARCHITECTURE.strategy() - for strategy in strategies: - if strategy == 'PATH': - paths = args.add_paths # This might be a parser method. Parsing method to add_paths - if not paths: - paths = get_path('PATH') - - compilers = [c for c in spack.compilers.find_compilers(*args.add_paths) - if c.spec not in spack.compilers.all_compilers()] - - elif strategy == "MODULES": - from spack.compilers.cray import Cray - compilers = Cray.find_in_modules() - #TODO: Find a way to locate the executables + paths = args.add_paths # This might be a parser method. Parsing method to add_paths + if not paths: + paths = get_path('PATH') + + compilers = [c for c in spack.compilers.find_compilers(*args.add_paths) + if c.spec not in spack.compilers.all_compilers()] - if compilers: - spack.compilers.add_compilers_to_config('user', *compilers) - n = len(compilers) - tty.msg("Added %d new compiler%s to %s" % ( - n, 's' if n > 1 else '', spack.config.get_config_scope_filename('user', 'compilers'))) - colify(reversed(sorted(c.spec for c in compilers)), indent=4) - else: - tty.msg("Found no new compilers") + + if compilers: + spack.compilers.add_compilers_to_config('user', *compilers) + n = len(compilers) + tty.msg("Added %d new compiler%s to %s" % ( + n, 's' if n > 1 else '', spack.config.get_config_scope_filename('user', 'compilers'))) + colify(reversed(sorted(c.spec for c in compilers)), indent=4) + else: + tty.msg("Found no new compilers") def compiler_remove(args): From 9a91da9ccd8044e147a416fcfa9b16405ef22b3d Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 30 Oct 2015 15:59:28 -0700 Subject: [PATCH 030/481] Added arch_path to list of modules path --- lib/spack/spack/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index caa09eb6e0..0f23f61614 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -37,6 +37,7 @@ lib_path = join_path(prefix, "lib", "spack") build_env_path = join_path(lib_path, "env") module_path = join_path(lib_path, "spack") +arch_path = join_path(module_path, 'architectures') compilers_path = join_path(module_path, "compilers") test_path = join_path(module_path, "test") hooks_path = join_path(module_path, "hooks") From eb2cf1698fa8cd606a8b80bcb39de2ee0b724c69 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 2 Nov 2015 11:08:08 -0800 Subject: [PATCH 031/481] Fixed some issues with naming --- lib/spack/spack/architecture.py | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index f488b65cc6..442180242b 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -49,13 +49,6 @@ def __init__(self): class Target(object): """ This is the processor type e.g. cray-ivybridge """ - # Front end or back end target. Target needs to know which one this is - # Should autodetect from the machine - # features of a target - # - a module name - # -a compiler finding strategy - # -a name - # architecture classes handling the aliasing for front-end, back-end and default def __init__(self,name, module_name=None): self.name = name # case of cray "ivybridge" but if it's x86_64 @@ -67,7 +60,8 @@ def compiler_strategy(self): return "MODULES" else: return "PATH" - + + class Architecture(object): """ Abstract class that each type of Architecture will subclass. Will return a instance of it once it is returned @@ -85,17 +79,19 @@ def __init__(self, name): def add_target(self, name, target): self.targets[name] = target + def target(self, name): """This is a getter method for the target dictionary that handles defaulting based on the values provided by default, front-end, and back-end. This can be overwritten by a subclass for which we want to provide further aliasing options. """ if name == 'default': - name = default + name = self.default elif name == 'front_end' or name == 'fe': - name = front-end + name = self.front_end elif name == 'back_end' or name == 'be': - name = back-end + name = self.back_end + return self.targets[name] @classmethod @@ -148,10 +144,7 @@ def get_sys_type_from_uname(): return None def get_sys_type_from_config_file(): - """ Should read in a sys_type from the config yaml file. This should be the first thing looked at since - The user can specify that the architecture is a cray-xc40. A template yaml should be created when spack - is installed. Similar to .spackconfig - """ + spack_home_dir = os.environ["HOME"] + "/.spack" yaml_file = os.path.join(spack_home_dir, 'architecture.yaml') try: From 058e72d29c3cd934f91ba626392b25ebaa50e2cc Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 2 Nov 2015 11:08:55 -0800 Subject: [PATCH 032/481] Added default target and also front end and back end targets --- lib/spack/spack/architectures/cray.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/architectures/cray.py b/lib/spack/spack/architectures/cray.py index e6d6d84922..a79c916684 100644 --- a/lib/spack/spack/architectures/cray.py +++ b/lib/spack/spack/architectures/cray.py @@ -4,9 +4,9 @@ class Cray(Architecture): priority = 20 - front_end = None - back_end = None - default = None + front_end = 'sandybridge' + back_end = 'ivybridge' + default = os.environ["CRAY_CPU_TARGET"] def __init__(self): super(Cray, self).__init__('cray') From 5ac974c9b2072631eab490cce8f2922420eef9e4 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 4 Nov 2015 12:50:22 -0800 Subject: [PATCH 033/481] Enforced that the architecture subclass cannot add a target that shares a name with a target alias --- lib/spack/spack/architecture.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 442180242b..890df9b1e5 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -77,9 +77,14 @@ def __init__(self, name): self.name = name def add_target(self, name, target): - self.targets[name] = target - - + """Used by the architecture specific subclass to list available targets. Raises an error + if the architecture specifies a name that is reserved by spack as an alias. + """ + if name in ['front_end', 'fe', 'back_end', 'be', 'default']: + raise ValueError("%s is a spack reserved alias and cannot be the name of a target" % name) + self.targets[name] = target + + def target(self, name): """This is a getter method for the target dictionary that handles defaulting based on the values provided by default, front-end, and back-end. This can be overwritten From 37260962e545366d0d882b205be807562c36f3b7 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 4 Nov 2015 12:57:29 -0800 Subject: [PATCH 034/481] changed some potential syntax errors and added a way for target to recognize class --- lib/spack/spack/architecture.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 442180242b..b251b82dcc 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -48,20 +48,29 @@ def __init__(self): class Target(object): - """ This is the processor type e.g. cray-ivybridge """ + """ Target is the processor of the host machine. The host machine may have different front-end + and back-end targets, especially if it is a Cray machine. The target will have a name and + also the module_name (e.g craype-compiler). Targets will also recognize which architecture + they came from using the set_architecture method. Targets will have compiler finding strategies + """ + default_strategy = None # Can probably add a compiler path here def __init__(self,name, module_name=None): self.name = name # case of cray "ivybridge" but if it's x86_64 self.module_name = module_name # craype-ivybridge + def set_architecture(self, architecture): # Target should get the architecture class. + self.architecture = architecture + @property def compiler_strategy(self): - if self.module_name: # If there is a module_name given then use MODULES + if default_strategy: + return default_strategy + elif self.module_name: # If there is a module_name given then use MODULES return "MODULES" else: return "PATH" - class Architecture(object): """ Abstract class that each type of Architecture will subclass. Will return a instance of it once it is returned @@ -77,8 +86,8 @@ def __init__(self, name): self.name = name def add_target(self, name, target): - self.targets[name] = target - + target.set_architecture(self) + self.targets[name] = target def target(self, name): """This is a getter method for the target dictionary that handles defaulting based @@ -163,7 +172,7 @@ def get_sys_type_from_config_file(): def all_architectures(): modules = [] for name in list_modules(spack.arch_path): - mod_name = 'spack.architectures.' + name + mod_name = 'spack.architectures' + name path = join_path(spack.arch_path, name) + ".py" mod = imp.load_source(mod_name, path) class_name = mod_to_class(name) From 35532d6b0aaa428e1d8234513e9e887d08c3914c Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 4 Nov 2015 13:00:35 -0800 Subject: [PATCH 035/481] Changed cray architecture subclass to add proper targets for front at back end nodes --- lib/spack/spack/architectures/cray.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/spack/spack/architectures/cray.py b/lib/spack/spack/architectures/cray.py index a79c916684..420b7c589e 100644 --- a/lib/spack/spack/architectures/cray.py +++ b/lib/spack/spack/architectures/cray.py @@ -10,6 +10,11 @@ class Cray(Architecture): def __init__(self): super(Cray, self).__init__('cray') + # Back End compiler needs the proper target module loaded. + self.add_target('ivybridge','craype-ivybridge') + # Could switch to use modules and fe targets for front end + # Currently using compilers by path for front end. + self.add_target('sandybridge') @classmethod def detect(self): From 9bf8e8573cff42581074ed9e492ab0e0bc6c7253 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 4 Nov 2015 13:08:48 -0800 Subject: [PATCH 036/481] generic linux architecture subclass --- lib/spack/spack/architectures/linux.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 lib/spack/spack/architectures/linux.py diff --git a/lib/spack/spack/architectures/linux.py b/lib/spack/spack/architectures/linux.py new file mode 100644 index 0000000000..7238575660 --- /dev/null +++ b/lib/spack/spack/architectures/linux.py @@ -0,0 +1,17 @@ +import subprocess +from spack.architecture import Architecture + +class Linux(Architecture): + priority = 60 + front_end = "x86_64" + back_end = "x86_64" + default = "x86_64" + + def __init__(self): + super(Linux, self).__init__('linux') + + @classmethod + def detect(self): + arch = subprocess.Popen(['uname', '-i'], stdout = subprocess.PIPE) + arch, _ = arch.communicate() + return 'x86_64' in arch.strip() From 3a73ae1683c4e83791f273d2034a99824e81aeca Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 4 Nov 2015 13:12:11 -0800 Subject: [PATCH 037/481] Fixed the previous commit --- lib/spack/spack/architectures/cray.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/architectures/cray.py b/lib/spack/spack/architectures/cray.py index 420b7c589e..640a3933e0 100644 --- a/lib/spack/spack/architectures/cray.py +++ b/lib/spack/spack/architectures/cray.py @@ -1,20 +1,20 @@ import os -from spack.architecture import Architecture +from spack.architecture import Architecture, Target class Cray(Architecture): priority = 20 front_end = 'sandybridge' back_end = 'ivybridge' - default = os.environ["CRAY_CPU_TARGET"] + default = 'ivybridge' def __init__(self): super(Cray, self).__init__('cray') # Back End compiler needs the proper target module loaded. - self.add_target('ivybridge','craype-ivybridge') + self.add_target('ivybridge', Target('ivybridge','craype-ivybridge')) # Could switch to use modules and fe targets for front end # Currently using compilers by path for front end. - self.add_target('sandybridge') + self.add_target('sandybridge', Target('sandybridge')) @classmethod def detect(self): From b61d554dc818c2514384b017c74dfa61c04a8c3a Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 4 Nov 2015 13:28:12 -0800 Subject: [PATCH 038/481] Fixed architecture.py file so it doesn't have the weird merge changes i.e <<>>>>>> 8b3f2ec1d117e1a8b206927f51db8684396c231b def target(self, name): """This is a getter method for the target dictionary that handles defaulting based on the values provided by default, front-end, and back-end. This can be overwritten From 08729315c617b42e6fd1eaa1af4919aa52f96a0f Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Thu, 5 Nov 2015 09:21:56 -0800 Subject: [PATCH 039/481] Added bgq architecture subclass: may require additional functionality to support --- lib/spack/spack/architectures/bgq.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 lib/spack/spack/architectures/bgq.py diff --git a/lib/spack/spack/architectures/bgq.py b/lib/spack/spack/architectures/bgq.py new file mode 100644 index 0000000000..d3d4446e09 --- /dev/null +++ b/lib/spack/spack/architectures/bgq.py @@ -0,0 +1,19 @@ +import os + +from spack.architecture import Architecture, Target + +class Bgq(Architecture): + priority = 30 + front_end = 'power7' + back_end = 'powerpc' + default = 'powerpc' + + def __init__(self): + super(Bgq, self).__init__('cray') + self.add_target('power7', Target('power7')) + self.add_target('powerpc', Target('powerpc')) + + @classmethod + def detect(self): + return os.path.exists('/bgsys') + From 0fcb368f478621b3eb8faf2b812018d43c46ffef Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 5 Nov 2015 13:42:50 -0800 Subject: [PATCH 040/481] Adding adios package to spack's package list --- var/spack/packages/adios/package.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 var/spack/packages/adios/package.py diff --git a/var/spack/packages/adios/package.py b/var/spack/packages/adios/package.py new file mode 100644 index 0000000000..5f298fc6b5 --- /dev/null +++ b/var/spack/packages/adios/package.py @@ -0,0 +1,28 @@ +from spack import * + +class Adios(Package): + """The Adaptable IO System (ADIOS) provides a simple, + flexible way for scientists to describe the data in their code that may need to be written, + read, or processed outside of the running simulation + """ + + homepage = "http://www.olcf.ornl.gov/center-projects/adios/" + url = "http://users.nccs.gov/~pnorbert/adios-1.9.0.tar.gz" + + version('1.9.0', 'dbf5cb10e32add2f04c9b4052b7ffa76') + + # Lots of setting up here for this package + # module swap PrgEnv-intel PrgEnv-$COMP + # module load cray-netcdf/4.3.3.1 + # module load cray-hdf5/1.8.14 + # module load python/2.7.10 + depends_on('mxml') + + def install(self, spec, prefix): + configure_args = ["--prefix=%s" % prefix, "--with-mxml=%s" % spec['mxml'].prefix, + "--with-hdf5=%s" % spec['hdf5'].prefix, "--with-netcdf=%s" % os.environ['NETCDF_DIR'], + "--with-infiniband=no"] + + configure(*configure_args) + make() + make("install") From e0498ce54ad15787db3c4f187902b424c33013d7 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 5 Nov 2015 13:43:11 -0800 Subject: [PATCH 041/481] mxml is adios's package dependency --- var/spack/packages/mxml/package.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 var/spack/packages/mxml/package.py diff --git a/var/spack/packages/mxml/package.py b/var/spack/packages/mxml/package.py new file mode 100644 index 0000000000..fc32b56323 --- /dev/null +++ b/var/spack/packages/mxml/package.py @@ -0,0 +1,24 @@ +from spack import * + +class Mxml(Package): + """Mini-XML is a small XML library that you can use to read and write XML + and XML-like data files in your application without requiring large + non-standard libraries""" + + homepage = "http://www.msweet.org" + url = "http://www.msweet.org/files/project3/mxml-2.9.tar.gz" + + version('2.9', 'e21cad0f7aacd18f942aa0568a8dee19') + version('2.8', 'd85ee6d30de053581242c4a86e79a5d2') + version('2.7', '76f2ae49bf0f5745d5cb5d9507774dc9') + version('2.6', '68977789ae64985dddbd1a1a1652642e') + version('2.5', 'f706377fba630b39fa02fd63642b17e5') + + # module swap PrgEnv-intel PrgEnv-$COMP (Can use whatever compiler you want to use) + # Case statement to change CC and CXX flags + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix, "--disable-shared") + make() + make("install") + From 5347f460b4f2ce8ad64f6ec225779c2a2f944ad4 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 11 Nov 2015 10:31:37 -0800 Subject: [PATCH 042/481] adding module find to find command --- lib/spack/spack/compiler.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index 4ce6e3f11d..e4bfeea608 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -106,7 +106,7 @@ class Compiler(object): PrgEnv_compiler = None - def __init__(self, cspec, cc, cxx, f77, fc, module=None): + def __init__(self, cspec, cc, cxx, f77, fc, modules=None): def check(exe): if exe is None: return None @@ -119,7 +119,7 @@ def check(exe): self.fc = check(fc) self.spec = cspec - self.modules = modules.split() + self.modules = modules @@ -216,6 +216,10 @@ def check(key): @classmethod def find(cls, *path): + return cls.find_in_path(*path) + cls.find_in_modules() + + @classmethod + def find_in_path(cls, *path): """Try to find this type of compiler in the user's environment. For each set of compilers found, this returns compiler objects with the cc, cxx, f77, fc paths and the @@ -273,7 +277,9 @@ def find_in_modules(cls): if not cls.PrgEnv_compiler: tty.die('Must supply PrgEnv_compiler with PrgEnv') - output = _shell('module avail %s' % cls.PrgEnv_compiler) +# output = _shell('module avail %s' % cls.PrgEnv_compiler) + modulecmd = which('modulecmd') + modulecmd matches = re.findall(r'(%s)/([^\s(]*)' % cls.PrgEnv_compiler, output) for name, version in matches: From 95a34628a3bebe3d263afbd2707cb5d47a2b7a33 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 11 Nov 2015 11:32:18 -0800 Subject: [PATCH 043/481] Add modules to compilers. Changed compiler to take paths as a list. Changed compiler_for_spec to be aware of different compiler stratigies --- lib/spack/spack/compiler.py | 61 +++++++-------------------- lib/spack/spack/compilers/__init__.py | 13 +++++- lib/spack/spack/package.py | 2 +- 3 files changed, 27 insertions(+), 49 deletions(-) diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index 67765dfb7b..4be573fb12 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -106,17 +106,20 @@ class Compiler(object): PrgEnv_compiler = None - def __init__(self, cspec, cc, cxx, f77, fc, modules=None): + def __init__(self, cspec, paths, modules=None): def check(exe): if exe is None: return None _verify_executables(exe) return exe - self.cc = check(cc) - self.cxx = check(cxx) - self.f77 = check(f77) - self.fc = check(fc) + self.cc = check(paths[0]) + self.cxx = check(paths[1]) + self.f77 = check(paths[2]) + if len(paths) == 3: + self.fc = self.f77 + else: + self.fc = check(paths[3]) self.spec = cspec self.modules = modules @@ -264,7 +267,7 @@ def find_in_path(cls, *path): if newcount <= prevcount: continue - compilers[ver] = cls(spec, *paths) + compilers[ver] = cls(spec, paths) return list(compilers.values()) @@ -277,60 +280,26 @@ def find_in_modules(cls): if not cls.PrgEnv_compiler: tty.die('Must supply PrgEnv_compiler with PrgEnv') -# output = _shell('module avail %s' % cls.PrgEnv_compiler) modulecmd = which('modulecmd') - modulecmd + modulecmd.add_default_arg('python') + output = modulecmd('avail', return_oe=True) matches = re.findall(r'(%s)/([^\s(]*)' % cls.PrgEnv_compiler, output) - loaded_modules = os.environ["LOADEDMODULES"].split(":") +# loaded_modules = os.environ["LOADEDMODULES"].split(":") #output = _shell('module avail %s' % cls.PrgEnv_compiler) - for module in loaded_modules: - match = re.findall(r'(%s)/([^\s(]*)' % cls.PrgEnv_compiler, module) +# for module in loaded_modules: +# match = re.findall(r'(%s)/([^\s(]*)' % cls.PrgEnv_compiler, module) for name, version in matches: v = version + '-craype' comp = cls(spack.spec.CompilerSpec(name + '@' + v), - 'cc', 'CC', 'ftn', 'ftn', name +'/' + v) + ['cc', 'CC', 'ftn'], [cls.PrgEnv, name +'/' + v]) compilers.append(comp) return compilers -def _cur_prgenv(): - out, err = subprocess.Popen( - ['module list'], shell=True, - stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() - matches = re.findall(r'(PrgEnv-[^/]*)/', err) - return matches[0] - - -def _module_shell(module, *args): - cmd = 'module swap %s %s;' % (_cur_prgenv(), module) - cmd += 'module load %s;' % compiler - cmd += 'module unload cray-libsci;' - -# + -# 'module load craype-network-gemini;' + -# 'module load %s;' % module + -# 'module swap gcc/4.6.1;' + -# 'module load eswrap; ' + -# 'module load craype-mc12; ' + -# 'module load cray-shmem; ' + -# 'module load cray-mpich; ') - cmd += ' '.join(args) - out, err = subprocess.Popen([cmd + ' '.join(args)], shell=True, - stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() - return out - - -def _shell(*args): - return subprocess.Popen([' '.join(args)], shell=True, - stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[1] - - - - def __repr__(self): """Return a string represntation of the compiler toolchain.""" return self.__str__() diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 8c4cb38926..6fc54daa61 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -197,18 +197,27 @@ def get_compiler(cspec): else: compiler_paths.append(None) - return cls(cspec, *compiler_paths) + for m in _optional_instance_vars: + if m not in items: + items[m] = None + mods = items[m] + + return cls(cspec, compiler_paths, mods) matches = find(compiler_spec) return [get_compiler(cspec) for cspec in matches] @_auto_compiler_spec -def compiler_for_spec(compiler_spec): +def compiler_for_spec(compiler_spec, target): """Get the compiler that satisfies compiler_spec. compiler_spec must be concrete.""" assert(compiler_spec.concrete) compilers = compilers_for_spec(compiler_spec) + if target.compiler_strategy == "PATH": + filter(lambda c: c.modules is None, compilers) + elif target.compiler_strategy == "MODULES": + filter(lambda c: c.modules is not None, compilers) assert(len(compilers) == 1) return compilers[0] diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 61606d0590..b6a98f3ee6 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -592,7 +592,7 @@ def compiler(self): """Get the spack.compiler.Compiler object used to build this package.""" if not self.spec.concrete: raise ValueError("Can only get a compiler for a concrete package.") - return spack.compilers.compiler_for_spec(self.spec.compiler) + return spack.compilers.compiler_for_spec(self.spec.compiler, self.spec.architecture) def url_version(self, version): From 271a839957ace370ce332f8edd3321e60127a9e6 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 11 Nov 2015 14:22:07 -0800 Subject: [PATCH 044/481] First possibly working version of the crayport. Not sufficiently tested at all. --- lib/spack/spack/architecture.py | 41 +++++++++++++++----------- lib/spack/spack/architectures/linux.py | 13 ++++---- lib/spack/spack/build_environment.py | 6 ++-- lib/spack/spack/compiler.py | 8 +++-- lib/spack/spack/compilers/__init__.py | 6 ++-- lib/spack/spack/concretize.py | 28 +----------------- lib/spack/spack/config.py | 5 ++-- 7 files changed, 46 insertions(+), 61 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 64416e8b4c..b1da8ae4a2 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -61,18 +61,20 @@ def __init__(self,name, module_name=None): def set_architecture(self, architecture): # Target should get the architecture class. self.architecture = architecture - + @property def compiler_strategy(self): - if default_strategy: - return default_strategy - elif self.module_name: # If there is a module_name given then use MODULES + if self.module_name: # If there is a module_name given then use MODULES return "MODULES" else: return "PATH" + def __str__(self): + return self.name + + class Architecture(object): - """ Abstract class that each type of Architecture will subclass. Will return a instance of it once it + """ Abstract class that each type of Architecture will subclass. Will return a instance of it once it is returned """ @@ -110,12 +112,15 @@ def target(self, name): @classmethod def detect(self): - """ Subclass is responsible for implementing this method. + """ Subclass is responsible for implementing this method. Returns True if the architecture detects if it is the current architecture and False if it's not. """ raise NotImplementedError() - + + def __repr__(self): + return self.__str__ + def __str__(self): return self.name @@ -123,7 +128,7 @@ def __str__(self): def get_sys_type_from_spack_globals(): """Return the SYS_TYPE from spack globals, or None if it isn't set.""" if not hasattr(spack, "sys_type"): - return None + return None elif hasattr(spack.sys_type, "__call__"): return spack.sys_type() #If in __init__.py there is a sys_type() then call that else: @@ -147,7 +152,7 @@ def get_mac_sys_type(): def get_sys_type_from_uname(): - """ Returns a sys_type from the uname argument + """ Returns a sys_type from the uname argument Front-end config """ try: @@ -158,8 +163,8 @@ def get_sys_type_from_uname(): return None def get_sys_type_from_config_file(): - - spack_home_dir = os.environ["HOME"] + "/.spack" + + spack_home_dir = os.environ["HOME"] + "/.spack" yaml_file = os.path.join(spack_home_dir, 'architecture.yaml') try: config_dict = yaml.load(open(yaml_file)) # Fix this to have yaml.load() @@ -167,7 +172,7 @@ def get_sys_type_from_config_file(): front = arch['front'] back = arch['back'] return Architecture(front,back) - + except: print "No architecture.yaml config file found" return None @@ -182,7 +187,7 @@ def all_architectures(): mod = imp.load_source(mod_name, path) class_name = mod_to_class(name) if not hasattr(mod, class_name): - tty.die('No class %s defined in %s' % (class_name, mod_name)) + tty.die('No class %s defined in %s' % (class_name, mod_name)) cls = getattr(mod, class_name) if not inspect.isclass(cls): tty.die('%s.%s is not a class' % (mod_name, class_name)) @@ -197,15 +202,15 @@ def sys_type(): 1. YAML file that the user specifies the name of the architecture. e.g Cray-XC40 or Cray-XC30 2. UNAME 3. GLOBALS - 4. MAC OSX + 4. MAC OSX Yaml should be a priority here because we want the user to be able to specify the type of architecture to use. - If there is no yaml present then it should move on to the next function and stop immediately once it gets a + If there is no yaml present then it should move on to the next function and stop immediately once it gets a arch name """ # Try to create an architecture object using the config file FIRST - architecture_list = all_architectures() - architecture_list.sort(key = lambda a: a.priority) - + architecture_list = all_architectures() + architecture_list.sort(key = lambda a: a.priority) + for arch in architecture_list: if arch.detect(): return arch() diff --git a/lib/spack/spack/architectures/linux.py b/lib/spack/spack/architectures/linux.py index 7238575660..d7d9a994fa 100644 --- a/lib/spack/spack/architectures/linux.py +++ b/lib/spack/spack/architectures/linux.py @@ -1,14 +1,15 @@ -import subprocess -from spack.architecture import Architecture +import subprocess +from spack.architecture import Architecture, Target class Linux(Architecture): - priority = 60 - front_end = "x86_64" - back_end = "x86_64" - default = "x86_64" + priority = 60 + front_end = 'linux' + back_end = 'linux' + default = 'linux' def __init__(self): super(Linux, self).__init__('linux') + self.add_target('linux', Target('linux')) @classmethod def detect(self): diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 191e858735..9e3be433fb 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -88,13 +88,13 @@ def __call__(self, *args, **kwargs): def load_module(mod): """Takes a module name and removes modules until it is possible to - load that module. It then loads the provided module. Depends on the + load that module. It then loads the provided module. Depends on the modulecmd implementation of modules used in cray and lmod. """ #Create an executable of the module command that will output python code modulecmd = which('modulecmd') modulecmd.add_default_arg('python') - + # Read the module and remove any conflicting modules # We do this without checking that they are already installed # for ease of programming because unloading a module that is not @@ -102,7 +102,7 @@ def load_module(mod): text = modulecmd('show', mod, return_oe=True).split() for i, word in enumerate(text): if word == 'conflict': - exec(compile(modulecmd('unload', text[i+1], return_oe=True), '', 'exec')) + exec(compile(modulecmd('unload', text[i+1], return_oe=True), '', 'exec')) # Load the module now that there are no conflicts load = modulecmd('load', mod, return_oe=True) exec(compile(load, '', 'exec')) diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index 4be573fb12..41ff89a151 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -307,8 +307,12 @@ def __repr__(self): def __str__(self): """Return a string represntation of the compiler toolchain.""" - return "%s(%s)" % ( - self.name, '\n '.join((str(s) for s in (self.cc, self.cxx, self.f77, self.fc)))) + if self.modules: + return "%s(%s)" % ( + self.name, '\n '.join((str(s) for s in (self.cc, self.cxx, self.f77, self.fc, self.modules)))) + else: + return "%s(%s)" % ( + self.name, '\n '.join((str(s) for s in (self.cc, self.cxx, self.f77, self.fc)))) class CompilerAccessError(spack.error.SpackError): diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 6fc54daa61..23ff6cced4 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -44,15 +44,15 @@ _imported_compilers_module = 'spack.compilers' _required_instance_vars = ['cc', 'cxx', 'f77', 'fc'] -_optional_instance_vars = ['module'] +_optional_instance_vars = ['modules'] _default_order = ['gcc', 'intel', 'pgi', 'clang', 'xlc'] def _auto_compiler_spec(function): - def converter(cspec_like): + def converter(cspec_like, *args): if not isinstance(cspec_like, spack.spec.CompilerSpec): cspec_like = spack.spec.CompilerSpec(cspec_like) - return function(cspec_like) + return function(cspec_like, *args) return converter diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index c5041d67be..4b4cb69033 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -97,32 +97,6 @@ def concretize_version(self, spec): def concretize_architecture(self, spec): - """If the spec already had an architecture, return. Otherwise if - the root of the DAG has an architecture, then use that. - Otherwise take the system's default architecture. - - Intuition: Architectures won't be set a lot, and generally you - want the host system's architecture. When architectures are - mised in a spec, it is likely because the tool requries a - cross-compiled component, e.g. for tools that run on BlueGene - or Cray machines. These constraints will likely come directly - from packages, so require the user to be explicit if they want - to mess with the architecture, and revert to the default when - they're not explicit. - """ - if spec.architecture is not None: - return False - - if spec.root.architecture: - spec.architecture = spec.root.architecture - else: - spec.architecture = spack.architecture.sys_type() - - assert(spec.architecture is not None) - return True # changed - - - def new_concretize_architecture(self, spec): """If the spec already has an architecture and it is a an architecture type, return. Otherwise, if it has an architecture that is a string type, generate an architecture based on that type. If it has no architecture and the root of the @@ -146,7 +120,7 @@ def new_concretize_architecture(self, spec): else: arch = spack.architecture.sys_type() spec.architecture = arch.target('default') - + return True #changed diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 3e91958c2c..bc655d6051 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -199,6 +199,7 @@ def get_config(category_name): category.result_dict = _merge_dicts(category.result_dict, result) else: category.result_dict = result + return category.result_dict @@ -208,7 +209,7 @@ def get_compilers_config(arch=None): configuration""" global _compiler_by_arch if not arch: - arch = spack.architecture.sys_type() + arch = str(spack.architecture.sys_type()) if arch in _compiler_by_arch: return _compiler_by_arch[arch] @@ -305,7 +306,7 @@ def add_to_compiler_config(addition_dict, scope=None, arch=None): """Add compilerss to the configuration files""" if not arch: arch = spack.architecture.sys_type() - add_to_config('compilers', { arch : addition_dict }, scope) + add_to_config('compilers', { str(arch) : addition_dict }, scope) clear_config_caches() From 8a13d344a898195c6eb33d8bea151487d0d209bd Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 11 Nov 2015 15:07:57 -0800 Subject: [PATCH 045/481] bug fix in spec.py --- lib/spack/spack/spec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index e1fbb84423..3ba9c139a6 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1998,4 +1998,4 @@ def __init__(self, provided, required): class SpackYAMLError(spack.error.SpackError): def __init__(self, msg, yaml_error): - super(SpackError, self).__init__(msg, str(yaml_error)) + super(SpackYAMLError, self).__init__(msg, str(yaml_error)) From 3067705c109aef062823e62f0bb8a9d977a10a35 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 11 Nov 2015 16:16:47 -0800 Subject: [PATCH 046/481] made a to_dict for targets. Also a from_dict for sanity --- lib/spack/spack/architecture.py | 19 +++++++++++++++++++ lib/spack/spack/spec.py | 4 ++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index b1da8ae4a2..642a589f35 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -69,6 +69,25 @@ def compiler_strategy(self): else: return "PATH" + def to_dict(self): + print "to_dict" + d = {} + d['name'] = self.name + d['module_name'] = self.module_name + if self.architecture: + d['architecture'] = self.architecture + return d + + @staticmethod + def from_dict(d): + print "from_dict" + target = Target.__new__(Target) + target.name = d['name'] + target.module_name = d['module_name'] + if 'architecture' in d: + target.architecture = d['architecture'] + return target + def __str__(self): return self.name diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 3ba9c139a6..395faecde6 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -651,7 +651,7 @@ def to_node_dict(self): d = { 'variants' : dict( (name,v.enabled) for name, v in self.variants.items()), - 'arch' : self.architecture, + 'arch' : self.architecture.to_dict(), 'dependencies' : dict((d, self.dependencies[d].dag_hash()) for d in sorted(self.dependencies)) } @@ -680,7 +680,7 @@ def from_node_dict(node): spec = Spec(name) spec.versions = VersionList.from_dict(node) - spec.architecture = node['arch'] + spec.architecture = spack.architecture.Target.from_dict(node['arch']) if node['compiler'] is None: spec.compiler = None From 185f40eb8bbc19ecd7e2634844c97067388d0422 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 11 Nov 2015 16:33:22 -0800 Subject: [PATCH 047/481] fixed my git problems --- lib/spack/spack/compiler.py | 2 +- lib/spack/spack/compilers/gcc.py | 2 +- lib/spack/spack/compilers/intel.py | 2 +- lib/spack/spack/compilers/pgi.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index 41ff89a151..abf7223117 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -291,7 +291,7 @@ def find_in_modules(cls): # match = re.findall(r'(%s)/([^\s(]*)' % cls.PrgEnv_compiler, module) for name, version in matches: - v = version + '-craype' + v = version comp = cls(spack.spec.CompilerSpec(name + '@' + v), ['cc', 'CC', 'ftn'], [cls.PrgEnv, name +'/' + v]) diff --git a/lib/spack/spack/compilers/gcc.py b/lib/spack/spack/compilers/gcc.py index ff0b8889a8..a9a4f729eb 100644 --- a/lib/spack/spack/compilers/gcc.py +++ b/lib/spack/spack/compilers/gcc.py @@ -42,7 +42,7 @@ class Gcc(Compiler): # MacPorts builds gcc versions with prefixes and -mp-X.Y suffixes. suffixes = [r'-mp-\d\.\d'] - PrgEnv = 'gnu' + PrgEnv = 'PrgEnv-gnu' PrgEnv_compiler = 'gcc' @property diff --git a/lib/spack/spack/compilers/intel.py b/lib/spack/spack/compilers/intel.py index 7c485fe69d..4096e32a08 100644 --- a/lib/spack/spack/compilers/intel.py +++ b/lib/spack/spack/compilers/intel.py @@ -37,7 +37,7 @@ class Intel(Compiler): # Subclasses use possible names of Fortran 90 compiler fc_names = ['ifort'] - PrgEnv = 'intel' + PrgEnv = 'PrgEnv-intel' PrgEnv_compiler = 'intel' @property diff --git a/lib/spack/spack/compilers/pgi.py b/lib/spack/spack/compilers/pgi.py index 8f1ed28825..8010c1d04d 100644 --- a/lib/spack/spack/compilers/pgi.py +++ b/lib/spack/spack/compilers/pgi.py @@ -37,7 +37,7 @@ class Pgi(Compiler): # Subclasses use possible names of Fortran 90 compiler fc_names = ['pgf95', 'pgf90'] - PrgEnv = 'pgi' + PrgEnv = 'PrgEnv-pgi' PrgEnv_compiler = 'pgi' @classmethod From eb2c08315905e0484dbd4d866396563487d7d249 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 11 Nov 2015 17:29:47 -0800 Subject: [PATCH 048/481] Fixed the to_dict from_dict for targets on cray. First version to build/find properly on Cray --- lib/spack/spack/architecture.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 642a589f35..349a7b33db 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -70,12 +70,11 @@ def compiler_strategy(self): return "PATH" def to_dict(self): - print "to_dict" d = {} d['name'] = self.name d['module_name'] = self.module_name - if self.architecture: - d['architecture'] = self.architecture +# if self.architecture: +# d['architecture'] = self.architecture return d @staticmethod @@ -84,10 +83,13 @@ def from_dict(d): target = Target.__new__(Target) target.name = d['name'] target.module_name = d['module_name'] - if 'architecture' in d: - target.architecture = d['architecture'] +# if 'architecture' in d: +# target.architecture = d['architecture'] return target + def __repr__(self): + return self.__str__() + def __str__(self): return self.name @@ -138,7 +140,7 @@ def detect(self): raise NotImplementedError() def __repr__(self): - return self.__str__ + return self.__str__() def __str__(self): return self.name From e78b8c16be5334aa4aea29313a77da8923319a8f Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Thu, 12 Nov 2015 13:27:10 -0800 Subject: [PATCH 049/481] used key_comparator decorator from util.lang to compare targets. Fixes find bug --- lib/spack/spack/architecture.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 349a7b33db..f4b8585c2f 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -27,7 +27,7 @@ import platform as py_platform import inspect -from llnl.util.lang import memoized, list_modules +from llnl.util.lang import memoized, list_modules, key_ordering from llnl.util.filesystem import join_path import llnl.util.tty as tty @@ -47,6 +47,7 @@ def __init__(self): super(NoSysTypeError, self).__init__("Could not determine sys_type for this machine.") +@key_ordering class Target(object): """ Target is the processor of the host machine. The host machine may have different front-end and back-end targets, especially if it is a Cray machine. The target will have a name and @@ -79,7 +80,6 @@ def to_dict(self): @staticmethod def from_dict(d): - print "from_dict" target = Target.__new__(Target) target.name = d['name'] target.module_name = d['module_name'] @@ -87,6 +87,10 @@ def from_dict(d): # target.architecture = d['architecture'] return target + + def _cmp_key(self): + return (self.name, self.module_name) + def __repr__(self): return self.__str__() From 297cebd83328040aef71ed3aaf486a336011116d Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 12 Nov 2015 13:34:40 -0800 Subject: [PATCH 050/481] Got rid of print statement --- lib/spack/spack/architecture.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 349a7b33db..d65acd92ba 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -79,7 +79,6 @@ def to_dict(self): @staticmethod def from_dict(d): - print "from_dict" target = Target.__new__(Target) target.name = d['name'] target.module_name = d['module_name'] From 7c89f9d18c57e3e0abe40cf608f63015ea9b68f3 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Thu, 12 Nov 2015 14:31:41 -0800 Subject: [PATCH 051/481] Added the basic ouline of a cray compiler class in lib/spack/spack/compilers/cray.py --- lib/spack/spack/compilers/cray.py | 50 +++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 lib/spack/spack/compilers/cray.py diff --git a/lib/spack/spack/compilers/cray.py b/lib/spack/spack/compilers/cray.py new file mode 100644 index 0000000000..c2f3ecd6f4 --- /dev/null +++ b/lib/spack/spack/compilers/cray.py @@ -0,0 +1,50 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import llnl.util.tty as tty +from spack.compiler import * +from spack.version import ver + +class Cray(Compiler): + # Subclasses use possible names of C compiler + cc_names = ['cc'] + + # Subclasses use possible names of C++ compiler + cxx_names = ['CC'] + + # Subclasses use possible names of Fortran 77 compiler + f77_names = ['ftn'] + + # Subclasses use possible names of Fortran 90 compiler + fc_names = ['ftn'] + + # MacPorts builds gcc versions with prefixes and -mp-X.Y suffixes. + suffixes = [r'-mp-\d\.\d'] + + PrgEnv = 'PrgEnv-cray' + PrgEnv_compiler = 'craype' + + @property + def cxx11_flag(self): + return "-hstd=c++11" From a5ba69d68d1472f22454a93d045632b1340e2b04 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 13 Nov 2015 11:46:13 -0800 Subject: [PATCH 052/481] Added a _cmp_key for the architecture class --- lib/spack/spack/architecture.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index f4b8585c2f..786ef34071 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -97,7 +97,7 @@ def __repr__(self): def __str__(self): return self.name - +@key_ordering class Architecture(object): """ Abstract class that each type of Architecture will subclass. Will return a instance of it once it is returned @@ -148,7 +148,9 @@ def __repr__(self): def __str__(self): return self.name - + + def _cmp_key(self): + return (self.name, (_cmp_key(t) for t in self.targets.values())) def get_sys_type_from_spack_globals(): """Return the SYS_TYPE from spack globals, or None if it isn't set.""" From 9458f7c7d0ffebfbe33345c1a6b4b07c39e41e5f Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 13 Nov 2015 11:47:36 -0800 Subject: [PATCH 053/481] Got rid of my old method of finding in LOADEDMODULES --- lib/spack/spack/compiler.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index abf7223117..5d87588245 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -283,13 +283,9 @@ def find_in_modules(cls): modulecmd = which('modulecmd') modulecmd.add_default_arg('python') output = modulecmd('avail', return_oe=True) - matches = re.findall(r'(%s)/([^\s(]*)' % cls.PrgEnv_compiler, output) - -# loaded_modules = os.environ["LOADEDMODULES"].split(":") - #output = _shell('module avail %s' % cls.PrgEnv_compiler) -# for module in loaded_modules: -# match = re.findall(r'(%s)/([^\s(]*)' % cls.PrgEnv_compiler, module) - + matches = re.findall(r'(%s)/(\d+[\.\d]+)' % cls.PrgEnv_compiler, output) +# It's finding a weird third attribute + print matches for name, version in matches: v = version comp = cls(spack.spec.CompilerSpec(name + '@' + v), From 72fa3c04922b3630c15c69ff51022d88a8bf1de9 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 13 Nov 2015 11:48:15 -0800 Subject: [PATCH 054/481] Added craype support --- lib/spack/spack/compilers/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 23ff6cced4..6731318f43 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -46,7 +46,7 @@ _required_instance_vars = ['cc', 'cxx', 'f77', 'fc'] _optional_instance_vars = ['modules'] -_default_order = ['gcc', 'intel', 'pgi', 'clang', 'xlc'] +_default_order = ['gcc', 'intel', 'pgi', 'clang', 'xlc','craype'] def _auto_compiler_spec(function): def converter(cspec_like, *args): @@ -180,6 +180,7 @@ def compilers_for_spec(compiler_spec): """This gets all compilers that satisfy the supplied CompilerSpec. Returns an empty list if none are found. """ + # cray issue might be located here config = _get_config() def get_compiler(cspec): From bfd05d3d2702bb2a75fe310fb04eb78ac2526824 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 13 Nov 2015 11:48:49 -0800 Subject: [PATCH 055/481] Changed name of file and class to craype to avoid spack freaking out about versions --- lib/spack/spack/compilers/craype.py | 57 +++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 lib/spack/spack/compilers/craype.py diff --git a/lib/spack/spack/compilers/craype.py b/lib/spack/spack/compilers/craype.py new file mode 100644 index 0000000000..56d6d8d36c --- /dev/null +++ b/lib/spack/spack/compilers/craype.py @@ -0,0 +1,57 @@ +##############################################################################} +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import llnl.util.tty as tty + +from spack.build_environment import load_module +from spack.compiler import * +from spack.version import ver + +class Craype(Compiler): + # Subclasses use possible names of C compiler + cc_names = ['cc'] + + # Subclasses use possible names of C++ compiler + cxx_names = ['CC'] + + # Subclasses use possible names of Fortran 77 compiler + f77_names = ['ftn'] + + # Subclasses use possible names of Fortran 90 compiler + fc_names = ['ftn'] + + # MacPorts builds gcc versions with prefixes and -mp-X.Y suffixes. + suffixes = [r'-mp-\d\.\d'] + + PrgEnv = 'PrgEnv-cray' + PrgEnv_compiler = 'craype' + + @property + def cxx11_flag(self): + return "-hstd=c++11" + + @classmethod + def default_version(cls, comp): + return get_compiler_version(comp, r'([Vv]ersion).*(\d+(\.\d+)+)') + From fceb5a75b06b639a6f6d7edf33c3b727e71a491b Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 13 Nov 2015 11:53:38 -0800 Subject: [PATCH 056/481] Adding new files to compilers --- lib/spack/spack/compilers/cray.py | 50 ------------------------------- 1 file changed, 50 deletions(-) delete mode 100644 lib/spack/spack/compilers/cray.py diff --git a/lib/spack/spack/compilers/cray.py b/lib/spack/spack/compilers/cray.py deleted file mode 100644 index c2f3ecd6f4..0000000000 --- a/lib/spack/spack/compilers/cray.py +++ /dev/null @@ -1,50 +0,0 @@ -############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://scalability-llnl.github.io/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -import llnl.util.tty as tty -from spack.compiler import * -from spack.version import ver - -class Cray(Compiler): - # Subclasses use possible names of C compiler - cc_names = ['cc'] - - # Subclasses use possible names of C++ compiler - cxx_names = ['CC'] - - # Subclasses use possible names of Fortran 77 compiler - f77_names = ['ftn'] - - # Subclasses use possible names of Fortran 90 compiler - fc_names = ['ftn'] - - # MacPorts builds gcc versions with prefixes and -mp-X.Y suffixes. - suffixes = [r'-mp-\d\.\d'] - - PrgEnv = 'PrgEnv-cray' - PrgEnv_compiler = 'craype' - - @property - def cxx11_flag(self): - return "-hstd=c++11" From 8f56f5cfccb681286ab5d31dbdbed29cdf49a930 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 23 Nov 2015 14:02:59 -0800 Subject: [PATCH 057/481] Added path to netcdf and hdf5 using the env vars NETCDF_DIR and HDF5_DIR. --- var/spack/packages/adios/package.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/var/spack/packages/adios/package.py b/var/spack/packages/adios/package.py index 5f298fc6b5..05cbb5f963 100644 --- a/var/spack/packages/adios/package.py +++ b/var/spack/packages/adios/package.py @@ -1,8 +1,10 @@ -from spack import * +import os +from spack import * class Adios(Package): """The Adaptable IO System (ADIOS) provides a simple, - flexible way for scientists to describe the data in their code that may need to be written, + flexible way for scientists to describe the + data in their code that may need to be written, read, or processed outside of the running simulation """ @@ -19,10 +21,12 @@ class Adios(Package): depends_on('mxml') def install(self, spec, prefix): - configure_args = ["--prefix=%s" % prefix, "--with-mxml=%s" % spec['mxml'].prefix, - "--with-hdf5=%s" % spec['hdf5'].prefix, "--with-netcdf=%s" % os.environ['NETCDF_DIR'], + configure_args = ["--prefix=%s" % prefix, + "--with-mxml=%s" % spec['mxml'].prefix, + "--with-hdf5="+os.environ["HDF5_DIR"], + "--with-netcdf="+os.environ["NETCDF_DIR"], "--with-infiniband=no"] - + configure(*configure_args) make() make("install") From 89fbe4fdfa01b2461a5355a8922e6083f528e409 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Tue, 1 Dec 2015 11:55:38 -0800 Subject: [PATCH 058/481] Changed add_target to include variables rather than hard-coded strings --- lib/spack/spack/architectures/bgq.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/architectures/bgq.py b/lib/spack/spack/architectures/bgq.py index d3d4446e09..85487626f1 100644 --- a/lib/spack/spack/architectures/bgq.py +++ b/lib/spack/spack/architectures/bgq.py @@ -10,8 +10,8 @@ class Bgq(Architecture): def __init__(self): super(Bgq, self).__init__('cray') - self.add_target('power7', Target('power7')) - self.add_target('powerpc', Target('powerpc')) + self.add_target(self.front_end, Target(self.front_end)) + self.add_target(self.back_end, Target(self.back_end)) @classmethod def detect(self): From 58f2b39bc844d1c8beffd6449a9047c5ec5522e5 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Tue, 1 Dec 2015 11:57:17 -0800 Subject: [PATCH 059/481] Changed default to CPU_TARGET env var. Helps deal with target differences between cori and edison --- lib/spack/spack/architectures/cray.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/architectures/cray.py b/lib/spack/spack/architectures/cray.py index 640a3933e0..e4a3617dc9 100644 --- a/lib/spack/spack/architectures/cray.py +++ b/lib/spack/spack/architectures/cray.py @@ -6,15 +6,23 @@ class Cray(Architecture): priority = 20 front_end = 'sandybridge' back_end = 'ivybridge' - default = 'ivybridge' - + default = os.environ['CRAY_CPU_TARGET'] + #default = 'ivybridge' + def __init__(self): + ''' Since cori doesn't have ivybridge as a front end it's better + if we use CRAY_CPU_TARGET as the default. This will ensure + that if we're on a XC-40 or XC-30 then we can detect the target + ''' super(Cray, self).__init__('cray') + # Back End compiler needs the proper target module loaded. - self.add_target('ivybridge', Target('ivybridge','craype-ivybridge')) + self.add_target(self.front_end, Target(self.front_end,'craype-' + self.front_end)) + self.add_target(self.default, Target(self.default,'craype-' + self.default)) # Could switch to use modules and fe targets for front end # Currently using compilers by path for front end. - self.add_target('sandybridge', Target('sandybridge')) + self.add_target(self.back_end, Target('craype-' + self.back_end)) + @classmethod def detect(self): From d7fdb8e015d5e6c5b0e06608166307f50f2f046f Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Tue, 1 Dec 2015 11:57:54 -0800 Subject: [PATCH 060/481] Got rid of hard coded strings for adding default --- lib/spack/spack/architectures/linux.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/architectures/linux.py b/lib/spack/spack/architectures/linux.py index d7d9a994fa..6c454f7a40 100644 --- a/lib/spack/spack/architectures/linux.py +++ b/lib/spack/spack/architectures/linux.py @@ -9,7 +9,7 @@ class Linux(Architecture): def __init__(self): super(Linux, self).__init__('linux') - self.add_target('linux', Target('linux')) + self.add_target(self.default, Target(self.default)) @classmethod def detect(self): From 4e0d47f1dd1d0eb31ef29e1e25ab68870f087604 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Tue, 1 Dec 2015 11:59:19 -0800 Subject: [PATCH 061/481] building with GCC needs special variables added. Intel compilers build just fine --- var/spack/packages/adios/package.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/var/spack/packages/adios/package.py b/var/spack/packages/adios/package.py index 05cbb5f963..224c3b37d5 100644 --- a/var/spack/packages/adios/package.py +++ b/var/spack/packages/adios/package.py @@ -19,14 +19,19 @@ class Adios(Package): # module load cray-hdf5/1.8.14 # module load python/2.7.10 depends_on('mxml') - + def install(self, spec, prefix): configure_args = ["--prefix=%s" % prefix, "--with-mxml=%s" % spec['mxml'].prefix, - "--with-hdf5="+os.environ["HDF5_DIR"], - "--with-netcdf="+os.environ["NETCDF_DIR"], - "--with-infiniband=no"] - + "--with-hdf5=%s" % os.environ["HDF5_DIR"], + "--with-netcdf=%s" % os.environ["NETCDF_DIR"], + "--with-infiniband=no", + "MPICC=cc","MPICXX=CC","MPIFC=ftn", + "CPPFLAGS=-DMPICH_IGNORE_CXX_SEEK"] + + if spec.satisfies('%gcc'): + configure_args.extend(["CC=gcc", "CXX=g++", "FC=gfortran"]) + configure(*configure_args) make() make("install") From 0544b164fc717f5221d7545139c18752129499d4 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Tue, 1 Dec 2015 11:59:49 -0800 Subject: [PATCH 062/481] Updated script of mini-xml for adios package --- var/spack/packages/mxml/package.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/var/spack/packages/mxml/package.py b/var/spack/packages/mxml/package.py index fc32b56323..f79251d312 100644 --- a/var/spack/packages/mxml/package.py +++ b/var/spack/packages/mxml/package.py @@ -1,9 +1,11 @@ +import os from spack import * class Mxml(Package): """Mini-XML is a small XML library that you can use to read and write XML and XML-like data files in your application without requiring large - non-standard libraries""" + non-standard libraries + """ homepage = "http://www.msweet.org" url = "http://www.msweet.org/files/project3/mxml-2.9.tar.gz" @@ -18,7 +20,7 @@ class Mxml(Package): # Case statement to change CC and CXX flags def install(self, spec, prefix): - configure('--prefix=%s' % prefix, "--disable-shared") + configure('--prefix=%s' % prefix, "--disable-shared", 'CFLAGS=-static') make() make("install") From 028cca16e66c4a7dd867ef6be4cf38db9daa6e51 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 2 Dec 2015 10:15:33 -0800 Subject: [PATCH 063/481] Got rid of the unusued default strategy=None --- lib/spack/spack/architecture.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 786ef34071..a53222d36f 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -54,7 +54,6 @@ class Target(object): also the module_name (e.g craype-compiler). Targets will also recognize which architecture they came from using the set_architecture method. Targets will have compiler finding strategies """ - default_strategy = None # Can probably add a compiler path here def __init__(self,name, module_name=None): self.name = name # case of cray "ivybridge" but if it's x86_64 @@ -99,7 +98,8 @@ def __str__(self): @key_ordering class Architecture(object): - """ Abstract class that each type of Architecture will subclass. Will return a instance of it once it + """ Abstract class that each type of Architecture will subclass. + Will return a instance of it once it is returned """ From fee88d289d00f6886d0cd336aab4cabbb5d45acc Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 11 Dec 2015 12:03:13 -0800 Subject: [PATCH 064/481] Rewrote docstrings for sys_type() and got rid of unused functions --- lib/spack/spack/architecture.py | 28 ++++------------------------ 1 file changed, 4 insertions(+), 24 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index a53222d36f..beebeba4bf 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -189,22 +189,6 @@ def get_sys_type_from_uname(): except: return None -def get_sys_type_from_config_file(): - - spack_home_dir = os.environ["HOME"] + "/.spack" - yaml_file = os.path.join(spack_home_dir, 'architecture.yaml') - try: - config_dict = yaml.load(open(yaml_file)) # Fix this to have yaml.load() - arch = config_dict['architecture'] - front = arch['front'] - back = arch['back'] - return Architecture(front,back) - - except: - print "No architecture.yaml config file found" - return None - - @memoized def all_architectures(): modules = [] @@ -225,14 +209,10 @@ def all_architectures(): @memoized def sys_type(): - """Priority of gathering sys-type. - 1. YAML file that the user specifies the name of the architecture. e.g Cray-XC40 or Cray-XC30 - 2. UNAME - 3. GLOBALS - 4. MAC OSX - Yaml should be a priority here because we want the user to be able to specify the type of architecture to use. - If there is no yaml present then it should move on to the next function and stop immediately once it gets a - arch name + """ Gather a list of all available subclasses of architectures. + Sorts the list according to their priority looking. Priority is + an arbitrarily set number. Detects arch either using uname or + a file path (/opt/cray...) """ # Try to create an architecture object using the config file FIRST architecture_list = all_architectures() From 31df2dd9dd713e75d161eb5a0187536886363469 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 11 Dec 2015 12:04:47 -0800 Subject: [PATCH 065/481] Fixed targets ti be x86_64, important note: have not tested this on linux machine --- lib/spack/spack/architectures/linux.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/architectures/linux.py b/lib/spack/spack/architectures/linux.py index 6c454f7a40..052b10c019 100644 --- a/lib/spack/spack/architectures/linux.py +++ b/lib/spack/spack/architectures/linux.py @@ -3,9 +3,9 @@ class Linux(Architecture): priority = 60 - front_end = 'linux' - back_end = 'linux' - default = 'linux' + front_end = 'x86_64' + back_end = 'x86_64' + default = 'x86_64' def __init__(self): super(Linux, self).__init__('linux') @@ -15,4 +15,4 @@ def __init__(self): def detect(self): arch = subprocess.Popen(['uname', '-i'], stdout = subprocess.PIPE) arch, _ = arch.communicate() - return 'x86_64' in arch.strip() + return 'linux' in arch.strip().lower() From b5216f6ec8fc5b1cd7061d8610b51ceec5f162e8 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 11 Dec 2015 12:07:32 -0800 Subject: [PATCH 066/481] Commented out cxxflag. Not 100% sure it's needed --- lib/spack/spack/compilers/craype.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/spack/spack/compilers/craype.py b/lib/spack/spack/compilers/craype.py index 56d6d8d36c..e8ae284f5b 100644 --- a/lib/spack/spack/compilers/craype.py +++ b/lib/spack/spack/compilers/craype.py @@ -24,9 +24,9 @@ ############################################################################## import llnl.util.tty as tty -from spack.build_environment import load_module +#from spack.build_environment import load_module from spack.compiler import * -from spack.version import ver +#from spack.version import ver class Craype(Compiler): # Subclasses use possible names of C compiler @@ -47,9 +47,9 @@ class Craype(Compiler): PrgEnv = 'PrgEnv-cray' PrgEnv_compiler = 'craype' - @property - def cxx11_flag(self): - return "-hstd=c++11" +# @property +# def cxx11_flag(self): +# return "-hstd=c++11" @classmethod def default_version(cls, comp): From 217a2d9ea5bd6184d36ea0c32429c979aa088fbc Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Thu, 17 Dec 2015 15:26:27 -0800 Subject: [PATCH 067/481] fixed yaml error --- lib/spack/spack/architecture.py | 2 ++ lib/spack/spack/spec.py | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index beebeba4bf..3d1de39354 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -79,6 +79,8 @@ def to_dict(self): @staticmethod def from_dict(d): + if d is None: + return None target = Target.__new__(Target) target.name = d['name'] target.module_name = d['module_name'] diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 395faecde6..ff58091656 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -651,10 +651,13 @@ def to_node_dict(self): d = { 'variants' : dict( (name,v.enabled) for name, v in self.variants.items()), - 'arch' : self.architecture.to_dict(), 'dependencies' : dict((d, self.dependencies[d].dag_hash()) for d in sorted(self.dependencies)) } + if self.architecture: + d['arch'] = self.architecture.to_dict() + else: + d['arch'] = None if self.compiler: d.update(self.compiler.to_dict()) else: From 53808f254efc6919ec66043c709039c78c34e11a Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Mon, 4 Jan 2016 12:36:48 -0800 Subject: [PATCH 068/481] Support for cray external dependencies implemented in modules --- lib/spack/spack/build_environment.py | 54 +++++++++++++++++++++++++++- lib/spack/spack/concretize.py | 6 +++- lib/spack/spack/config.py | 13 +++++-- lib/spack/spack/spec.py | 9 +++-- 4 files changed, 75 insertions(+), 7 deletions(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 9e3be433fb..781039e073 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -32,10 +32,11 @@ import shutil import multiprocessing import platform +import re + from llnl.util.filesystem import * import spack -import spack.compilers as compilers from spack.util.executable import Executable, which from spack.util.environment import * @@ -108,6 +109,46 @@ def load_module(mod): exec(compile(load, '', 'exec')) +def get_path_from_module(mod): + """Inspects a TCL module for entries that indicate the absolute path + at which the library supported by said module can be found. + """ + # Create a modulecmd executable + modulecmd = which('modulecmd') + modulecmd.add_default_arg('python') + + # Read the module + text = modulecmd('show', mod, return_oe=True).split('\n') + + # If it lists its package directory, return that + for line in text: + if line.find(mod.upper()+'_DIR') >= 0: + words = line.split() + return words[2] + + # If it lists a -rpath instruction, use that + for line in text: + rpath = line.find('-rpath/') + if rpath >= 0: + return line[rpath+6:line.find('/lib')] + + # If it lists a -L instruction, use that + for line in text: + L = line.find('-L/') + if L >= 0: + return line[L+2:line.find('/lib')] + + # If it sets the LD_LIBRARY_PATH or CRAY_LD_LIBRARY_PATH, use that + for line in text: + if line.find('LD_LIBRARY_PATH') >= 0: + words = line.split() + path = words[2] + return path[:path.find('/lib')] + + # Unable to find module path + return None + + def set_compiler_environment_variables(pkg): assert(pkg.spec.concrete) compiler = pkg.compiler @@ -251,6 +292,17 @@ def set_module_variables_for_package(pkg): def get_rpaths(pkg): """Get a list of all the rpaths for a package.""" + + # First load all modules for external packages and update the external + # packages' paths to reflect what is found in the modules so that we can + # rpath through the modules when possible, but if not possible they are + # already loaded. + for spec in pkg.spec.traverse(root=False): + if spec.external_module: + load_module(spec.external_module) + spec.external = get_path_from_module(spec.external_module) + + # Construct rpaths from the paths of each dep rpaths = [pkg.prefix.lib, pkg.prefix.lib64] rpaths.extend(d.prefix.lib for d in pkg.spec.traverse(root=False) if os.path.isdir(d.prefix.lib)) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 13e9b477dc..7c7453353a 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -97,7 +97,7 @@ def _valid_virtuals_and_externals(self, spec): externals = spec_externals(pkg) buildable = not is_spec_nobuild(pkg) if buildable: - result.append((pkg, None)) + result.append((pkg, None, None)) if externals: sorted_externals = sorted(externals, cmp=lambda a,b: a[0].__cmp__(b[0])) for external in sorted_externals: @@ -131,6 +131,7 @@ def concretize_virtual_and_external(self, spec): if not candidate: #No ABI matches. Pick the top choice based on the orignal preferences. candidate = candidates[0] + external_module = candidate[2] external = candidate[1] candidate_spec = candidate[0] @@ -144,6 +145,9 @@ def concretize_virtual_and_external(self, spec): if not spec.external and external: spec.external = external changed = True + if not spec.external_module and external_module: + spec.external_module = external_module + changed = True #If we're external then trim the dependencies if external and spec.dependencies: diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 9919dcc045..d66ffb338e 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -103,6 +103,8 @@ from llnl.util.filesystem import mkdirp import copy +from spack.build_environment import get_path_from_module + _config_sections = {} class _ConfigCategory: name = None @@ -255,7 +257,8 @@ def get_packages_config(): package_name = spack.spec.Spec(p.keys()[0]).name if package_name not in indexed_packages: indexed_packages[package_name] = [] - indexed_packages[package_name].append({ spack.spec.Spec(key) : val for key, val in p.iteritems() }) + pkg_dict = dict([ (spack.spec.Spec(key), val) for key, val in p.iteritems()]) + indexed_packages[package_name].append( pkg_dict ) return indexed_packages @@ -286,9 +289,13 @@ def spec_externals(spec): if not pkg.satisfies(spec): continue path = conf.get('path', None) + module = conf.get('module', None) if not path: - continue - spec_locations.append( (pkg, path) ) + if not module: + continue + else: + path = get_path_from_module(module) + spec_locations.append( (pkg, path, module) ) return spec_locations diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 4289fe19cb..f496011d62 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -420,6 +420,7 @@ def __init__(self, spec_like, *dep_like, **kwargs): self._normal = kwargs.get('normal', False) self._concrete = kwargs.get('concrete', False) self.external = None + self.external_module = None # This allows users to construct a spec DAG with literals. # Note that given two specs a and b, Spec(a) copies a, but @@ -1352,8 +1353,9 @@ def _dup(self, other, **kwargs): changed = (self.name != other.name and self.versions != other.versions and \ self.architecture != other.architecture and self.compiler != other.compiler and \ self.variants != other.variants and self._normal != other._normal and \ - self.concrete != other.concrete and self.external != other.external) - + self.concrete != other.concrete and self.external != other.external and \ + self.external_module != other.external_module) + # Local node attributes get copied first. self.name = other.name self.versions = other.versions.copy() @@ -1365,6 +1367,7 @@ def _dup(self, other, **kwargs): self.variants = other.variants.copy() self.variants.spec = self self.external = other.external + self.external_module = other.external_module # If we copy dependencies, preserve DAG structure in the new spec if kwargs.get('deps', True): @@ -1383,6 +1386,7 @@ def _dup(self, other, **kwargs): self._normal = other._normal self._concrete = other._concrete self.external = other.external + self.external_module = other.external_module return changed @@ -1809,6 +1813,7 @@ def spec(self): spec.architecture = None spec.compiler = None spec.external = None + spec.external_module = None spec.dependents = DependencyMap() spec.dependencies = DependencyMap() From 751208cedf3506ebcd5119fd0d4d904bc28392e8 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Mon, 4 Jan 2016 16:35:06 -0800 Subject: [PATCH 069/481] pick last compiler instead of first, will be overwritten by more advanced preferences as mentioned in the email group --- lib/spack/spack/concretize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 7c7453353a..48d05f7f8d 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -287,7 +287,7 @@ def concretize_compiler(self, spec): raise UnavailableCompilerVersionError(other_compiler) # copy concrete version into other_compiler - spec.compiler = matches[0].copy() + spec.compiler = matches[len(matches)-1].copy() assert(spec.compiler.concrete) return True # things changed. From d2c2c46541d54e4dcde34e6199ba5dafa949bef1 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Tue, 5 Jan 2016 11:12:14 -0800 Subject: [PATCH 070/481] bug hunting --- lib/spack/spack/compilers/__init__.py | 4 ++-- lib/spack/spack/config.py | 2 +- lib/spack/spack/spec.py | 5 +++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 6731318f43..87106282cf 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -216,9 +216,9 @@ def compiler_for_spec(compiler_spec, target): assert(compiler_spec.concrete) compilers = compilers_for_spec(compiler_spec) if target.compiler_strategy == "PATH": - filter(lambda c: c.modules is None, compilers) + compilers = [c for c in compilers if c.modules is None] elif target.compiler_strategy == "MODULES": - filter(lambda c: c.modules is not None, compilers) + compilers = [c for c in compilers if c.modules is not None] assert(len(compilers) == 1) return compilers[0] diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index d66ffb338e..6e11cfa475 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -277,7 +277,7 @@ def is_spec_nobuild(spec): def spec_externals(spec): - """Return a list of spec, directory pairs for each external location for spec""" + """Return a list of spec, directory, module triples for each external location for spec""" allpkgs = get_packages_config() name = spec.name spec_locations = [] diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index f496011d62..126ffc4f95 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -771,8 +771,9 @@ def _replace_with(self, concrete): """Replace this virtual spec with a concrete spec.""" assert(self.virtual) for name, dependent in self.dependents.items(): - del dependent.dependencies[self.name] - dependent._add_dependency(concrete) + if not dependent.external: + del dependent.dependencies[self.name] + dependent._add_dependency(concrete) def _expand_virtual_packages(self): From 9848ad32fdadaf7decfb19aba03829708d8b6c85 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Tue, 5 Jan 2016 13:39:53 -0800 Subject: [PATCH 071/481] fixed errors caused by crayport on linux and added a darwin architecture --- lib/spack/spack/architecture.py | 6 +++-- lib/spack/spack/architectures/cray.py | 21 +++++++++------- lib/spack/spack/architectures/darwin.py | 18 ++++++++++++++ lib/spack/spack/architectures/linux.py | 4 ++-- lib/spack/spack/test/__init__.py | 5 ++-- lib/spack/spack/test/architecture.py | 32 ++++++++++++++++++------- 6 files changed, 62 insertions(+), 24 deletions(-) create mode 100644 lib/spack/spack/architectures/darwin.py diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 3d1de39354..1f608bd187 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -96,11 +96,13 @@ def __repr__(self): return self.__str__() def __str__(self): + if self.module_name: + return self.name + ' module: ' + self.module_name return self.name @key_ordering class Architecture(object): - """ Abstract class that each type of Architecture will subclass. + """ Abstract class that each type of Architecture will subclass. Will return a instance of it once it is returned """ @@ -150,7 +152,7 @@ def __repr__(self): def __str__(self): return self.name - + def _cmp_key(self): return (self.name, (_cmp_key(t) for t in self.targets.values())) diff --git a/lib/spack/spack/architectures/cray.py b/lib/spack/spack/architectures/cray.py index e4a3617dc9..47ede30145 100644 --- a/lib/spack/spack/architectures/cray.py +++ b/lib/spack/spack/architectures/cray.py @@ -4,27 +4,30 @@ class Cray(Architecture): priority = 20 - front_end = 'sandybridge' + front_end = 'sandybridge' back_end = 'ivybridge' - default = os.environ['CRAY_CPU_TARGET'] - #default = 'ivybridge' - + default = 'ivybridge' + def __init__(self): ''' Since cori doesn't have ivybridge as a front end it's better if we use CRAY_CPU_TARGET as the default. This will ensure that if we're on a XC-40 or XC-30 then we can detect the target ''' super(Cray, self).__init__('cray') - + + # Handle the default here so we can check for a key error + if 'CRAY_CPU_TARGET' in os.environ: + default = os.environ['CRAY_CPU_TARGET'] + # Back End compiler needs the proper target module loaded. - self.add_target(self.front_end, Target(self.front_end,'craype-' + self.front_end)) + self.add_target(self.back_end, Target(self.front_end,'craype-'+ self.back_end)) self.add_target(self.default, Target(self.default,'craype-' + self.default)) # Could switch to use modules and fe targets for front end # Currently using compilers by path for front end. - self.add_target(self.back_end, Target('craype-' + self.back_end)) - + self.add_target(self.front_end, Target(self.front_end)) + @classmethod def detect(self): return os.path.exists('/opt/cray/craype') - + diff --git a/lib/spack/spack/architectures/darwin.py b/lib/spack/spack/architectures/darwin.py new file mode 100644 index 0000000000..2f0d34c38d --- /dev/null +++ b/lib/spack/spack/architectures/darwin.py @@ -0,0 +1,18 @@ +import subprocess +from spack.architecture import Architecture, Target + +class Darwin(Architecture): + priority = 89 + front_end = 'x86_64' + back_end = 'x86_64' + default = 'x86_64' + + def __init__(self): + super(Darwin, self).__init__('darwin') + self.add_target(self.default, Target(self.default)) + + @classmethod + def detect(self): + arch = subprocess.Popen(['uname', '-a'], stdout = subprocess.PIPE) + arch, _ = arch.communicate() + return 'darwin' in arch.strip().lower() diff --git a/lib/spack/spack/architectures/linux.py b/lib/spack/spack/architectures/linux.py index 052b10c019..d63cf9179a 100644 --- a/lib/spack/spack/architectures/linux.py +++ b/lib/spack/spack/architectures/linux.py @@ -2,7 +2,7 @@ from spack.architecture import Architecture, Target class Linux(Architecture): - priority = 60 + priority = 90 front_end = 'x86_64' back_end = 'x86_64' default = 'x86_64' @@ -13,6 +13,6 @@ def __init__(self): @classmethod def detect(self): - arch = subprocess.Popen(['uname', '-i'], stdout = subprocess.PIPE) + arch = subprocess.Popen(['uname', '-a'], stdout = subprocess.PIPE) arch, _ = arch.communicate() return 'linux' in arch.strip().lower() diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py index ed51fac33a..96dc1f4331 100644 --- a/lib/spack/spack/test/__init__.py +++ b/lib/spack/spack/test/__init__.py @@ -31,7 +31,7 @@ import spack """Names of tests to be included in Spack's test suite""" -"""test_names = ['architecture', +test_names = ['architecture', 'versions', 'url_parse', 'url_substitution', @@ -58,8 +58,7 @@ 'optional_deps', 'make_executable', 'configure_guess'] -""" -test_names = ['architecture'] + def list_tests(): """Return names of all tests that can be run for Spack.""" diff --git a/lib/spack/spack/test/architecture.py b/lib/spack/spack/test/architecture.py index 6ff22aaa59..3a7474799a 100644 --- a/lib/spack/spack/test/architecture.py +++ b/lib/spack/spack/test/architecture.py @@ -2,18 +2,34 @@ the functions are looking for the correct architecture name """ import unittest +import os +import platform import spack from spack.architecture import * +from spack.architectures.cray import Cray +from spack.architectures.linux import Linux +from spack.architectures.bgq import Bgq class ArchitectureTest(unittest.TestCase): - def test_Architecture_class(self): - a = Architecture('Cray-XC40') - a.add_arch_strategy() - self.assertEquals(a.get_arch_dict(), {'Cray-XC40': 'MODULES'}) + def test_Architecture_class_and_compiler_strategies(self): + a = Cray() + t = a.target('default') + self.assertEquals(t.compiler_strategy, 'MODULES') + b = Linux() + s = b.target('default') + self.assertEquals(s.compiler_strategy, 'PATH') - def test_get_sys_type_from_config_file(self): - output_arch_class = get_sys_type_from_config_file() - my_arch_class = Architecture('Linux x86_64','Cray-xc40') + def test_sys_type(self): + output_arch_class = sys_type() + my_arch_class = None + if os.path.exists('/opt/cray/craype'): + my_arch_class = Cray() + elif os.path.exists('/bgsys'): + my_arch_class = Bgq() + elif 'Linux' in platform.system(): + my_arch_class = Linux() +# elif 'Darwin' in platform.system(): +# my_arch_class = Darwin() - self.assertEqual(output_arch_class, my_arch_class) + self.assertEqual(str(output_arch_class), str(my_arch_class)) From 1edbaa4e62d2eb59b53cf6b10f7854d4edcf8497 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Tue, 5 Jan 2016 14:13:09 -0800 Subject: [PATCH 072/481] removed debug printing --- lib/spack/spack/architecture.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 1f608bd187..1b4b7730b2 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -96,8 +96,6 @@ def __repr__(self): return self.__str__() def __str__(self): - if self.module_name: - return self.name + ' module: ' + self.module_name return self.name @key_ordering From 751503c434bd249ed9f76b6ad353d2adcdcc3997 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Tue, 5 Jan 2016 14:47:14 -0800 Subject: [PATCH 073/481] fixed haswell targeting bug --- lib/spack/spack/architectures/cray.py | 17 ++++++++++++----- lib/spack/spack/compilers/__init__.py | 1 + 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/lib/spack/spack/architectures/cray.py b/lib/spack/spack/architectures/cray.py index 47ede30145..dac3943fb7 100644 --- a/lib/spack/spack/architectures/cray.py +++ b/lib/spack/spack/architectures/cray.py @@ -17,15 +17,22 @@ def __init__(self): # Handle the default here so we can check for a key error if 'CRAY_CPU_TARGET' in os.environ: - default = os.environ['CRAY_CPU_TARGET'] + self.default = os.environ['CRAY_CPU_TARGET'] + + # Change the defaults to haswell if we're on an XC40 + if self.default == 'haswell': + self.front_end = self.default + self.back_end = self.default - # Back End compiler needs the proper target module loaded. - self.add_target(self.back_end, Target(self.front_end,'craype-'+ self.back_end)) - self.add_target(self.default, Target(self.default,'craype-' + self.default)) # Could switch to use modules and fe targets for front end # Currently using compilers by path for front end. self.add_target(self.front_end, Target(self.front_end)) - + # Back End compiler needs the proper target module loaded. + self.add_target(self.back_end, Target(self.front_end,'craype-'+ self.back_end)) + self.add_target(self.default, Target(self.default,'craype-' + self.default)) + # This is kludgy and the order matters when the targets are all haswell + # This is because the last one overwrites the others when they have the + # same name. @classmethod def detect(self): diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 87106282cf..21a41f38ea 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -206,6 +206,7 @@ def get_compiler(cspec): return cls(cspec, compiler_paths, mods) matches = find(compiler_spec) + print matches, 'matches' return [get_compiler(cspec) for cspec in matches] From 48b9023de48ba01f789705db421c9bb326202c0e Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Tue, 5 Jan 2016 14:51:18 -0800 Subject: [PATCH 074/481] removed debug printing --- lib/spack/spack/compilers/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 21a41f38ea..87106282cf 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -206,7 +206,6 @@ def get_compiler(cspec): return cls(cspec, compiler_paths, mods) matches = find(compiler_spec) - print matches, 'matches' return [get_compiler(cspec) for cspec in matches] From d6768cf9210288ec562330fac0978cfa0b8b8683 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Tue, 5 Jan 2016 16:23:32 -0800 Subject: [PATCH 075/481] minor bug fix --- lib/spack/spack/compiler.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index 5d87588245..b40f68b2c5 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -284,8 +284,7 @@ def find_in_modules(cls): modulecmd.add_default_arg('python') output = modulecmd('avail', return_oe=True) matches = re.findall(r'(%s)/(\d+[\.\d]+)' % cls.PrgEnv_compiler, output) -# It's finding a weird third attribute - print matches + for name, version in matches: v = version comp = cls(spack.spec.CompilerSpec(name + '@' + v), From c649610473a722388a09bf1000a8e3e9d6a4f9ff Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Tue, 5 Jan 2016 17:00:28 -0800 Subject: [PATCH 076/481] fixed concretization to only consider compilers found by the proper strategy --- lib/spack/spack/concretize.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 48d05f7f8d..f32e956a13 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -260,8 +260,27 @@ def concretize_compiler(self, spec): build with the compiler that will be used by libraries that link to this one, to maximize compatibility. """ + # Pass on concretizing the compiler if the architecture is not yet determined + if not spec.architecture: + #Although this usually means changed, this means awaiting other changes + return True + + # Examine only those compilers found by the proper compiler strategy for this architecture + # Takes advantage of the proper logic already existing in compiler_for_spec + # Should be redone more efficiently if this works all_compilers = spack.compilers.all_compilers() + def _proper_compiler_style(cspec, target): + compilers = spack.compilers.compilers_for_spec(cspec) + if target.compiler_strategy == 'PATH': + filter(lambda c: not c.modules, compilers) + if target.compiler_strategy == 'MODULES': + filter(lambda c: c.modules, compilers) + return compilers + + filter(lambda c: _proper_compiler_style(c, spec.architecture), all_compilers) + + if (spec.compiler and spec.compiler.concrete and spec.compiler in all_compilers): @@ -287,7 +306,7 @@ def concretize_compiler(self, spec): raise UnavailableCompilerVersionError(other_compiler) # copy concrete version into other_compiler - spec.compiler = matches[len(matches)-1].copy() + spec.compiler = matches[0].copy() assert(spec.compiler.concrete) return True # things changed. From 29e0ff61d7dc10b20b4b5d0d627cc066690ea84b Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Tue, 5 Jan 2016 17:14:35 -0800 Subject: [PATCH 077/481] fixed compiler finding so as not to identify non-existent versions of the intel compiler based on the version numbers of the PrgEnv-intel module --- lib/spack/spack/compiler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index b40f68b2c5..225c65917a 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -282,7 +282,7 @@ def find_in_modules(cls): modulecmd = which('modulecmd') modulecmd.add_default_arg('python') - output = modulecmd('avail', return_oe=True) + output = modulecmd('avail', cls.PrgEnv_compiler, return_oe=True) matches = re.findall(r'(%s)/(\d+[\.\d]+)' % cls.PrgEnv_compiler, output) for name, version in matches: From 93c9c45580d89ccbab441c5808192e149731c85f Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Tue, 5 Jan 2016 17:19:04 -0800 Subject: [PATCH 078/481] minor tweak of compiler priority while waiting for customizable compiler priorities --- lib/spack/spack/concretize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index f32e956a13..2013ae2e84 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -306,7 +306,7 @@ def _proper_compiler_style(cspec, target): raise UnavailableCompilerVersionError(other_compiler) # copy concrete version into other_compiler - spec.compiler = matches[0].copy() + spec.compiler = matches[-1].copy() assert(spec.compiler.concrete) return True # things changed. From 61b03b72b0fb1aa5c4ddbaff8079d9c0258efb76 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Tue, 5 Jan 2016 19:03:25 -0800 Subject: [PATCH 079/481] improved concretize efficiency for determining whether compilers come from the proper strategy --- lib/spack/spack/concretize.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 2013ae2e84..854db949e3 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -265,11 +265,9 @@ def concretize_compiler(self, spec): #Although this usually means changed, this means awaiting other changes return True - # Examine only those compilers found by the proper compiler strategy for this architecture + # Only use a matching compiler if it is of the proper style # Takes advantage of the proper logic already existing in compiler_for_spec - # Should be redone more efficiently if this works - all_compilers = spack.compilers.all_compilers() - + # Should think whether this can be more efficient def _proper_compiler_style(cspec, target): compilers = spack.compilers.compilers_for_spec(cspec) if target.compiler_strategy == 'PATH': @@ -278,8 +276,8 @@ def _proper_compiler_style(cspec, target): filter(lambda c: c.modules, compilers) return compilers - filter(lambda c: _proper_compiler_style(c, spec.architecture), all_compilers) - + + all_compilers = spack.compilers.all_compilers() if (spec.compiler and spec.compiler.concrete and @@ -306,7 +304,12 @@ def _proper_compiler_style(cspec, target): raise UnavailableCompilerVersionError(other_compiler) # copy concrete version into other_compiler - spec.compiler = matches[-1].copy() + index = len(matches)-1 + while not _proper_compiler_style(matches[index], spec.architecture): + index -= 1 + if index == 0: + raise NoValidVersionError(spec) + spec.compiler = matches[index].copy() assert(spec.compiler.concrete) return True # things changed. From 51bd91edc0679d21c26df8af17eaafb2a0ca6d56 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 6 Jan 2016 13:00:22 -0800 Subject: [PATCH 080/481] made the compiler strategy more easily controllable --- lib/spack/spack/architecture.py | 15 ++++++++------- lib/spack/spack/architectures/bgq.py | 4 ++-- lib/spack/spack/architectures/cray.py | 6 +++--- lib/spack/spack/architectures/darwin.py | 2 +- lib/spack/spack/architectures/linux.py | 2 +- lib/spack/spack/build_environment.py | 2 +- 6 files changed, 16 insertions(+), 15 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 1b4b7730b2..81aac6cc33 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -55,19 +55,20 @@ class Target(object): they came from using the set_architecture method. Targets will have compiler finding strategies """ - def __init__(self,name, module_name=None): + def __init__(self, name, compiler_strategy, module_name=None): self.name = name # case of cray "ivybridge" but if it's x86_64 + self.compiler_strategy = compiler_strategy self.module_name = module_name # craype-ivybridge def set_architecture(self, architecture): # Target should get the architecture class. self.architecture = architecture - @property - def compiler_strategy(self): - if self.module_name: # If there is a module_name given then use MODULES - return "MODULES" - else: - return "PATH" +# @property +# def compiler_strategy(self): +# if self.module_name: # If there is a module_name given then use MODULES +# return "MODULES" +# else: +# return "PATH" def to_dict(self): d = {} diff --git a/lib/spack/spack/architectures/bgq.py b/lib/spack/spack/architectures/bgq.py index 85487626f1..3ac5a59546 100644 --- a/lib/spack/spack/architectures/bgq.py +++ b/lib/spack/spack/architectures/bgq.py @@ -10,8 +10,8 @@ class Bgq(Architecture): def __init__(self): super(Bgq, self).__init__('cray') - self.add_target(self.front_end, Target(self.front_end)) - self.add_target(self.back_end, Target(self.back_end)) + self.add_target(self.front_end, Target(self.front_end, 'PATH')) + self.add_target(self.back_end, Target(self.back_end, 'PATH')) @classmethod def detect(self): diff --git a/lib/spack/spack/architectures/cray.py b/lib/spack/spack/architectures/cray.py index dac3943fb7..f109b48867 100644 --- a/lib/spack/spack/architectures/cray.py +++ b/lib/spack/spack/architectures/cray.py @@ -26,10 +26,10 @@ def __init__(self): # Could switch to use modules and fe targets for front end # Currently using compilers by path for front end. - self.add_target(self.front_end, Target(self.front_end)) + self.add_target(self.front_end, Target(self.front_end, 'PATH')) # Back End compiler needs the proper target module loaded. - self.add_target(self.back_end, Target(self.front_end,'craype-'+ self.back_end)) - self.add_target(self.default, Target(self.default,'craype-' + self.default)) +# self.add_target(self.back_end, Target(self.front_end, 'MODULES', 'craype-'+ self.back_end)) + self.add_target(self.default, Target(self.default, 'MODULES', 'craype-' + self.default)) # This is kludgy and the order matters when the targets are all haswell # This is because the last one overwrites the others when they have the # same name. diff --git a/lib/spack/spack/architectures/darwin.py b/lib/spack/spack/architectures/darwin.py index 2f0d34c38d..30fbde39bb 100644 --- a/lib/spack/spack/architectures/darwin.py +++ b/lib/spack/spack/architectures/darwin.py @@ -9,7 +9,7 @@ class Darwin(Architecture): def __init__(self): super(Darwin, self).__init__('darwin') - self.add_target(self.default, Target(self.default)) + self.add_target(self.default, Target(self.default, 'PATH')) @classmethod def detect(self): diff --git a/lib/spack/spack/architectures/linux.py b/lib/spack/spack/architectures/linux.py index d63cf9179a..bb3089ebf6 100644 --- a/lib/spack/spack/architectures/linux.py +++ b/lib/spack/spack/architectures/linux.py @@ -9,7 +9,7 @@ class Linux(Architecture): def __init__(self): super(Linux, self).__init__('linux') - self.add_target(self.default, Target(self.default)) + self.add_target(self.default, Target(self.default, 'PATH')) @classmethod def detect(self): diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 781039e073..fc71ae0331 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -226,7 +226,7 @@ def set_build_environment_variables(pkg): pkg_config_dirs.append(pcdir) path_put_first("PKG_CONFIG_PATH", pkg_config_dirs) - if pkg.spec.architecture.compiler_strategy.lower() == 'module': + if pkg.spec.architecture.module_name: load_module(pkg.spec.architecture.module_name) def set_module_variables_for_package(pkg): From b4a0004f44af89c034423adef5444a234c49d464 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 6 Jan 2016 14:50:31 -0800 Subject: [PATCH 081/481] Improved target cmp_key and to/from yaml functions --- lib/spack/spack/architecture.py | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 81aac6cc33..aaea7c81ac 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -60,22 +60,17 @@ def __init__(self, name, compiler_strategy, module_name=None): self.compiler_strategy = compiler_strategy self.module_name = module_name # craype-ivybridge - def set_architecture(self, architecture): # Target should get the architecture class. - self.architecture = architecture - -# @property -# def compiler_strategy(self): -# if self.module_name: # If there is a module_name given then use MODULES -# return "MODULES" -# else: -# return "PATH" + # Sets only the architecture name to avoid recursiveness + def set_architecture(self, architecture): + self.architecture_name = architecture.name def to_dict(self): d = {} d['name'] = self.name + d['compiler_strategy'] = self.compiler_strategy d['module_name'] = self.module_name -# if self.architecture: -# d['architecture'] = self.architecture + if self.architecture_name: + d['architecture'] = self.architecture_name return d @staticmethod @@ -84,14 +79,15 @@ def from_dict(d): return None target = Target.__new__(Target) target.name = d['name'] + target.compiler_strategy = d['compiler_strategy'] target.module_name = d['module_name'] -# if 'architecture' in d: -# target.architecture = d['architecture'] + if 'architecture' in d: + target.architecture_name = d['architecture'] return target def _cmp_key(self): - return (self.name, self.module_name) + return (self.name, self.compiler_strategy, self.module_name) def __repr__(self): return self.__str__() From 6ccd9d6fa4baa9b14e0cea19e55b24ad19d053f0 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Thu, 7 Jan 2016 11:49:01 -0800 Subject: [PATCH 082/481] Cleaned up naming conventions for architecture, split into platform and target --- lib/spack/spack/__init__.py | 2 +- lib/spack/spack/abi.py | 8 +- lib/spack/spack/architecture.py | 62 +++++----- lib/spack/spack/build_environment.py | 4 +- lib/spack/spack/cmd/find.py | 10 +- lib/spack/spack/concretize.py | 40 +++---- lib/spack/spack/config.py | 42 +++---- lib/spack/spack/directory_layout.py | 4 +- lib/spack/spack/modules.py | 4 +- lib/spack/spack/package.py | 2 +- .../{architectures => platforms}/__init__.py | 0 .../spack/{architectures => platforms}/bgq.py | 4 +- .../{architectures => platforms}/cray.py | 4 +- .../{architectures => platforms}/darwin.py | 10 +- .../{architectures => platforms}/linux.py | 10 +- lib/spack/spack/preferred_packages.py | 10 +- lib/spack/spack/spec.py | 108 +++++++++--------- lib/spack/spack/test/architecture.py | 23 ++-- lib/spack/spack/test/concretize.py | 4 +- lib/spack/spack/test/multimethod.py | 12 +- lib/spack/spack/test/spec_dag.py | 4 +- lib/spack/spack/test/spec_semantics.py | 4 +- .../mock_packages/multimethod/package.py | 10 +- 23 files changed, 191 insertions(+), 190 deletions(-) rename lib/spack/spack/{architectures => platforms}/__init__.py (100%) rename lib/spack/spack/{architectures => platforms}/bgq.py (84%) rename lib/spack/spack/{architectures => platforms}/cray.py (95%) rename lib/spack/spack/{architectures => platforms}/darwin.py (54%) rename lib/spack/spack/{architectures => platforms}/linux.py (54%) diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index b2ff2c692b..e1ef094f17 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -37,7 +37,7 @@ lib_path = join_path(prefix, "lib", "spack") build_env_path = join_path(lib_path, "env") module_path = join_path(lib_path, "spack") -arch_path = join_path(module_path, 'architectures') +platform_path = join_path(module_path, 'platforms') compilers_path = join_path(module_path, "compilers") test_path = join_path(module_path, "test") hooks_path = join_path(module_path, "hooks") diff --git a/lib/spack/spack/abi.py b/lib/spack/spack/abi.py index f0a997703c..1dd49d6b2c 100644 --- a/lib/spack/spack/abi.py +++ b/lib/spack/spack/abi.py @@ -34,9 +34,9 @@ class ABI(object): """This class provides methods to test ABI compatibility between specs. The current implementation is rather rough and could be improved.""" - def architecture_compatible(self, parent, child): - """Returns true iff the parent and child specs have ABI compatible architectures.""" - return not parent.architecture or not child.architecture or parent.architecture == child.architecture + def target_compatible(self, parent, child): + """Returns true iff the parent and child specs have ABI compatible targets.""" + return not parent.target or not child.target or parent.target == child.target @memoized @@ -123,6 +123,6 @@ def compiler_compatible(self, parent, child, **kwargs): def compatible(self, parent, child, **kwargs): """Returns true iff a parent and child spec are ABI compatible""" loosematch = kwargs.get('loose', False) - return self.architecture_compatible(parent, child) and \ + return self.target_compatible(parent, child) and \ self.compiler_compatible(parent, child, loose=loosematch) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index aaea7c81ac..a269767fab 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -51,8 +51,8 @@ def __init__(self): class Target(object): """ Target is the processor of the host machine. The host machine may have different front-end and back-end targets, especially if it is a Cray machine. The target will have a name and - also the module_name (e.g craype-compiler). Targets will also recognize which architecture - they came from using the set_architecture method. Targets will have compiler finding strategies + also the module_name (e.g craype-compiler). Targets will also recognize which platform + they came from using the set_platform method. Targets will have compiler finding strategies """ def __init__(self, name, compiler_strategy, module_name=None): @@ -60,17 +60,17 @@ def __init__(self, name, compiler_strategy, module_name=None): self.compiler_strategy = compiler_strategy self.module_name = module_name # craype-ivybridge - # Sets only the architecture name to avoid recursiveness - def set_architecture(self, architecture): - self.architecture_name = architecture.name + # Sets only the platform name to avoid recursiveness + def set_platform(self, platform): + self.platform_name = platform.name def to_dict(self): d = {} d['name'] = self.name d['compiler_strategy'] = self.compiler_strategy d['module_name'] = self.module_name - if self.architecture_name: - d['architecture'] = self.architecture_name + if self.platform_name: + d['platform'] = self.platform_name return d @staticmethod @@ -81,8 +81,8 @@ def from_dict(d): target.name = d['name'] target.compiler_strategy = d['compiler_strategy'] target.module_name = d['module_name'] - if 'architecture' in d: - target.architecture_name = d['architecture'] + if 'platform' in d: + target.platform_name = d['platform'] return target @@ -96,13 +96,13 @@ def __str__(self): return self.name @key_ordering -class Architecture(object): - """ Abstract class that each type of Architecture will subclass. +class Platform(object): + """ Abstract class that each type of Platform will subclass. Will return a instance of it once it is returned """ - priority = None # Subclass needs to set this number. This controls order in which arch is detected. + priority = None # Subclass needs to set this number. This controls order in which platform is detected. front_end = None back_end = None default = None # The default back end target. On cray ivybridge @@ -112,12 +112,12 @@ def __init__(self, name): self.name = name def add_target(self, name, target): - """Used by the architecture specific subclass to list available targets. Raises an error - if the architecture specifies a name that is reserved by spack as an alias. + """Used by the platform specific subclass to list available targets. Raises an error + if the platform specifies a name that is reserved by spack as an alias. """ if name in ['front_end', 'fe', 'back_end', 'be', 'default']: raise ValueError("%s is a spack reserved alias and cannot be the name of a target" % name) - target.set_architecture(self) + target.set_platform(self) self.targets[name] = target def target(self, name): @@ -137,7 +137,7 @@ def target(self, name): @classmethod def detect(self): """ Subclass is responsible for implementing this method. - Returns True if the architecture detects if it is the current architecture + Returns True if the Platform class detects that it is the current platform and False if it's not. """ raise NotImplementedError() @@ -182,18 +182,18 @@ def get_sys_type_from_uname(): Front-end config """ try: - arch_proc = subprocess.Popen(['uname', '-i'], stdout = subprocess.PIPE) - arch, _ = arch_proc.communicate() - return arch.strip() + platform_proc = subprocess.Popen(['uname', '-i'], stdout = subprocess.PIPE) + platform, _ = platform_proc.communicate() + return platform.strip() except: return None @memoized -def all_architectures(): +def all_platforms(): modules = [] - for name in list_modules(spack.arch_path): - mod_name = 'spack.architectures' + name - path = join_path(spack.arch_path, name) + ".py" + for name in list_modules(spack.platform_path): + mod_name = 'spack.platformss' + name + path = join_path(spack.platform_path, name) + ".py" mod = imp.load_source(mod_name, path) class_name = mod_to_class(name) if not hasattr(mod, class_name): @@ -208,16 +208,16 @@ def all_architectures(): @memoized def sys_type(): - """ Gather a list of all available subclasses of architectures. + """ Gather a list of all available subclasses of platforms. Sorts the list according to their priority looking. Priority is - an arbitrarily set number. Detects arch either using uname or + an arbitrarily set number. Detects platform either using uname or a file path (/opt/cray...) """ - # Try to create an architecture object using the config file FIRST - architecture_list = all_architectures() - architecture_list.sort(key = lambda a: a.priority) + # Try to create a Platform object using the config file FIRST + platform_list = all_platforms() + platform_list.sort(key = lambda a: a.priority) - for arch in architecture_list: - if arch.detect(): - return arch() + for platform in platform_list: + if platform.detect(): + return platform() diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index fc71ae0331..d849b4b56c 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -226,8 +226,8 @@ def set_build_environment_variables(pkg): pkg_config_dirs.append(pcdir) path_put_first("PKG_CONFIG_PATH", pkg_config_dirs) - if pkg.spec.architecture.module_name: - load_module(pkg.spec.architecture.module_name) + if pkg.spec.target.module_name: + load_module(pkg.spec.target.module_name) def set_module_variables_for_package(pkg): """Populate the module scope of install() with some useful functions. diff --git a/lib/spack/spack/cmd/find.py b/lib/spack/spack/cmd/find.py index 3c993990b1..7ea16bf09f 100644 --- a/lib/spack/spack/cmd/find.py +++ b/lib/spack/spack/cmd/find.py @@ -72,19 +72,19 @@ def display_specs(specs, **kwargs): hashes = True hlen = None - # Make a dict with specs keyed by architecture and compiler. - index = index_by(specs, ('architecture', 'compiler')) + # Make a dict with specs keyed by target and compiler. + index = index_by(specs, ('target', 'compiler')) # Traverse the index and print out each package - for i, (architecture, compiler) in enumerate(sorted(index)): + for i, (target, compiler) in enumerate(sorted(index)): if i > 0: print header = "%s{%s} / %s{%s}" % ( - spack.spec.architecture_color, architecture, + spack.spec.target_color, target, spack.spec.compiler_color, compiler) tty.hline(colorize(header), char='-') - specs = index[(architecture,compiler)] + specs = index[(target,compiler)] specs.sort() abbreviated = [s.format('$_$@$+', color=True) for s in specs] diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 854db949e3..fcd23a6055 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -208,30 +208,30 @@ def concretize_version(self, spec): return True # Things changed - def concretize_architecture(self, spec): - """If the spec already has an architecture and it is a an architecture type, - return. Otherwise, if it has an architecture that is a string type, generate an - architecture based on that type. If it has no architecture and the root of the - DAG has an architecture, then use that. Otherwise, take the system's default - architecture. + def concretize_target(self, spec): + """If the spec already has an target and it is a an target type, + return. Otherwise, if it has a target that is a string type, generate a + target based on that type. If it has no target and the root of the + DAG has an target, then use that. Otherwise, take the system's default + target. """ - if spec.architecture is not None: - if isinstance(spec.architecture,spack.architecture.Target): + if spec.target is not None: + if isinstance(spec.target,spack.architecture.Target): return False else: - arch = spack.architecture.sys_type() - spec.architecture = arch.target(spec.architecture) + platform = spack.architecture.sys_type() + spec.target = platform.target(spec.target) return True #changed - if spec.root.architecture: - if isinstance(spec.root.architecture,spack.architecture.Target): - spec.architecture = spec.root.architecture + if spec.root.target: + if isinstance(spec.root.target,spack.architecture.Target): + spec.target = spec.root.target else: - arch = spack.architecture.sys_type() - spec.architecture = arch.target(spec.root.architecture) + platform = spack.architecture.sys_type() + spec.target = platform.target(spec.root.target) else: - arch = spack.architecture.sys_type() - spec.architecture = arch.target('default') + platform = spack.architecture.sys_type() + spec.target = platform.target('default') return True #changed @@ -260,8 +260,8 @@ def concretize_compiler(self, spec): build with the compiler that will be used by libraries that link to this one, to maximize compatibility. """ - # Pass on concretizing the compiler if the architecture is not yet determined - if not spec.architecture: + # Pass on concretizing the compiler if the target is not yet determined + if not spec.target: #Although this usually means changed, this means awaiting other changes return True @@ -305,7 +305,7 @@ def _proper_compiler_style(cspec, target): # copy concrete version into other_compiler index = len(matches)-1 - while not _proper_compiler_style(matches[index], spec.architecture): + while not _proper_compiler_style(matches[index], spec.target): index -= 1 if index == 0: raise NoValidVersionError(spec) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 6e11cfa475..0622606a4f 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -129,7 +129,7 @@ def __init__(self, n, f, m): config_scopes = [('site', os.path.join(spack.etc_path, 'spack')), ('user', os.path.expanduser('~/.spack'))] -_compiler_by_arch = {} +_compiler_by_platform = {} _read_config_file_result = {} def _read_config_file(filename): """Read a given YAML configuration file""" @@ -156,7 +156,7 @@ def clear_config_caches(): s.files_read_from = [] s.result_dict = {} spack.config._read_config_file_result = {} - spack.config._compiler_by_arch = {} + spack.config._compiler_by_platform = {} spack.compilers._cached_default_compiler = None @@ -213,27 +213,27 @@ def get_config(category_name): return category.result_dict -def get_compilers_config(arch=None): +def get_compilers_config(platform=None): """Get the compiler configuration from config files for the given - architecture. Strips off the architecture component of the + platform. Strips off the platform component of the configuration""" - global _compiler_by_arch - if not arch: - arch = str(spack.architecture.sys_type()) - if arch in _compiler_by_arch: - return _compiler_by_arch[arch] + global _compiler_by_platform + if not platform: + platform = str(spack.architecture.sys_type()) + if platform in _compiler_by_platform: + return _compiler_by_platform[platform] cc_config = get_config('compilers') - if arch in cc_config and 'all' in cc_config: - arch_compiler = dict(cc_config[arch]) - _compiler_by_arch[arch] = _merge_dict(arch_compiler, cc_config['all']) - elif arch in cc_config: - _compiler_by_arch[arch] = cc_config[arch] + if platform in cc_config and 'all' in cc_config: + platform_compiler = dict(cc_config[platform]) + _compiler_by_platform[platform] = _merge_dict(platform_compiler, cc_config['all']) + elif platform in cc_config: + _compiler_by_platform[platform] = cc_config[platform] elif 'all' in cc_config: - _compiler_by_arch[arch] = cc_config['all'] + _compiler_by_platform[platform] = cc_config['all'] else: - _compiler_by_arch[arch] = {} - return _compiler_by_arch[arch] + _compiler_by_platform[platform] = {} + return _compiler_by_platform[platform] def get_mirror_config(): @@ -371,11 +371,11 @@ def add_to_mirror_config(addition_dict, scope=None): add_to_config('mirrors', addition_dict, scope) -def add_to_compiler_config(addition_dict, scope=None, arch=None): +def add_to_compiler_config(addition_dict, scope=None, platform=None): """Add compilers to the configuration files""" - if not arch: - arch = spack.architecture.sys_type() - add_to_config('compilers', { str(arch) : addition_dict }, scope) + if not platform: + platform = spack.architecture.sys_type() + add_to_config('compilers', { str(platform) : addition_dict }, scope) clear_config_caches() diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index 83e6eb566a..da3441fa13 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -156,7 +156,7 @@ def remove_install_directory(self, spec): class YamlDirectoryLayout(DirectoryLayout): """Lays out installation directories like this:: / - / + / -/ --- @@ -201,7 +201,7 @@ def relative_path_for_spec(self, spec): spec.dag_hash(self.hash_len)) path = join_path( - spec.architecture, + spec.target, "%s-%s" % (spec.compiler.name, spec.compiler.version), dir_name) diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index 56a61adefb..f84ac75c77 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -193,7 +193,7 @@ class Dotkit(EnvModule): @property def file_name(self): - return join_path(Dotkit.path, self.spec.architecture, + return join_path(Dotkit.path, self.spec.target, self.spec.format('$_$@$%@$+$#.dk')) @property @@ -230,7 +230,7 @@ class TclModule(EnvModule): @property def file_name(self): - return join_path(TclModule.path, self.spec.architecture, self.use_name) + return join_path(TclModule.path, self.spec.target, self.use_name) @property diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index e0097fa88d..1148e4ac3d 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -592,7 +592,7 @@ def compiler(self): """Get the spack.compiler.Compiler object used to build this package.""" if not self.spec.concrete: raise ValueError("Can only get a compiler for a concrete package.") - return spack.compilers.compiler_for_spec(self.spec.compiler, self.spec.architecture) + return spack.compilers.compiler_for_spec(self.spec.compiler, self.spec.target) def url_version(self, version): diff --git a/lib/spack/spack/architectures/__init__.py b/lib/spack/spack/platforms/__init__.py similarity index 100% rename from lib/spack/spack/architectures/__init__.py rename to lib/spack/spack/platforms/__init__.py diff --git a/lib/spack/spack/architectures/bgq.py b/lib/spack/spack/platforms/bgq.py similarity index 84% rename from lib/spack/spack/architectures/bgq.py rename to lib/spack/spack/platforms/bgq.py index 3ac5a59546..988e67023c 100644 --- a/lib/spack/spack/architectures/bgq.py +++ b/lib/spack/spack/platforms/bgq.py @@ -1,8 +1,8 @@ import os -from spack.architecture import Architecture, Target +from spack.architecture import Platform, Target -class Bgq(Architecture): +class Bgq(Platform): priority = 30 front_end = 'power7' back_end = 'powerpc' diff --git a/lib/spack/spack/architectures/cray.py b/lib/spack/spack/platforms/cray.py similarity index 95% rename from lib/spack/spack/architectures/cray.py rename to lib/spack/spack/platforms/cray.py index f109b48867..815a9aea98 100644 --- a/lib/spack/spack/architectures/cray.py +++ b/lib/spack/spack/platforms/cray.py @@ -1,8 +1,8 @@ import os -from spack.architecture import Architecture, Target +from spack.architecture import Platform, Target -class Cray(Architecture): +class Cray(Platform): priority = 20 front_end = 'sandybridge' back_end = 'ivybridge' diff --git a/lib/spack/spack/architectures/darwin.py b/lib/spack/spack/platforms/darwin.py similarity index 54% rename from lib/spack/spack/architectures/darwin.py rename to lib/spack/spack/platforms/darwin.py index 30fbde39bb..52284826b1 100644 --- a/lib/spack/spack/architectures/darwin.py +++ b/lib/spack/spack/platforms/darwin.py @@ -1,7 +1,7 @@ import subprocess -from spack.architecture import Architecture, Target +from spack.architecture import Platform, Target -class Darwin(Architecture): +class Darwin(Platform): priority = 89 front_end = 'x86_64' back_end = 'x86_64' @@ -13,6 +13,6 @@ def __init__(self): @classmethod def detect(self): - arch = subprocess.Popen(['uname', '-a'], stdout = subprocess.PIPE) - arch, _ = arch.communicate() - return 'darwin' in arch.strip().lower() + platform = subprocess.Popen(['uname', '-a'], stdout = subprocess.PIPE) + platform, _ = platform.communicate() + return 'darwin' in platform.strip().lower() diff --git a/lib/spack/spack/architectures/linux.py b/lib/spack/spack/platforms/linux.py similarity index 54% rename from lib/spack/spack/architectures/linux.py rename to lib/spack/spack/platforms/linux.py index bb3089ebf6..7f94d80c34 100644 --- a/lib/spack/spack/architectures/linux.py +++ b/lib/spack/spack/platforms/linux.py @@ -1,7 +1,7 @@ import subprocess -from spack.architecture import Architecture, Target +from spack.architecture import Platform, Target -class Linux(Architecture): +class Linux(Platform): priority = 90 front_end = 'x86_64' back_end = 'x86_64' @@ -13,6 +13,6 @@ def __init__(self): @classmethod def detect(self): - arch = subprocess.Popen(['uname', '-a'], stdout = subprocess.PIPE) - arch, _ = arch.communicate() - return 'linux' in arch.strip().lower() + platform = subprocess.Popen(['uname', '-a'], stdout = subprocess.PIPE) + platform, _ = platform.communicate() + return 'linux' in platform.strip().lower() diff --git a/lib/spack/spack/preferred_packages.py b/lib/spack/spack/preferred_packages.py index bc2a4ac234..993683a62f 100644 --- a/lib/spack/spack/preferred_packages.py +++ b/lib/spack/spack/preferred_packages.py @@ -163,11 +163,11 @@ def variant_compare(self, pkgname, a, b): return self._component_compare(pkgname, 'variant', a, b, False, None) - def architecture_compare(self, pkgname, a, b): - """Return less-than-0, 0, or greater than 0 if architecture a of pkgname is - respecively less-than, equal-to, or greater-than architecture b of pkgname. - One architecture is less-than another if it is preferred over the other.""" - return self._component_compare(pkgname, 'architecture', a, b, False, None) + def target_compare(self, pkgname, a, b): + """Return less-than-0, 0, or greater than 0 if target a of pkgname is + respecively less-than, equal-to, or greater-than target b of pkgname. + One target is less-than another if it is preferred over the other.""" + return self._component_compare(pkgname, 'target', a, b, False, None) def compiler_compare(self, pkgname, a, b): diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 126ffc4f95..9e011bfb9f 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -63,7 +63,7 @@ if it comes immediately after the compiler name. Otherwise it will be associated with the current package spec. -6. The architecture to build with. This is needed on machines where +6. The target to build with. This is needed on machines where cross-compilation is required Here is the EBNF grammar for a spec:: @@ -72,9 +72,9 @@ dep_list = { ^ spec } spec = id [ options ] options = { @version-list | +variant | -variant | ~variant | - %compiler | =architecture } + %compiler | =target } variant = id - architecture = id + target = id compiler = id [ version-list ] version-list = version [ { , version } ] version = id | id: | :id | id:id @@ -119,7 +119,7 @@ # Convenient names for color formats so that other things can use them compiler_color = '@g' version_color = '@c' -architecture_color = '@m' +target_color = '@m' enabled_variant_color = '@B' disabled_variant_color = '@r' dependency_color = '@.' @@ -130,7 +130,7 @@ See spack.color for descriptions of the color codes. """ color_formats = {'%' : compiler_color, '@' : version_color, - '=' : architecture_color, + '=' : target_color, '+' : enabled_variant_color, '~' : disabled_variant_color, '^' : dependency_color, @@ -407,7 +407,7 @@ def __init__(self, spec_like, *dep_like, **kwargs): self.name = other.name self.dependents = other.dependents self.versions = other.versions - self.architecture = other.architecture + self.target = other.target self.compiler = other.compiler self.dependencies = other.dependencies self.variants = other.variants @@ -452,11 +452,11 @@ def _set_compiler(self, compiler): self.compiler = compiler - def _set_architecture(self, architecture): - """Called by the parser to set the architecture.""" - if self.architecture: raise DuplicateArchitectureError( - "Spec for '%s' cannot have two architectures." % self.name) - self.architecture = architecture + def _set_target(self, target): + """Called by the parser to set the target.""" + if self.target: raise DuplicateTargetError( + "Spec for '%s' cannot have two targets." % self.name) + self.target = target def _add_dependency(self, spec): @@ -512,7 +512,7 @@ def is_virtual(name): @property def concrete(self): """A spec is concrete if it can describe only ONE build of a package. - If any of the name, version, architecture, compiler, + If any of the name, version, target, compiler, variants, or depdenencies are ambiguous,then it is not concrete. """ if self._concrete: @@ -521,7 +521,7 @@ def concrete(self): self._concrete = bool(not self.virtual and self.versions.concrete and self.variants.concrete - and self.architecture + and self.target and self.compiler and self.compiler.concrete and self.dependencies.concrete) return self._concrete @@ -656,10 +656,10 @@ def to_node_dict(self): 'dependencies' : dict((d, self.dependencies[d].dag_hash()) for d in sorted(self.dependencies)) } - if self.architecture: - d['arch'] = self.architecture.to_dict() + if self.target: + d['target'] = self.target.to_dict() else: - d['arch'] = None + d['target'] = None if self.compiler: d.update(self.compiler.to_dict()) else: @@ -685,7 +685,7 @@ def from_node_dict(node): spec = Spec(name) spec.versions = VersionList.from_dict(node) - spec.architecture = spack.architecture.Target.from_dict(node['arch']) + spec.target = spack.architecture.Target.from_dict(node['target']) if node['compiler'] is None: spec.compiler = None @@ -757,7 +757,7 @@ def _concretize_helper(self, presets=None, visited=None): # to presets below, their constraints will all be merged, but we'll # still need to select a concrete package later. changed |= any( - (spack.concretizer.concretize_architecture(self), + (spack.concretizer.concretize_target(self), spack.concretizer.concretize_compiler(self), spack.concretizer.concretize_version(self), spack.concretizer.concretize_variants(self))) @@ -1145,10 +1145,10 @@ def constrain(self, other, deps=True): raise UnsatisfiableVariantSpecError(self.variants[v], other.variants[v]) - if self.architecture is not None and other.architecture is not None: - if self.architecture != other.architecture: - raise UnsatisfiableArchitectureSpecError(self.architecture, - other.architecture) + if self.target is not None and other.target is not None: + if self.target != other.target: + raise UnsatisfiableTargetSpecError(self.target, + other.target) changed = False if self.compiler is not None and other.compiler is not None: @@ -1160,9 +1160,9 @@ def constrain(self, other, deps=True): changed |= self.versions.intersect(other.versions) changed |= self.variants.constrain(other.variants) - old = self.architecture - self.architecture = self.architecture or other.architecture - changed |= (self.architecture != old) + old = self.target + self.target = self.target or other.target + changed |= (self.target != old) if deps: changed |= self._constrain_dependencies(other) @@ -1273,12 +1273,12 @@ def satisfies(self, other, deps=True, strict=False): if not self.variants.satisfies(other.variants, strict=strict): return False - # Architecture satisfaction is currently just string equality. + # Target satisfaction is currently just class equality. # If not strict, None means unconstrained. - if self.architecture and other.architecture: - if self.architecture != other.architecture: + if self.target and other.target: + if self.target != other.target: return False - elif strict and (other.architecture and not self.architecture): + elif strict and (other.target and not self.target): return False # If we need to descend into dependencies, do it, otherwise we're done. @@ -1352,7 +1352,7 @@ def _dup(self, other, **kwargs): changed = True if hasattr(self, 'name'): changed = (self.name != other.name and self.versions != other.versions and \ - self.architecture != other.architecture and self.compiler != other.compiler and \ + self.target != other.target and self.compiler != other.compiler and \ self.variants != other.variants and self._normal != other._normal and \ self.concrete != other.concrete and self.external != other.external and \ self.external_module != other.external_module) @@ -1360,7 +1360,7 @@ def _dup(self, other, **kwargs): # Local node attributes get copied first. self.name = other.name self.versions = other.versions.copy() - self.architecture = other.architecture + self.target = other.target self.compiler = other.compiler.copy() if other.compiler else None if kwargs.get('cleardeps', True): self.dependents = DependencyMap() @@ -1490,7 +1490,7 @@ def ne_dag(self, other): def _cmp_node(self): """Comparison key for just *this node* and not its deps.""" return (self.name, self.versions, self.variants, - self.architecture, self.compiler) + self.target, self.compiler) def eq_node(self, other): @@ -1524,7 +1524,7 @@ def format(self, format_string='$_$@$%@$+$=', **kwargs): $% Compiler with '%' prefix $%@ Compiler with '%' prefix & compiler version with '@' prefix $+ Options - $= Architecture with '=' prefix + $= Target with '=' prefix $# 7-char prefix of DAG hash with '-' prefix $$ $ @@ -1536,7 +1536,7 @@ def format(self, format_string='$_$@$%@$+$=', **kwargs): ${COMPILERNAME} Compiler name ${COMPILERVER} Compiler version ${OPTIONS} Options - ${ARCHITECTURE} Architecture + ${TARGET} Target ${SHA1} Dependencies 8-char sha1 prefix ${SPACK_ROOT} The spack root directory @@ -1549,7 +1549,7 @@ def format(self, format_string='$_$@$%@$+$=', **kwargs): Anything else is copied verbatim into the output stream. *Example:* ``$_$@$+`` translates to the name, version, and options - of the package, but no dependencies, arch, or compiler. + of the package, but no dependencies, target, or compiler. TODO: allow, e.g., $6# to customize short hash length TODO: allow, e.g., $## for full hash. @@ -1593,8 +1593,8 @@ def write(s, c): if self.variants: write(fmt % str(self.variants), c) elif c == '=': - if self.architecture: - write(fmt % (c + str(self.architecture)), c) + if self.target: + write(fmt % (c + str(self.target)), c) elif c == '#': out.write('-' + fmt % (self.dag_hash(7))) elif c == '$': @@ -1641,9 +1641,9 @@ def write(s, c): elif named_str == 'OPTIONS': if self.variants: write(fmt % str(self.variants), '+') - elif named_str == 'ARCHITECTURE': - if self.architecture: - write(fmt % str(self.architecture), '=') + elif named_str == 'TARGET': + if self.target: + write(fmt % str(self.target), '=') elif named_str == 'SHA1': if self.dependencies: out.write(fmt % str(self.dep_hash(8))) @@ -1691,10 +1691,10 @@ def __cmp__(self, other): return spack.pkgsort.variant_compare(pkgname, self.variants, other.variants) - #Architecture - if self.architecture != other.architecture: - return spack.pkgsort.architecture_compare(pkgname, - self.architecture, other.architecture) + #Target + if self.target != other.target: + return spack.pkgsort.target_compare(pkgname, + self.target, other.target) #Dependency is not configurable if self.dep_hash() != other.dep_hash(): @@ -1811,7 +1811,7 @@ def spec(self): spec.name = self.token.value spec.versions = VersionList() spec.variants = VariantMap(spec) - spec.architecture = None + spec.target = None spec.compiler = None spec.external = None spec.external_module = None @@ -1842,7 +1842,7 @@ def spec(self): spec._set_compiler(self.compiler()) elif self.accept(EQ): - spec._set_architecture(self.architecture()) + spec._set_target(self.target()) else: break @@ -1860,7 +1860,7 @@ def variant(self): return self.token.value - def architecture(self): + def target(self): self.expect(ID) return self.token.value @@ -2000,10 +2000,10 @@ def __init__(self, pkg, variant): "Package %s has no variant %s!" % (pkg, variant)) -class DuplicateArchitectureError(SpecError): - """Raised when the same architecture occurs in a spec twice.""" +class DuplicateTargetError(SpecError): + """Raised when the same target occurs in a spec twice.""" def __init__(self, message): - super(DuplicateArchitectureError, self).__init__(message) + super(DuplicateTargetError, self).__init__(message) class InconsistentSpecError(SpecError): @@ -2082,11 +2082,11 @@ def __init__(self, provided, required): provided, required, "variant") -class UnsatisfiableArchitectureSpecError(UnsatisfiableSpecError): - """Raised when a spec architecture conflicts with package constraints.""" +class UnsatisfiableTargetSpecError(UnsatisfiableSpecError): + """Raised when a spec target conflicts with package constraints.""" def __init__(self, provided, required): - super(UnsatisfiableArchitectureSpecError, self).__init__( - provided, required, "architecture") + super(UnsatisfiableTargetSpecError, self).__init__( + provided, required, "target") class UnsatisfiableProviderSpecError(UnsatisfiableSpecError): diff --git a/lib/spack/spack/test/architecture.py b/lib/spack/spack/test/architecture.py index 3a7474799a..32b16feeb1 100644 --- a/lib/spack/spack/test/architecture.py +++ b/lib/spack/spack/test/architecture.py @@ -6,13 +6,14 @@ import platform import spack from spack.architecture import * -from spack.architectures.cray import Cray -from spack.architectures.linux import Linux -from spack.architectures.bgq import Bgq +from spack.platforms.cray import Cray +from spack.platforms.linux import Linux +from spack.platforms.bgq import Bgq +from spack.platforms.darwin import Darwin class ArchitectureTest(unittest.TestCase): - def test_Architecture_class_and_compiler_strategies(self): + def test_platform_class_and_compiler_strategies(self): a = Cray() t = a.target('default') self.assertEquals(t.compiler_strategy, 'MODULES') @@ -21,15 +22,15 @@ def test_Architecture_class_and_compiler_strategies(self): self.assertEquals(s.compiler_strategy, 'PATH') def test_sys_type(self): - output_arch_class = sys_type() + output_platform_class = sys_type() my_arch_class = None if os.path.exists('/opt/cray/craype'): - my_arch_class = Cray() + my_platform_class = Cray() elif os.path.exists('/bgsys'): - my_arch_class = Bgq() + my_platform_class = Bgq() elif 'Linux' in platform.system(): - my_arch_class = Linux() -# elif 'Darwin' in platform.system(): -# my_arch_class = Darwin() + my_platform_class = Linux() + elif 'Darwin' in platform.system(): + my_platform_class = Darwin() - self.assertEqual(str(output_arch_class), str(my_arch_class)) + self.assertEqual(str(output_platform_class), str(my_platform_class)) diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index f81a2f5af8..2403719134 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -46,8 +46,8 @@ def check_spec(self, abstract, concrete): if abstract.compiler and abstract.compiler.concrete: self.assertEqual(abstract.compiler, concrete.compiler) - if abstract.architecture and abstract.architecture.concrete: - self.assertEqual(abstract.architecture, concrete.architecture) + if abstract.target and abstract.target.concrete: + self.assertEqual(abstract.target, concrete.target) def check_concretize(self, abstract_spec): diff --git a/lib/spack/spack/test/multimethod.py b/lib/spack/spack/test/multimethod.py index cd5d9e625e..741dc96072 100644 --- a/lib/spack/spack/test/multimethod.py +++ b/lib/spack/spack/test/multimethod.py @@ -91,21 +91,21 @@ def test_default_works(self): self.assertEqual(pkg.has_a_default(), 'default') - def test_architecture_match(self): + def test_target_match(self): pkg = spack.db.get('multimethod=x86_64') - self.assertEqual(pkg.different_by_architecture(), 'x86_64') + self.assertEqual(pkg.different_by_target(), 'x86_64') pkg = spack.db.get('multimethod=ppc64') - self.assertEqual(pkg.different_by_architecture(), 'ppc64') + self.assertEqual(pkg.different_by_target(), 'ppc64') pkg = spack.db.get('multimethod=ppc32') - self.assertEqual(pkg.different_by_architecture(), 'ppc32') + self.assertEqual(pkg.different_by_target(), 'ppc32') pkg = spack.db.get('multimethod=arm64') - self.assertEqual(pkg.different_by_architecture(), 'arm64') + self.assertEqual(pkg.different_by_target(), 'arm64') pkg = spack.db.get('multimethod=macos') - self.assertRaises(NoSuchMethodError, pkg.different_by_architecture) + self.assertRaises(NoSuchMethodError, pkg.different_by_target) def test_dependency_match(self): diff --git a/lib/spack/spack/test/spec_dag.py b/lib/spack/spack/test/spec_dag.py index 549f829d3e..6a2dd6140f 100644 --- a/lib/spack/spack/test/spec_dag.py +++ b/lib/spack/spack/test/spec_dag.py @@ -240,10 +240,10 @@ def test_unsatisfiable_compiler_version(self): self.assertRaises(spack.spec.UnsatisfiableCompilerSpecError, spec.normalize) - def test_unsatisfiable_architecture(self): + def test_unsatisfiable_target(self): set_pkg_dep('mpileaks', 'mpich=bgqos_0') spec = Spec('mpileaks ^mpich=sles_10_ppc64 ^callpath ^dyninst ^libelf ^libdwarf') - self.assertRaises(spack.spec.UnsatisfiableArchitectureSpecError, spec.normalize) + self.assertRaises(spack.spec.UnsatisfiableTargetSpecError, spec.normalize) def test_invalid_dep(self): diff --git a/lib/spack/spack/test/spec_semantics.py b/lib/spack/spack/test/spec_semantics.py index 6666dbbb52..b8b4fb951c 100644 --- a/lib/spack/spack/test/spec_semantics.py +++ b/lib/spack/spack/test/spec_semantics.py @@ -110,7 +110,7 @@ def test_satisfies_compiler_version(self): self.check_unsatisfiable('foo %gcc@4.7', '%gcc@4.7.3') - def test_satisfies_architecture(self): + def test_satisfies_target(self): self.check_satisfies('foo=chaos_5_x86_64_ib', '=chaos_5_x86_64_ib') self.check_satisfies('foo=bgqos_0', '=bgqos_0') @@ -266,7 +266,7 @@ def test_constrain_variants(self): self.check_constrain('libelf+debug~foo', 'libelf+debug', 'libelf+debug~foo') - def test_constrain_arch(self): + def test_constrain_target(self): self.check_constrain('libelf=bgqos_0', 'libelf=bgqos_0', 'libelf=bgqos_0') self.check_constrain('libelf=bgqos_0', 'libelf', 'libelf=bgqos_0') diff --git a/var/spack/mock_packages/multimethod/package.py b/var/spack/mock_packages/multimethod/package.py index 75b1606ffc..f78ef3bb3d 100644 --- a/var/spack/mock_packages/multimethod/package.py +++ b/var/spack/mock_packages/multimethod/package.py @@ -101,22 +101,22 @@ def has_a_default(self): # - # Make sure we can switch methods on different architectures + # Make sure we can switch methods on different target # @when('=x86_64') - def different_by_architecture(self): + def different_by_target(self): return 'x86_64' @when('=ppc64') - def different_by_architecture(self): + def different_by_target(self): return 'ppc64' @when('=ppc32') - def different_by_architecture(self): + def different_by_target(self): return 'ppc32' @when('=arm64') - def different_by_architecture(self): + def different_by_target(self): return 'arm64' From 83917c4c302851a0d4ff91ef652fdd1b26fb1e08 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Thu, 7 Jan 2016 12:43:39 -0800 Subject: [PATCH 083/481] Improved target specification --- lib/spack/spack/architecture.py | 2 ++ lib/spack/spack/concretize.py | 38 ++++++++++++++++++++++++++++---- lib/spack/spack/platforms/bgq.py | 2 +- 3 files changed, 37 insertions(+), 5 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index a269767fab..6b9ae33b3e 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -93,6 +93,8 @@ def __repr__(self): return self.__str__() def __str__(self): + if self.platform_name: + return self.platform_name + '-' + self.name return self.name @key_ordering diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index fcd23a6055..ce86786004 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -33,11 +33,13 @@ TODO: make this customizable and allow users to configure concretization policies. """ +from llnl.util.filesystem import join_path import spack import spack.spec import spack.compilers import spack.architecture import spack.error +from spack.util.naming import mod_to_class from spack.version import * from functools import partial from spec import DependencyMap @@ -207,6 +209,36 @@ def concretize_version(self, spec): return True # Things changed + def class_from_platform_name(self, platform_name): + file_path = join_path(spack.platform_path, platform_name) + platform_mod = imp.load_source('spack.platforms', file_path + '.py') + cls = getattr(platform_mod, mod_to_class(platform_name)) + + return cls + + def spec_add_target_from_string(self, spec, target): + """If only a target is provided, spack will assume the default architecture. + A platform-target pair can be input delimited by a '-'. If either portion of + a platform-target pair is empty, spack will supply a default, in the case of + a blank target the default will be dependent on the platform. + E.g. x86_64 -> 64 bit x86 + bgq- -> default bgq target (back end/powerpc) + cray-hawswell -> haswell target on cray platform + """ + if '-' in target: + platform, target = target.split('-') + else: + platform = '' + + if platform != '': + cls = self.class_from_platform_name(platform) + platform = cls() + else: + platform = spack.architecture.sys_type() + if target != '': + spec.target = platform.target(target) + else: + spec.target = platform.target('default') def concretize_target(self, spec): """If the spec already has an target and it is a an target type, @@ -219,16 +251,14 @@ def concretize_target(self, spec): if isinstance(spec.target,spack.architecture.Target): return False else: - platform = spack.architecture.sys_type() - spec.target = platform.target(spec.target) + self.spec_add_target_from_string(spec, spec.target) return True #changed if spec.root.target: if isinstance(spec.root.target,spack.architecture.Target): spec.target = spec.root.target else: - platform = spack.architecture.sys_type() - spec.target = platform.target(spec.root.target) + self.spec_add_target_from_string(spec, spec.root.target) else: platform = spack.architecture.sys_type() spec.target = platform.target('default') diff --git a/lib/spack/spack/platforms/bgq.py b/lib/spack/spack/platforms/bgq.py index 988e67023c..6e872d2e72 100644 --- a/lib/spack/spack/platforms/bgq.py +++ b/lib/spack/spack/platforms/bgq.py @@ -9,7 +9,7 @@ class Bgq(Platform): default = 'powerpc' def __init__(self): - super(Bgq, self).__init__('cray') + super(Bgq, self).__init__('bgq') self.add_target(self.front_end, Target(self.front_end, 'PATH')) self.add_target(self.back_end, Target(self.back_end, 'PATH')) From 53d4f82ce1863d0871f53a7e20b75d73b8d4fad6 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Thu, 7 Jan 2016 13:01:05 -0800 Subject: [PATCH 084/481] Improved cray architecture class --- lib/spack/spack/platforms/{cray.py => cray_xc.py} | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) rename lib/spack/spack/platforms/{cray.py => cray_xc.py} (73%) diff --git a/lib/spack/spack/platforms/cray.py b/lib/spack/spack/platforms/cray_xc.py similarity index 73% rename from lib/spack/spack/platforms/cray.py rename to lib/spack/spack/platforms/cray_xc.py index 815a9aea98..e3adb182ea 100644 --- a/lib/spack/spack/platforms/cray.py +++ b/lib/spack/spack/platforms/cray_xc.py @@ -2,7 +2,7 @@ from spack.architecture import Platform, Target -class Cray(Platform): +class CrayXc(Platform): priority = 20 front_end = 'sandybridge' back_end = 'ivybridge' @@ -13,7 +13,7 @@ def __init__(self): if we use CRAY_CPU_TARGET as the default. This will ensure that if we're on a XC-40 or XC-30 then we can detect the target ''' - super(Cray, self).__init__('cray') + super(CrayXc, self).__init__('cray_xc') # Handle the default here so we can check for a key error if 'CRAY_CPU_TARGET' in os.environ: @@ -24,12 +24,17 @@ def __init__(self): self.front_end = self.default self.back_end = self.default + # Could switch to use modules and fe targets for front end # Currently using compilers by path for front end. - self.add_target(self.front_end, Target(self.front_end, 'PATH')) + self.add_target('sandybridge', Target('sandybridge', 'PATH')) + self.add_target('ivybridge', Target('ivybridge', 'MODULES', 'craype-ivybridge')) + self.add_target('haswell', Target('haswell', 'MODULES', 'craype-haswell')) + +# self.add_target(self.front_end, Target(self.front_end, 'PATH')) # Back End compiler needs the proper target module loaded. # self.add_target(self.back_end, Target(self.front_end, 'MODULES', 'craype-'+ self.back_end)) - self.add_target(self.default, Target(self.default, 'MODULES', 'craype-' + self.default)) +# self.add_target(self.default, Target(self.default, 'MODULES', 'craype-' + self.default)) # This is kludgy and the order matters when the targets are all haswell # This is because the last one overwrites the others when they have the # same name. From ba63111f453e258a0231a285dcbb11a876cccc19 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Thu, 7 Jan 2016 13:05:58 -0800 Subject: [PATCH 085/481] brought the architecture test up to date with new changes --- lib/spack/spack/test/architecture.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/test/architecture.py b/lib/spack/spack/test/architecture.py index 32b16feeb1..cf7938f5d6 100644 --- a/lib/spack/spack/test/architecture.py +++ b/lib/spack/spack/test/architecture.py @@ -6,7 +6,7 @@ import platform import spack from spack.architecture import * -from spack.platforms.cray import Cray +from spack.platforms.cray_xc import CrayXc from spack.platforms.linux import Linux from spack.platforms.bgq import Bgq from spack.platforms.darwin import Darwin @@ -14,7 +14,7 @@ class ArchitectureTest(unittest.TestCase): def test_platform_class_and_compiler_strategies(self): - a = Cray() + a = CrayXc() t = a.target('default') self.assertEquals(t.compiler_strategy, 'MODULES') b = Linux() @@ -25,7 +25,7 @@ def test_sys_type(self): output_platform_class = sys_type() my_arch_class = None if os.path.exists('/opt/cray/craype'): - my_platform_class = Cray() + my_platform_class = CrayXc() elif os.path.exists('/bgsys'): my_platform_class = Bgq() elif 'Linux' in platform.system(): From 49908386ef5d14be5cb34adb2040a7f2d535019c Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 8 Jan 2016 11:48:59 -0800 Subject: [PATCH 086/481] changing problematic package.py file permissions back to 0644 --- var/spack/packages/ImageMagick/package.py | 0 var/spack/packages/Mitos/package.py | 0 var/spack/packages/R/package.py | 0 var/spack/packages/SAMRAI/package.py | 0 var/spack/packages/adept-utils/package.py | 0 var/spack/packages/adios/package.py | 5 +++-- var/spack/packages/arpack/package.py | 0 var/spack/packages/atk/package.py | 0 var/spack/packages/atlas/package.py | 0 var/spack/packages/autoconf/package.py | 0 var/spack/packages/automaded/package.py | 0 var/spack/packages/automake/package.py | 0 var/spack/packages/bear/package.py | 0 var/spack/packages/bib2xhtml/package.py | 0 var/spack/packages/binutils/package.py | 0 var/spack/packages/bison/package.py | 0 var/spack/packages/boost/package.py | 0 var/spack/packages/boxlib/package.py | 0 var/spack/packages/bzip2/package.py | 0 var/spack/packages/cairo/package.py | 0 var/spack/packages/callpath/package.py | 0 var/spack/packages/cblas/package.py | 0 var/spack/packages/cgm/package.py | 0 var/spack/packages/clang/package.py | 0 var/spack/packages/cloog/package.py | 0 var/spack/packages/cmake/package.py | 0 var/spack/packages/coreutils/package.py | 0 var/spack/packages/cppcheck/package.py | 0 var/spack/packages/cram/package.py | 0 var/spack/packages/cube/package.py | 0 var/spack/packages/dbus/package.py | 0 var/spack/packages/dri2proto/package.py | 0 var/spack/packages/dtcmp/package.py | 0 var/spack/packages/dyninst/package.py | 0 var/spack/packages/extrae/package.py | 0 var/spack/packages/exuberant-ctags/package.py | 0 var/spack/packages/flex/package.py | 0 var/spack/packages/fontconfig/package.py | 0 var/spack/packages/freetype/package.py | 0 var/spack/packages/gasnet/package.py | 0 var/spack/packages/gcc/package.py | 0 var/spack/packages/gdk-pixbuf/package.py | 0 var/spack/packages/geos/package.py | 0 var/spack/packages/git/package.py | 0 var/spack/packages/glib/package.py | 0 var/spack/packages/global/package.py | 0 var/spack/packages/gmp/package.py | 0 var/spack/packages/gnutls/package.py | 0 var/spack/packages/gperf/package.py | 0 var/spack/packages/gperftools/package.py | 0 var/spack/packages/graphlib/package.py | 0 var/spack/packages/gtkplus/package.py | 0 var/spack/packages/harfbuzz/package.py | 0 var/spack/packages/hdf5/package.py | 0 var/spack/packages/hwloc/package.py | 0 var/spack/packages/hypre/package.py | 0 var/spack/packages/icu/package.py | 0 var/spack/packages/icu4c/package.py | 0 var/spack/packages/isl/package.py | 0 var/spack/packages/jdk/package.py | 0 var/spack/packages/jpeg/package.py | 0 var/spack/packages/lapack/package.py | 0 var/spack/packages/launchmon/package.py | 0 var/spack/packages/lcms/package.py | 0 var/spack/packages/libNBC/package.py | 0 var/spack/packages/libarchive/package.py | 0 var/spack/packages/libcircle/package.py | 0 var/spack/packages/libdrm/package.py | 0 var/spack/packages/libdwarf/package.py | 0 var/spack/packages/libelf/package.py | 0 var/spack/packages/libevent/package.py | 0 var/spack/packages/libffi/package.py | 0 var/spack/packages/libgcrypt/package.py | 0 var/spack/packages/libgpg-error/package.py | 0 var/spack/packages/libjpeg-turbo/package.py | 0 var/spack/packages/libmng/package.py | 0 var/spack/packages/libmonitor/package.py | 0 var/spack/packages/libpciaccess/package.py | 0 var/spack/packages/libpng/package.py | 0 var/spack/packages/libtiff/package.py | 0 var/spack/packages/libtool/package.py | 0 var/spack/packages/libunwind/package.py | 0 var/spack/packages/libuuid/package.py | 0 var/spack/packages/libxcb/package.py | 0 var/spack/packages/libxml2/package.py | 0 var/spack/packages/libxshmfence/package.py | 0 var/spack/packages/libxslt/package.py | 0 var/spack/packages/llvm-lld/package.py | 0 var/spack/packages/llvm/package.py | 0 var/spack/packages/lwgrp/package.py | 0 var/spack/packages/lwm2/package.py | 0 var/spack/packages/memaxes/package.py | 0 var/spack/packages/mesa/package.py | 0 var/spack/packages/metis/package.py | 0 var/spack/packages/mpc/package.py | 0 var/spack/packages/mpe2/package.py | 0 var/spack/packages/mpfr/package.py | 0 var/spack/packages/mpibash/package.py | 0 var/spack/packages/mpich/package.py | 0 var/spack/packages/mpileaks/package.py | 0 var/spack/packages/mrnet/package.py | 0 var/spack/packages/muster/package.py | 0 var/spack/packages/mvapich2/package.py | 0 var/spack/packages/nasm/package.py | 0 var/spack/packages/ncurses/package.py | 0 var/spack/packages/netcdf/package.py | 0 var/spack/packages/netgauge/package.py | 0 var/spack/packages/netlib-blas/package.py | 0 var/spack/packages/nettle/package.py | 0 var/spack/packages/ompss/package.py | 0 var/spack/packages/opari2/package.py | 0 var/spack/packages/openmpi/package.py | 0 var/spack/packages/openssl/package.py | 0 var/spack/packages/otf/package.py | 0 var/spack/packages/otf2/package.py | 0 var/spack/packages/pango/package.py | 0 var/spack/packages/papi/package.py | 0 var/spack/packages/paraver/package.py | 0 var/spack/packages/parmetis/package.py | 0 var/spack/packages/parpack/package.py | 0 var/spack/packages/pcre/package.py | 0 var/spack/packages/petsc/package.py | 0 var/spack/packages/pixman/package.py | 0 var/spack/packages/pmgr_collective/package.py | 0 var/spack/packages/postgresql/package.py | 0 var/spack/packages/ppl/package.py | 0 var/spack/packages/py-basemap/package.py | 0 var/spack/packages/py-biopython/package.py | 0 var/spack/packages/py-cffi/package.py | 0 var/spack/packages/py-cython/package.py | 0 var/spack/packages/py-dateutil/package.py | 0 var/spack/packages/py-epydoc/package.py | 0 var/spack/packages/py-genders/package.py | 0 var/spack/packages/py-gnuplot/package.py | 0 var/spack/packages/py-h5py/package.py | 0 var/spack/packages/py-ipython/package.py | 0 var/spack/packages/py-jsonschema/package.py | 0 var/spack/packages/py-libxml2/package.py | 0 var/spack/packages/py-lockfile/package.py | 0 var/spack/packages/py-mako/package.py | 0 var/spack/packages/py-matplotlib/package.py | 0 var/spack/packages/py-mpi4py/package.py | 0 var/spack/packages/py-mx/package.py | 0 var/spack/packages/py-nose/package.py | 0 var/spack/packages/py-numpy/package.py | 0 var/spack/packages/py-pandas/package.py | 0 var/spack/packages/py-pexpect/package.py | 0 var/spack/packages/py-pil/package.py | 0 var/spack/packages/py-pmw/package.py | 0 var/spack/packages/py-py4j/package.py | 0 var/spack/packages/py-pychecker/package.py | 0 var/spack/packages/py-pycparser/package.py | 0 var/spack/packages/py-pyelftools/package.py | 0 var/spack/packages/py-pygments/package.py | 0 var/spack/packages/py-pylint/package.py | 0 var/spack/packages/py-pypar/package.py | 0 var/spack/packages/py-pyparsing/package.py | 0 var/spack/packages/py-pyqt/package.py | 0 var/spack/packages/py-pyside/package.py | 0 var/spack/packages/py-python-daemon/package.py | 0 var/spack/packages/py-pytz/package.py | 0 var/spack/packages/py-pyzmq/package.py | 0 var/spack/packages/py-rpy2/package.py | 0 var/spack/packages/py-scientificpython/package.py | 0 var/spack/packages/py-scikit-learn/package.py | 0 var/spack/packages/py-scipy/package.py | 0 var/spack/packages/py-setuptools/package.py | 0 var/spack/packages/py-shiboken/package.py | 0 var/spack/packages/py-sip/package.py | 0 var/spack/packages/py-six/package.py | 0 var/spack/packages/py-sympy/package.py | 0 var/spack/packages/py-tornado/package.py | 0 var/spack/packages/py-traitlets/package.py | 0 var/spack/packages/py-virtualenv/package.py | 0 var/spack/packages/py-yapf/package.py | 0 var/spack/packages/python/package.py | 0 var/spack/packages/qhull/package.py | 0 var/spack/packages/qt/package.py | 0 var/spack/packages/qthreads/package.py | 0 var/spack/packages/ravel/package.py | 0 var/spack/packages/readline/package.py | 0 var/spack/packages/rose/package.py | 0 var/spack/packages/ruby/package.py | 0 var/spack/packages/scalasca/package.py | 0 var/spack/packages/scorep/package.py | 0 var/spack/packages/scotch/package.py | 0 var/spack/packages/scr/package.py | 0 var/spack/packages/silo/package.py | 0 var/spack/packages/spindle/package.py | 0 var/spack/packages/sqlite/package.py | 0 var/spack/packages/stat/package.py | 0 var/spack/packages/sundials/package.py | 0 var/spack/packages/swig/package.py | 0 var/spack/packages/task/package.py | 0 var/spack/packages/taskd/package.py | 0 var/spack/packages/tau/package.py | 0 var/spack/packages/tcl/package.py | 0 var/spack/packages/the_silver_searcher/package.py | 0 var/spack/packages/thrift/package.py | 0 var/spack/packages/tk/package.py | 0 var/spack/packages/tmux/package.py | 0 var/spack/packages/tmuxinator/package.py | 0 var/spack/packages/uncrustify/package.py | 0 var/spack/packages/util-linux/package.py | 0 var/spack/packages/vim/package.py | 0 var/spack/packages/vtk/package.py | 0 var/spack/packages/wget/package.py | 0 var/spack/packages/wx/package.py | 0 var/spack/packages/wxpropgrid/package.py | 0 var/spack/packages/xcb-proto/package.py | 0 var/spack/packages/xz/package.py | 0 var/spack/packages/yasm/package.py | 0 var/spack/packages/zlib/package.py | 0 var/spack/packages/zsh/package.py | 0 214 files changed, 3 insertions(+), 2 deletions(-) mode change 100755 => 100644 var/spack/packages/ImageMagick/package.py mode change 100755 => 100644 var/spack/packages/Mitos/package.py mode change 100755 => 100644 var/spack/packages/R/package.py mode change 100755 => 100644 var/spack/packages/SAMRAI/package.py mode change 100755 => 100644 var/spack/packages/adept-utils/package.py mode change 100755 => 100644 var/spack/packages/arpack/package.py mode change 100755 => 100644 var/spack/packages/atk/package.py mode change 100755 => 100644 var/spack/packages/atlas/package.py mode change 100755 => 100644 var/spack/packages/autoconf/package.py mode change 100755 => 100644 var/spack/packages/automaded/package.py mode change 100755 => 100644 var/spack/packages/automake/package.py mode change 100755 => 100644 var/spack/packages/bear/package.py mode change 100755 => 100644 var/spack/packages/bib2xhtml/package.py mode change 100755 => 100644 var/spack/packages/binutils/package.py mode change 100755 => 100644 var/spack/packages/bison/package.py mode change 100755 => 100644 var/spack/packages/boost/package.py mode change 100755 => 100644 var/spack/packages/boxlib/package.py mode change 100755 => 100644 var/spack/packages/bzip2/package.py mode change 100755 => 100644 var/spack/packages/cairo/package.py mode change 100755 => 100644 var/spack/packages/callpath/package.py mode change 100755 => 100644 var/spack/packages/cblas/package.py mode change 100755 => 100644 var/spack/packages/cgm/package.py mode change 100755 => 100644 var/spack/packages/clang/package.py mode change 100755 => 100644 var/spack/packages/cloog/package.py mode change 100755 => 100644 var/spack/packages/cmake/package.py mode change 100755 => 100644 var/spack/packages/coreutils/package.py mode change 100755 => 100644 var/spack/packages/cppcheck/package.py mode change 100755 => 100644 var/spack/packages/cram/package.py mode change 100755 => 100644 var/spack/packages/cube/package.py mode change 100755 => 100644 var/spack/packages/dbus/package.py mode change 100755 => 100644 var/spack/packages/dri2proto/package.py mode change 100755 => 100644 var/spack/packages/dtcmp/package.py mode change 100755 => 100644 var/spack/packages/dyninst/package.py mode change 100755 => 100644 var/spack/packages/extrae/package.py mode change 100755 => 100644 var/spack/packages/exuberant-ctags/package.py mode change 100755 => 100644 var/spack/packages/flex/package.py mode change 100755 => 100644 var/spack/packages/fontconfig/package.py mode change 100755 => 100644 var/spack/packages/freetype/package.py mode change 100755 => 100644 var/spack/packages/gasnet/package.py mode change 100755 => 100644 var/spack/packages/gcc/package.py mode change 100755 => 100644 var/spack/packages/gdk-pixbuf/package.py mode change 100755 => 100644 var/spack/packages/geos/package.py mode change 100755 => 100644 var/spack/packages/git/package.py mode change 100755 => 100644 var/spack/packages/glib/package.py mode change 100755 => 100644 var/spack/packages/global/package.py mode change 100755 => 100644 var/spack/packages/gmp/package.py mode change 100755 => 100644 var/spack/packages/gnutls/package.py mode change 100755 => 100644 var/spack/packages/gperf/package.py mode change 100755 => 100644 var/spack/packages/gperftools/package.py mode change 100755 => 100644 var/spack/packages/graphlib/package.py mode change 100755 => 100644 var/spack/packages/gtkplus/package.py mode change 100755 => 100644 var/spack/packages/harfbuzz/package.py mode change 100755 => 100644 var/spack/packages/hdf5/package.py mode change 100755 => 100644 var/spack/packages/hwloc/package.py mode change 100755 => 100644 var/spack/packages/hypre/package.py mode change 100755 => 100644 var/spack/packages/icu/package.py mode change 100755 => 100644 var/spack/packages/icu4c/package.py mode change 100755 => 100644 var/spack/packages/isl/package.py mode change 100755 => 100644 var/spack/packages/jdk/package.py mode change 100755 => 100644 var/spack/packages/jpeg/package.py mode change 100755 => 100644 var/spack/packages/lapack/package.py mode change 100755 => 100644 var/spack/packages/launchmon/package.py mode change 100755 => 100644 var/spack/packages/lcms/package.py mode change 100755 => 100644 var/spack/packages/libNBC/package.py mode change 100755 => 100644 var/spack/packages/libarchive/package.py mode change 100755 => 100644 var/spack/packages/libcircle/package.py mode change 100755 => 100644 var/spack/packages/libdrm/package.py mode change 100755 => 100644 var/spack/packages/libdwarf/package.py mode change 100755 => 100644 var/spack/packages/libelf/package.py mode change 100755 => 100644 var/spack/packages/libevent/package.py mode change 100755 => 100644 var/spack/packages/libffi/package.py mode change 100755 => 100644 var/spack/packages/libgcrypt/package.py mode change 100755 => 100644 var/spack/packages/libgpg-error/package.py mode change 100755 => 100644 var/spack/packages/libjpeg-turbo/package.py mode change 100755 => 100644 var/spack/packages/libmng/package.py mode change 100755 => 100644 var/spack/packages/libmonitor/package.py mode change 100755 => 100644 var/spack/packages/libpciaccess/package.py mode change 100755 => 100644 var/spack/packages/libpng/package.py mode change 100755 => 100644 var/spack/packages/libtiff/package.py mode change 100755 => 100644 var/spack/packages/libtool/package.py mode change 100755 => 100644 var/spack/packages/libunwind/package.py mode change 100755 => 100644 var/spack/packages/libuuid/package.py mode change 100755 => 100644 var/spack/packages/libxcb/package.py mode change 100755 => 100644 var/spack/packages/libxml2/package.py mode change 100755 => 100644 var/spack/packages/libxshmfence/package.py mode change 100755 => 100644 var/spack/packages/libxslt/package.py mode change 100755 => 100644 var/spack/packages/llvm-lld/package.py mode change 100755 => 100644 var/spack/packages/llvm/package.py mode change 100755 => 100644 var/spack/packages/lwgrp/package.py mode change 100755 => 100644 var/spack/packages/lwm2/package.py mode change 100755 => 100644 var/spack/packages/memaxes/package.py mode change 100755 => 100644 var/spack/packages/mesa/package.py mode change 100755 => 100644 var/spack/packages/metis/package.py mode change 100755 => 100644 var/spack/packages/mpc/package.py mode change 100755 => 100644 var/spack/packages/mpe2/package.py mode change 100755 => 100644 var/spack/packages/mpfr/package.py mode change 100755 => 100644 var/spack/packages/mpibash/package.py mode change 100755 => 100644 var/spack/packages/mpich/package.py mode change 100755 => 100644 var/spack/packages/mpileaks/package.py mode change 100755 => 100644 var/spack/packages/mrnet/package.py mode change 100755 => 100644 var/spack/packages/muster/package.py mode change 100755 => 100644 var/spack/packages/mvapich2/package.py mode change 100755 => 100644 var/spack/packages/nasm/package.py mode change 100755 => 100644 var/spack/packages/ncurses/package.py mode change 100755 => 100644 var/spack/packages/netcdf/package.py mode change 100755 => 100644 var/spack/packages/netgauge/package.py mode change 100755 => 100644 var/spack/packages/netlib-blas/package.py mode change 100755 => 100644 var/spack/packages/nettle/package.py mode change 100755 => 100644 var/spack/packages/ompss/package.py mode change 100755 => 100644 var/spack/packages/opari2/package.py mode change 100755 => 100644 var/spack/packages/openmpi/package.py mode change 100755 => 100644 var/spack/packages/openssl/package.py mode change 100755 => 100644 var/spack/packages/otf/package.py mode change 100755 => 100644 var/spack/packages/otf2/package.py mode change 100755 => 100644 var/spack/packages/pango/package.py mode change 100755 => 100644 var/spack/packages/papi/package.py mode change 100755 => 100644 var/spack/packages/paraver/package.py mode change 100755 => 100644 var/spack/packages/parmetis/package.py mode change 100755 => 100644 var/spack/packages/parpack/package.py mode change 100755 => 100644 var/spack/packages/pcre/package.py mode change 100755 => 100644 var/spack/packages/petsc/package.py mode change 100755 => 100644 var/spack/packages/pixman/package.py mode change 100755 => 100644 var/spack/packages/pmgr_collective/package.py mode change 100755 => 100644 var/spack/packages/postgresql/package.py mode change 100755 => 100644 var/spack/packages/ppl/package.py mode change 100755 => 100644 var/spack/packages/py-basemap/package.py mode change 100755 => 100644 var/spack/packages/py-biopython/package.py mode change 100755 => 100644 var/spack/packages/py-cffi/package.py mode change 100755 => 100644 var/spack/packages/py-cython/package.py mode change 100755 => 100644 var/spack/packages/py-dateutil/package.py mode change 100755 => 100644 var/spack/packages/py-epydoc/package.py mode change 100755 => 100644 var/spack/packages/py-genders/package.py mode change 100755 => 100644 var/spack/packages/py-gnuplot/package.py mode change 100755 => 100644 var/spack/packages/py-h5py/package.py mode change 100755 => 100644 var/spack/packages/py-ipython/package.py mode change 100755 => 100644 var/spack/packages/py-jsonschema/package.py mode change 100755 => 100644 var/spack/packages/py-libxml2/package.py mode change 100755 => 100644 var/spack/packages/py-lockfile/package.py mode change 100755 => 100644 var/spack/packages/py-mako/package.py mode change 100755 => 100644 var/spack/packages/py-matplotlib/package.py mode change 100755 => 100644 var/spack/packages/py-mpi4py/package.py mode change 100755 => 100644 var/spack/packages/py-mx/package.py mode change 100755 => 100644 var/spack/packages/py-nose/package.py mode change 100755 => 100644 var/spack/packages/py-numpy/package.py mode change 100755 => 100644 var/spack/packages/py-pandas/package.py mode change 100755 => 100644 var/spack/packages/py-pexpect/package.py mode change 100755 => 100644 var/spack/packages/py-pil/package.py mode change 100755 => 100644 var/spack/packages/py-pmw/package.py mode change 100755 => 100644 var/spack/packages/py-py4j/package.py mode change 100755 => 100644 var/spack/packages/py-pychecker/package.py mode change 100755 => 100644 var/spack/packages/py-pycparser/package.py mode change 100755 => 100644 var/spack/packages/py-pyelftools/package.py mode change 100755 => 100644 var/spack/packages/py-pygments/package.py mode change 100755 => 100644 var/spack/packages/py-pylint/package.py mode change 100755 => 100644 var/spack/packages/py-pypar/package.py mode change 100755 => 100644 var/spack/packages/py-pyparsing/package.py mode change 100755 => 100644 var/spack/packages/py-pyqt/package.py mode change 100755 => 100644 var/spack/packages/py-pyside/package.py mode change 100755 => 100644 var/spack/packages/py-python-daemon/package.py mode change 100755 => 100644 var/spack/packages/py-pytz/package.py mode change 100755 => 100644 var/spack/packages/py-pyzmq/package.py mode change 100755 => 100644 var/spack/packages/py-rpy2/package.py mode change 100755 => 100644 var/spack/packages/py-scientificpython/package.py mode change 100755 => 100644 var/spack/packages/py-scikit-learn/package.py mode change 100755 => 100644 var/spack/packages/py-scipy/package.py mode change 100755 => 100644 var/spack/packages/py-setuptools/package.py mode change 100755 => 100644 var/spack/packages/py-shiboken/package.py mode change 100755 => 100644 var/spack/packages/py-sip/package.py mode change 100755 => 100644 var/spack/packages/py-six/package.py mode change 100755 => 100644 var/spack/packages/py-sympy/package.py mode change 100755 => 100644 var/spack/packages/py-tornado/package.py mode change 100755 => 100644 var/spack/packages/py-traitlets/package.py mode change 100755 => 100644 var/spack/packages/py-virtualenv/package.py mode change 100755 => 100644 var/spack/packages/py-yapf/package.py mode change 100755 => 100644 var/spack/packages/python/package.py mode change 100755 => 100644 var/spack/packages/qhull/package.py mode change 100755 => 100644 var/spack/packages/qt/package.py mode change 100755 => 100644 var/spack/packages/qthreads/package.py mode change 100755 => 100644 var/spack/packages/ravel/package.py mode change 100755 => 100644 var/spack/packages/readline/package.py mode change 100755 => 100644 var/spack/packages/rose/package.py mode change 100755 => 100644 var/spack/packages/ruby/package.py mode change 100755 => 100644 var/spack/packages/scalasca/package.py mode change 100755 => 100644 var/spack/packages/scorep/package.py mode change 100755 => 100644 var/spack/packages/scotch/package.py mode change 100755 => 100644 var/spack/packages/scr/package.py mode change 100755 => 100644 var/spack/packages/silo/package.py mode change 100755 => 100644 var/spack/packages/spindle/package.py mode change 100755 => 100644 var/spack/packages/sqlite/package.py mode change 100755 => 100644 var/spack/packages/stat/package.py mode change 100755 => 100644 var/spack/packages/sundials/package.py mode change 100755 => 100644 var/spack/packages/swig/package.py mode change 100755 => 100644 var/spack/packages/task/package.py mode change 100755 => 100644 var/spack/packages/taskd/package.py mode change 100755 => 100644 var/spack/packages/tau/package.py mode change 100755 => 100644 var/spack/packages/tcl/package.py mode change 100755 => 100644 var/spack/packages/the_silver_searcher/package.py mode change 100755 => 100644 var/spack/packages/thrift/package.py mode change 100755 => 100644 var/spack/packages/tk/package.py mode change 100755 => 100644 var/spack/packages/tmux/package.py mode change 100755 => 100644 var/spack/packages/tmuxinator/package.py mode change 100755 => 100644 var/spack/packages/uncrustify/package.py mode change 100755 => 100644 var/spack/packages/util-linux/package.py mode change 100755 => 100644 var/spack/packages/vim/package.py mode change 100755 => 100644 var/spack/packages/vtk/package.py mode change 100755 => 100644 var/spack/packages/wget/package.py mode change 100755 => 100644 var/spack/packages/wx/package.py mode change 100755 => 100644 var/spack/packages/wxpropgrid/package.py mode change 100755 => 100644 var/spack/packages/xcb-proto/package.py mode change 100755 => 100644 var/spack/packages/xz/package.py mode change 100755 => 100644 var/spack/packages/yasm/package.py mode change 100755 => 100644 var/spack/packages/zlib/package.py mode change 100755 => 100644 var/spack/packages/zsh/package.py diff --git a/var/spack/packages/ImageMagick/package.py b/var/spack/packages/ImageMagick/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/Mitos/package.py b/var/spack/packages/Mitos/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/R/package.py b/var/spack/packages/R/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/SAMRAI/package.py b/var/spack/packages/SAMRAI/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/adept-utils/package.py b/var/spack/packages/adept-utils/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/adios/package.py b/var/spack/packages/adios/package.py index 224c3b37d5..260dcbe851 100644 --- a/var/spack/packages/adios/package.py +++ b/var/spack/packages/adios/package.py @@ -18,12 +18,13 @@ class Adios(Package): # module load cray-netcdf/4.3.3.1 # module load cray-hdf5/1.8.14 # module load python/2.7.10 + depends_on('hdf5') depends_on('mxml') - + def install(self, spec, prefix): configure_args = ["--prefix=%s" % prefix, "--with-mxml=%s" % spec['mxml'].prefix, - "--with-hdf5=%s" % os.environ["HDF5_DIR"], + "--with-hdf5=%s" % spec['hdf5'].prefix, "--with-netcdf=%s" % os.environ["NETCDF_DIR"], "--with-infiniband=no", "MPICC=cc","MPICXX=CC","MPIFC=ftn", diff --git a/var/spack/packages/arpack/package.py b/var/spack/packages/arpack/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/atk/package.py b/var/spack/packages/atk/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/atlas/package.py b/var/spack/packages/atlas/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/autoconf/package.py b/var/spack/packages/autoconf/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/automaded/package.py b/var/spack/packages/automaded/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/automake/package.py b/var/spack/packages/automake/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/bear/package.py b/var/spack/packages/bear/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/bib2xhtml/package.py b/var/spack/packages/bib2xhtml/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/binutils/package.py b/var/spack/packages/binutils/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/bison/package.py b/var/spack/packages/bison/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/boost/package.py b/var/spack/packages/boost/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/boxlib/package.py b/var/spack/packages/boxlib/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/bzip2/package.py b/var/spack/packages/bzip2/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/cairo/package.py b/var/spack/packages/cairo/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/callpath/package.py b/var/spack/packages/callpath/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/cblas/package.py b/var/spack/packages/cblas/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/cgm/package.py b/var/spack/packages/cgm/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/clang/package.py b/var/spack/packages/clang/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/cloog/package.py b/var/spack/packages/cloog/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/cmake/package.py b/var/spack/packages/cmake/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/coreutils/package.py b/var/spack/packages/coreutils/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/cppcheck/package.py b/var/spack/packages/cppcheck/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/cram/package.py b/var/spack/packages/cram/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/cube/package.py b/var/spack/packages/cube/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/dbus/package.py b/var/spack/packages/dbus/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/dri2proto/package.py b/var/spack/packages/dri2proto/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/dtcmp/package.py b/var/spack/packages/dtcmp/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/dyninst/package.py b/var/spack/packages/dyninst/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/extrae/package.py b/var/spack/packages/extrae/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/exuberant-ctags/package.py b/var/spack/packages/exuberant-ctags/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/flex/package.py b/var/spack/packages/flex/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/fontconfig/package.py b/var/spack/packages/fontconfig/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/freetype/package.py b/var/spack/packages/freetype/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/gasnet/package.py b/var/spack/packages/gasnet/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/gcc/package.py b/var/spack/packages/gcc/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/gdk-pixbuf/package.py b/var/spack/packages/gdk-pixbuf/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/geos/package.py b/var/spack/packages/geos/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/git/package.py b/var/spack/packages/git/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/glib/package.py b/var/spack/packages/glib/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/global/package.py b/var/spack/packages/global/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/gmp/package.py b/var/spack/packages/gmp/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/gnutls/package.py b/var/spack/packages/gnutls/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/gperf/package.py b/var/spack/packages/gperf/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/gperftools/package.py b/var/spack/packages/gperftools/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/graphlib/package.py b/var/spack/packages/graphlib/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/gtkplus/package.py b/var/spack/packages/gtkplus/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/harfbuzz/package.py b/var/spack/packages/harfbuzz/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/hdf5/package.py b/var/spack/packages/hdf5/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/hwloc/package.py b/var/spack/packages/hwloc/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/hypre/package.py b/var/spack/packages/hypre/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/icu/package.py b/var/spack/packages/icu/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/icu4c/package.py b/var/spack/packages/icu4c/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/isl/package.py b/var/spack/packages/isl/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/jdk/package.py b/var/spack/packages/jdk/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/jpeg/package.py b/var/spack/packages/jpeg/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/lapack/package.py b/var/spack/packages/lapack/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/launchmon/package.py b/var/spack/packages/launchmon/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/lcms/package.py b/var/spack/packages/lcms/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/libNBC/package.py b/var/spack/packages/libNBC/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/libarchive/package.py b/var/spack/packages/libarchive/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/libcircle/package.py b/var/spack/packages/libcircle/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/libdrm/package.py b/var/spack/packages/libdrm/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/libdwarf/package.py b/var/spack/packages/libdwarf/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/libelf/package.py b/var/spack/packages/libelf/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/libevent/package.py b/var/spack/packages/libevent/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/libffi/package.py b/var/spack/packages/libffi/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/libgcrypt/package.py b/var/spack/packages/libgcrypt/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/libgpg-error/package.py b/var/spack/packages/libgpg-error/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/libjpeg-turbo/package.py b/var/spack/packages/libjpeg-turbo/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/libmng/package.py b/var/spack/packages/libmng/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/libmonitor/package.py b/var/spack/packages/libmonitor/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/libpciaccess/package.py b/var/spack/packages/libpciaccess/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/libpng/package.py b/var/spack/packages/libpng/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/libtiff/package.py b/var/spack/packages/libtiff/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/libtool/package.py b/var/spack/packages/libtool/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/libunwind/package.py b/var/spack/packages/libunwind/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/libuuid/package.py b/var/spack/packages/libuuid/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/libxcb/package.py b/var/spack/packages/libxcb/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/libxml2/package.py b/var/spack/packages/libxml2/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/libxshmfence/package.py b/var/spack/packages/libxshmfence/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/libxslt/package.py b/var/spack/packages/libxslt/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/llvm-lld/package.py b/var/spack/packages/llvm-lld/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/llvm/package.py b/var/spack/packages/llvm/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/lwgrp/package.py b/var/spack/packages/lwgrp/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/lwm2/package.py b/var/spack/packages/lwm2/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/memaxes/package.py b/var/spack/packages/memaxes/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/mesa/package.py b/var/spack/packages/mesa/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/metis/package.py b/var/spack/packages/metis/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/mpc/package.py b/var/spack/packages/mpc/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/mpe2/package.py b/var/spack/packages/mpe2/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/mpfr/package.py b/var/spack/packages/mpfr/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/mpibash/package.py b/var/spack/packages/mpibash/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/mpich/package.py b/var/spack/packages/mpich/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/mpileaks/package.py b/var/spack/packages/mpileaks/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/mrnet/package.py b/var/spack/packages/mrnet/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/muster/package.py b/var/spack/packages/muster/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/mvapich2/package.py b/var/spack/packages/mvapich2/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/nasm/package.py b/var/spack/packages/nasm/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/ncurses/package.py b/var/spack/packages/ncurses/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/netcdf/package.py b/var/spack/packages/netcdf/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/netgauge/package.py b/var/spack/packages/netgauge/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/netlib-blas/package.py b/var/spack/packages/netlib-blas/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/nettle/package.py b/var/spack/packages/nettle/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/ompss/package.py b/var/spack/packages/ompss/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/opari2/package.py b/var/spack/packages/opari2/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/openmpi/package.py b/var/spack/packages/openmpi/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/openssl/package.py b/var/spack/packages/openssl/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/otf/package.py b/var/spack/packages/otf/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/otf2/package.py b/var/spack/packages/otf2/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/pango/package.py b/var/spack/packages/pango/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/papi/package.py b/var/spack/packages/papi/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/paraver/package.py b/var/spack/packages/paraver/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/parmetis/package.py b/var/spack/packages/parmetis/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/parpack/package.py b/var/spack/packages/parpack/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/pcre/package.py b/var/spack/packages/pcre/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/petsc/package.py b/var/spack/packages/petsc/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/pixman/package.py b/var/spack/packages/pixman/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/pmgr_collective/package.py b/var/spack/packages/pmgr_collective/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/postgresql/package.py b/var/spack/packages/postgresql/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/ppl/package.py b/var/spack/packages/ppl/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-basemap/package.py b/var/spack/packages/py-basemap/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-biopython/package.py b/var/spack/packages/py-biopython/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-cffi/package.py b/var/spack/packages/py-cffi/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-cython/package.py b/var/spack/packages/py-cython/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-dateutil/package.py b/var/spack/packages/py-dateutil/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-epydoc/package.py b/var/spack/packages/py-epydoc/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-genders/package.py b/var/spack/packages/py-genders/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-gnuplot/package.py b/var/spack/packages/py-gnuplot/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-h5py/package.py b/var/spack/packages/py-h5py/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-ipython/package.py b/var/spack/packages/py-ipython/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-jsonschema/package.py b/var/spack/packages/py-jsonschema/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-libxml2/package.py b/var/spack/packages/py-libxml2/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-lockfile/package.py b/var/spack/packages/py-lockfile/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-mako/package.py b/var/spack/packages/py-mako/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-matplotlib/package.py b/var/spack/packages/py-matplotlib/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-mpi4py/package.py b/var/spack/packages/py-mpi4py/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-mx/package.py b/var/spack/packages/py-mx/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-nose/package.py b/var/spack/packages/py-nose/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-numpy/package.py b/var/spack/packages/py-numpy/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-pandas/package.py b/var/spack/packages/py-pandas/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-pexpect/package.py b/var/spack/packages/py-pexpect/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-pil/package.py b/var/spack/packages/py-pil/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-pmw/package.py b/var/spack/packages/py-pmw/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-py4j/package.py b/var/spack/packages/py-py4j/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-pychecker/package.py b/var/spack/packages/py-pychecker/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-pycparser/package.py b/var/spack/packages/py-pycparser/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-pyelftools/package.py b/var/spack/packages/py-pyelftools/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-pygments/package.py b/var/spack/packages/py-pygments/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-pylint/package.py b/var/spack/packages/py-pylint/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-pypar/package.py b/var/spack/packages/py-pypar/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-pyparsing/package.py b/var/spack/packages/py-pyparsing/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-pyqt/package.py b/var/spack/packages/py-pyqt/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-pyside/package.py b/var/spack/packages/py-pyside/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-python-daemon/package.py b/var/spack/packages/py-python-daemon/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-pytz/package.py b/var/spack/packages/py-pytz/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-pyzmq/package.py b/var/spack/packages/py-pyzmq/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-rpy2/package.py b/var/spack/packages/py-rpy2/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-scientificpython/package.py b/var/spack/packages/py-scientificpython/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-scikit-learn/package.py b/var/spack/packages/py-scikit-learn/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-scipy/package.py b/var/spack/packages/py-scipy/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-setuptools/package.py b/var/spack/packages/py-setuptools/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-shiboken/package.py b/var/spack/packages/py-shiboken/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-sip/package.py b/var/spack/packages/py-sip/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-six/package.py b/var/spack/packages/py-six/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-sympy/package.py b/var/spack/packages/py-sympy/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-tornado/package.py b/var/spack/packages/py-tornado/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-traitlets/package.py b/var/spack/packages/py-traitlets/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-virtualenv/package.py b/var/spack/packages/py-virtualenv/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-yapf/package.py b/var/spack/packages/py-yapf/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/python/package.py b/var/spack/packages/python/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/qhull/package.py b/var/spack/packages/qhull/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/qt/package.py b/var/spack/packages/qt/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/qthreads/package.py b/var/spack/packages/qthreads/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/ravel/package.py b/var/spack/packages/ravel/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/readline/package.py b/var/spack/packages/readline/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/rose/package.py b/var/spack/packages/rose/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/ruby/package.py b/var/spack/packages/ruby/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/scalasca/package.py b/var/spack/packages/scalasca/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/scorep/package.py b/var/spack/packages/scorep/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/scotch/package.py b/var/spack/packages/scotch/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/scr/package.py b/var/spack/packages/scr/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/silo/package.py b/var/spack/packages/silo/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/spindle/package.py b/var/spack/packages/spindle/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/sqlite/package.py b/var/spack/packages/sqlite/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/stat/package.py b/var/spack/packages/stat/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/sundials/package.py b/var/spack/packages/sundials/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/swig/package.py b/var/spack/packages/swig/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/task/package.py b/var/spack/packages/task/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/taskd/package.py b/var/spack/packages/taskd/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/tau/package.py b/var/spack/packages/tau/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/tcl/package.py b/var/spack/packages/tcl/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/the_silver_searcher/package.py b/var/spack/packages/the_silver_searcher/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/thrift/package.py b/var/spack/packages/thrift/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/tk/package.py b/var/spack/packages/tk/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/tmux/package.py b/var/spack/packages/tmux/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/tmuxinator/package.py b/var/spack/packages/tmuxinator/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/uncrustify/package.py b/var/spack/packages/uncrustify/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/util-linux/package.py b/var/spack/packages/util-linux/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/vim/package.py b/var/spack/packages/vim/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/vtk/package.py b/var/spack/packages/vtk/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/wget/package.py b/var/spack/packages/wget/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/wx/package.py b/var/spack/packages/wx/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/wxpropgrid/package.py b/var/spack/packages/wxpropgrid/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/xcb-proto/package.py b/var/spack/packages/xcb-proto/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/xz/package.py b/var/spack/packages/xz/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/yasm/package.py b/var/spack/packages/yasm/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/zlib/package.py b/var/spack/packages/zlib/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/zsh/package.py b/var/spack/packages/zsh/package.py old mode 100755 new mode 100644 From d2da41b0bd21d4150ec1e161576efb1e356db1cb Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 8 Jan 2016 12:12:03 -0800 Subject: [PATCH 087/481] More file permissions chasing --- var/spack/packages/SAMRAI/no-tool-build.patch | 0 var/spack/packages/launchmon/patch.lmon_install_dir | 0 var/spack/packages/mpe2/mpe2.patch | 0 var/spack/packages/mpibash/mpibash-4.3.patch | 0 var/spack/packages/mvapich2/ad_lustre_rwcontig_open_source.patch | 0 var/spack/packages/openmpi/ad_lustre_rwcontig_open_source.patch | 0 var/spack/packages/openmpi/llnl-platforms.patch | 0 var/spack/packages/rose/add_spack_compiler_recognition.patch | 0 var/spack/packages/stat/configure_mpicxx.patch | 0 9 files changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 var/spack/packages/SAMRAI/no-tool-build.patch mode change 100755 => 100644 var/spack/packages/launchmon/patch.lmon_install_dir mode change 100755 => 100644 var/spack/packages/mpe2/mpe2.patch mode change 100755 => 100644 var/spack/packages/mpibash/mpibash-4.3.patch mode change 100755 => 100644 var/spack/packages/mvapich2/ad_lustre_rwcontig_open_source.patch mode change 100755 => 100644 var/spack/packages/openmpi/ad_lustre_rwcontig_open_source.patch mode change 100755 => 100644 var/spack/packages/openmpi/llnl-platforms.patch mode change 100755 => 100644 var/spack/packages/rose/add_spack_compiler_recognition.patch mode change 100755 => 100644 var/spack/packages/stat/configure_mpicxx.patch diff --git a/var/spack/packages/SAMRAI/no-tool-build.patch b/var/spack/packages/SAMRAI/no-tool-build.patch old mode 100755 new mode 100644 diff --git a/var/spack/packages/launchmon/patch.lmon_install_dir b/var/spack/packages/launchmon/patch.lmon_install_dir old mode 100755 new mode 100644 diff --git a/var/spack/packages/mpe2/mpe2.patch b/var/spack/packages/mpe2/mpe2.patch old mode 100755 new mode 100644 diff --git a/var/spack/packages/mpibash/mpibash-4.3.patch b/var/spack/packages/mpibash/mpibash-4.3.patch old mode 100755 new mode 100644 diff --git a/var/spack/packages/mvapich2/ad_lustre_rwcontig_open_source.patch b/var/spack/packages/mvapich2/ad_lustre_rwcontig_open_source.patch old mode 100755 new mode 100644 diff --git a/var/spack/packages/openmpi/ad_lustre_rwcontig_open_source.patch b/var/spack/packages/openmpi/ad_lustre_rwcontig_open_source.patch old mode 100755 new mode 100644 diff --git a/var/spack/packages/openmpi/llnl-platforms.patch b/var/spack/packages/openmpi/llnl-platforms.patch old mode 100755 new mode 100644 diff --git a/var/spack/packages/rose/add_spack_compiler_recognition.patch b/var/spack/packages/rose/add_spack_compiler_recognition.patch old mode 100755 new mode 100644 diff --git a/var/spack/packages/stat/configure_mpicxx.patch b/var/spack/packages/stat/configure_mpicxx.patch old mode 100755 new mode 100644 From 1b0d06474359797d4f9e0c59aa813e0e607f0e5a Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 8 Jan 2016 14:13:33 -0800 Subject: [PATCH 088/481] Revert "clean up package.py file and remove duplicate file" This reverts commit c809cc92735c7ff57dab2aca02bc7fa40c901a43. --- var/spack/packages/py-ipython/package.py | 4 +++ .../packages/py-ipython/package.py.backup | 27 +++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100755 var/spack/packages/py-ipython/package.py.backup diff --git a/var/spack/packages/py-ipython/package.py b/var/spack/packages/py-ipython/package.py index 225366552a..3b94b71f29 100644 --- a/var/spack/packages/py-ipython/package.py +++ b/var/spack/packages/py-ipython/package.py @@ -24,4 +24,8 @@ class PyIpython(Package): depends_on('py-jinja2') def install(self, spec, prefix): + import os + # os.system("spack activate py-setuptools@18.1") + # os.system("easy_install pip") + # os.system("pip install ipython[notebook]") python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-ipython/package.py.backup b/var/spack/packages/py-ipython/package.py.backup new file mode 100755 index 0000000000..1ed8b57e60 --- /dev/null +++ b/var/spack/packages/py-ipython/package.py.backup @@ -0,0 +1,27 @@ +from spack import * + +class PyIpython(Package): + """IPython provides a rich toolkit to help you make the most out of using Python interactively.""" + homepage = "https://pypi.python.org/pypi/ipython" + + version('2.3.1', '2b7085525dac11190bfb45bb8ec8dcbf', + url="https://pypi.python.org/packages/source/i/ipython/ipython-2.3.1.tar.gz") + version('3.1.0', 'a749d90c16068687b0ec45a27e72ef8f') + version('3.2.1', 'f4c93d67ac4b2d4fc69df693b6f3c9e0', + url='https://github.com/ipython/ipython/archive/rel-3.2.1.tar.gz') + + extends('python') + depends_on('py-pygments') + depends_on('py-setuptools@18.1') + depends_on('py-traitlets') + depends_on('py-pyzmq') + depends_on('py-pexpect') + depends_on('py-tornado') + depends_on('py-Jinja2') + + def install(self, spec, prefix): + import os + # os.system("spack activate py-setuptools@18.1") + # os.system("easy_install pip") + # os.system("pip install ipython[notebook]") + python('setup.py', 'install', '--prefix=%s' % prefix) From a9e9f9afb04394ddb85ad197fa7b19999ecc3b32 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 8 Jan 2016 14:14:26 -0800 Subject: [PATCH 089/481] Revert "removed package for python packages with problematic / duplicate names" This reverts commit f3254ff02d589c9e177aab58276f5c1971866018. --- var/spack/packages/py-Cython/package.py | 12 ++++++++++++ var/spack/packages/py-Distutils2/package.py | 15 +++++++++++++++ 2 files changed, 27 insertions(+) create mode 100644 var/spack/packages/py-Cython/package.py create mode 100644 var/spack/packages/py-Distutils2/package.py diff --git a/var/spack/packages/py-Cython/package.py b/var/spack/packages/py-Cython/package.py new file mode 100644 index 0000000000..c250139114 --- /dev/null +++ b/var/spack/packages/py-Cython/package.py @@ -0,0 +1,12 @@ +from spack import * + +class PyCython(Package): + """The Cython compiler for writing C extensions for the Python language.""" + homepage = "http://www.cython.org" + version("0.21.2", "d21adb870c75680dc857cd05d41046a4", + url="https://pypi.python.org/packages/source/C/Cython/Cython-0.21.2.tar.gz") + + extends("python") + + def install(self, spec, prefix): + python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-Distutils2/package.py b/var/spack/packages/py-Distutils2/package.py new file mode 100644 index 0000000000..8690f569fe --- /dev/null +++ b/var/spack/packages/py-Distutils2/package.py @@ -0,0 +1,15 @@ +from spack import * + +class PyDistutils2(Package): + """Python Packaging Library.""" + + homepage = "https://hg.python.org/distutils2" + url = "https://pypi.python.org/packages/source/D/Distutils2/Distutils2-1.0a4.tar.gz" + + version('1.0a4', '52bc9dffb394970c27e02853ae3a3241') + + depends_on("python") + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) + From d3d37ad0cebe4d42d48f5f5f5a53077ae6126631 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 8 Jan 2016 14:29:01 -0800 Subject: [PATCH 090/481] removed the submodule as part of reversion of extraneous commits --- lib/spack/spack/util/python_recipe_parser | 1 - 1 file changed, 1 deletion(-) delete mode 160000 lib/spack/spack/util/python_recipe_parser diff --git a/lib/spack/spack/util/python_recipe_parser b/lib/spack/spack/util/python_recipe_parser deleted file mode 160000 index 437a62abb3..0000000000 --- a/lib/spack/spack/util/python_recipe_parser +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 437a62abb3df7212e3ee20269c0089a0a9766fe0 From 86230b9ac8ed42f3f3b309a0a8a77483978a306c Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 8 Jan 2016 15:01:46 -0800 Subject: [PATCH 091/481] Revert "make sure submodule is added in the correct location" This reverts commit e9f7d033ff04138905c99cb427e2071cc55cdc16. --- .gitmodules | 6 +++--- var/spack/packages/py-astropy/package.py | 1 - var/spack/packages/py-jsonschema/package.py | 2 -- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/.gitmodules b/.gitmodules index d5995e8e4b..1d770f415b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ -[submodule "lib/spack/spack/util/python_recipe_parser"] - path = lib/spack/spack/util/python_recipe_parser - url = git@github.com:karenyyng/spack_python_package_parser.git +[submodule "/lib/spack/spack/util/python_recipe_parser"] + path = /lib/spack/spack/util/python_recipe_parser + url = https://github.com/karenyyng/spack_python_package_parser diff --git a/var/spack/packages/py-astropy/package.py b/var/spack/packages/py-astropy/package.py index 9fda37519e..5d2d0619cc 100644 --- a/var/spack/packages/py-astropy/package.py +++ b/var/spack/packages/py-astropy/package.py @@ -7,7 +7,6 @@ class PyAstropy(Package): url="https://pypi.python.org/packages/source/a/astropy/astropy-0.4.2.tar.gz") extends("python") - depends_on("py-numpy") def install(self, spec, prefix): python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-jsonschema/package.py b/var/spack/packages/py-jsonschema/package.py index fa233635ce..9a7fc21862 100644 --- a/var/spack/packages/py-jsonschema/package.py +++ b/var/spack/packages/py-jsonschema/package.py @@ -11,8 +11,6 @@ class PyJsonschema(Package): extends("python") depends_on("py-setuptools@18.1") - depends_on("py-vcversioner") - depends_on("py-functools32") def install(self, spec, prefix): python('setup.py', 'install', '--prefix=%s' % prefix) From 6ca1e5c4b07e1677363cf0f3787144f67af688c8 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 8 Jan 2016 15:02:29 -0800 Subject: [PATCH 092/481] Revert "added a bunch of packages and added submodule" This reverts commit 79808da760eabcb3e692c88e29ea3c7c558b131d. --- .gitignore | 1 - .gitmodules | 4 +- utils/python_recipe_parser | 1 + var/spack/packages/asciidoc/package.py | 18 +++++++++ var/spack/packages/boost/package.py | 4 ++ var/spack/packages/bowtie2/bowtie2-2.5.patch | 16 ++++++++ var/spack/packages/bowtie2/package.py | 24 ++++++++++++ var/spack/packages/cscope/package.py | 17 ++++++++ var/spack/packages/czmq/package.py | 19 +++++++++ var/spack/packages/docbook-xml/package.py | 19 +++++++++ var/spack/packages/doxygen/package.py | 25 ++++++++++++ var/spack/packages/eigen/package.py | 17 -------- var/spack/packages/elfutils/package.py | 26 +++++++++++++ var/spack/packages/fish/package.py | 18 +++++++++ var/spack/packages/flux/package.py | 36 +++++++++++++++++ var/spack/packages/gflags/package.py | 21 ++++++++++ var/spack/packages/ghostscript/package.py | 17 ++++++++ var/spack/packages/glog/package.py | 15 +++++++ var/spack/packages/graphviz/package.py | 19 ++++----- var/spack/packages/leveldb/package.py | 29 ++++++++++++++ var/spack/packages/libdrm/package.py | 5 +-- var/spack/packages/libelf/package.py | 2 + var/spack/packages/libffi/package.py | 6 +-- var/spack/packages/libjson-c/package.py | 14 +++++++ var/spack/packages/libpciaccess/package.py | 19 +++++---- var/spack/packages/libsodium/package.py | 19 +++++++++ var/spack/packages/libxml2/package.py | 5 ++- var/spack/packages/lmdb/package.py | 39 +++++++++++++++++++ var/spack/packages/lua/package.py | 26 +++++++++++++ var/spack/packages/mesa/package.py | 5 ++- var/spack/packages/mpich/package.py | 1 - var/spack/packages/munge/package.py | 20 ++++++++++ var/spack/packages/ncurses/package.py | 8 ++++ var/spack/packages/netlib-blas/package.py | 6 +++ .../{lapack => netlib-lapack}/package.py | 18 ++++++++- var/spack/packages/pidx/package.py | 21 ++++++++++ var/spack/packages/pkg-config/package.py | 17 ++++++++ var/spack/packages/protobuf/package.py | 16 ++++++++ var/spack/packages/py-Cython/package.py | 12 ------ var/spack/packages/py-Distutils2/package.py | 15 ------- var/spack/packages/py-astroML/package.py | 12 ------ var/spack/packages/py-astropy/package.py | 12 ------ var/spack/packages/py-autopep8/package.py | 12 ------ var/spack/packages/py-certifi/package.py | 12 ------ var/spack/packages/py-decorator/package.py | 12 ------ var/spack/packages/py-emcee/package.py | 12 ------ var/spack/packages/py-fitsio/package.py | 13 ------- var/spack/packages/py-gnureadline/package.py | 12 ------ var/spack/packages/py-ipython/package.py | 21 ++-------- .../packages/py-ipython/package.py.backup | 27 ------------- .../packages/py-ipython_genutils/package.py | 12 ------ var/spack/packages/py-joblib/package.py | 12 ------ var/spack/packages/py-jsonschema/package.py | 16 -------- var/spack/packages/py-matplotlib/package.py | 1 + var/spack/packages/py-mock/package.py | 17 +++++--- var/spack/packages/py-numexpr/package.py | 12 ------ var/spack/packages/py-numpy/package.py | 15 ++++++- var/spack/packages/py-pandas/package.py | 2 - var/spack/packages/py-pep8/package.py | 12 ------ var/spack/packages/py-pickleshare/package.py | 12 ------ var/spack/packages/py-pip/package.py | 12 ------ var/spack/packages/py-py/package.py | 12 ------ var/spack/packages/py-py4j/package.py | 15 ------- var/spack/packages/py-pyside/package.py | 5 ++- var/spack/packages/py-pytest/package.py | 12 ------ .../packages/py-python-dateutil/package.py | 12 ------ var/spack/packages/py-pytz/package.py | 2 - var/spack/packages/py-pyzmq/package.py | 16 -------- var/spack/packages/py-scikit-learn/package.py | 3 -- var/spack/packages/py-scipy/package.py | 4 +- var/spack/packages/py-seaborn/package.py | 12 ------ var/spack/packages/py-setuptools/package.py | 9 ++--- var/spack/packages/py-six/package.py | 2 - var/spack/packages/py-statsmodels/package.py | 12 ------ var/spack/packages/py-tables/package.py | 12 ------ var/spack/packages/py-tornado/package.py | 17 -------- var/spack/packages/py-traitlets/package.py | 14 ------- .../packages/py-triangle-plot/package.py | 12 ------ var/spack/packages/samtools/package.py | 18 +++++++++ var/spack/packages/samtools/samtools1.2.patch | 20 ++++++++++ var/spack/packages/scorep/package.py | 15 ++++--- var/spack/packages/snappy/package.py | 15 +++++++ var/spack/packages/stat/package.py | 11 +++++- var/spack/packages/vim/package.py | 23 +++++++++-- var/spack/packages/zeromq/package.py | 20 ++++++++++ 85 files changed, 689 insertions(+), 492 deletions(-) create mode 160000 utils/python_recipe_parser create mode 100644 var/spack/packages/asciidoc/package.py create mode 100644 var/spack/packages/bowtie2/bowtie2-2.5.patch create mode 100644 var/spack/packages/bowtie2/package.py create mode 100644 var/spack/packages/cscope/package.py create mode 100644 var/spack/packages/czmq/package.py create mode 100644 var/spack/packages/docbook-xml/package.py create mode 100644 var/spack/packages/doxygen/package.py delete mode 100644 var/spack/packages/eigen/package.py create mode 100644 var/spack/packages/elfutils/package.py create mode 100644 var/spack/packages/fish/package.py create mode 100644 var/spack/packages/flux/package.py create mode 100644 var/spack/packages/gflags/package.py create mode 100644 var/spack/packages/ghostscript/package.py create mode 100644 var/spack/packages/glog/package.py create mode 100644 var/spack/packages/leveldb/package.py create mode 100644 var/spack/packages/libjson-c/package.py create mode 100644 var/spack/packages/libsodium/package.py create mode 100644 var/spack/packages/lmdb/package.py create mode 100644 var/spack/packages/lua/package.py create mode 100644 var/spack/packages/munge/package.py rename var/spack/packages/{lapack => netlib-lapack}/package.py (77%) create mode 100644 var/spack/packages/pidx/package.py create mode 100644 var/spack/packages/pkg-config/package.py create mode 100644 var/spack/packages/protobuf/package.py delete mode 100644 var/spack/packages/py-Cython/package.py delete mode 100644 var/spack/packages/py-Distutils2/package.py delete mode 100644 var/spack/packages/py-astroML/package.py delete mode 100644 var/spack/packages/py-astropy/package.py delete mode 100644 var/spack/packages/py-autopep8/package.py delete mode 100644 var/spack/packages/py-certifi/package.py delete mode 100644 var/spack/packages/py-decorator/package.py delete mode 100644 var/spack/packages/py-emcee/package.py delete mode 100644 var/spack/packages/py-fitsio/package.py delete mode 100644 var/spack/packages/py-gnureadline/package.py delete mode 100755 var/spack/packages/py-ipython/package.py.backup delete mode 100644 var/spack/packages/py-ipython_genutils/package.py delete mode 100644 var/spack/packages/py-joblib/package.py delete mode 100644 var/spack/packages/py-jsonschema/package.py delete mode 100644 var/spack/packages/py-numexpr/package.py delete mode 100644 var/spack/packages/py-pep8/package.py delete mode 100644 var/spack/packages/py-pickleshare/package.py delete mode 100644 var/spack/packages/py-pip/package.py delete mode 100644 var/spack/packages/py-py/package.py delete mode 100644 var/spack/packages/py-py4j/package.py delete mode 100644 var/spack/packages/py-pytest/package.py delete mode 100644 var/spack/packages/py-python-dateutil/package.py delete mode 100644 var/spack/packages/py-pyzmq/package.py delete mode 100644 var/spack/packages/py-seaborn/package.py delete mode 100644 var/spack/packages/py-statsmodels/package.py delete mode 100644 var/spack/packages/py-tables/package.py delete mode 100644 var/spack/packages/py-tornado/package.py delete mode 100644 var/spack/packages/py-traitlets/package.py delete mode 100644 var/spack/packages/py-triangle-plot/package.py create mode 100644 var/spack/packages/samtools/package.py create mode 100644 var/spack/packages/samtools/samtools1.2.patch create mode 100644 var/spack/packages/snappy/package.py create mode 100644 var/spack/packages/zeromq/package.py diff --git a/.gitignore b/.gitignore index 8146921fd8..1c6ca4c99e 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,3 @@ /etc/spackconfig /share/spack/dotkit /share/spack/modules -*.ropeproject diff --git a/.gitmodules b/.gitmodules index 1d770f415b..a097fcd8cf 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ -[submodule "/lib/spack/spack/util/python_recipe_parser"] - path = /lib/spack/spack/util/python_recipe_parser +[submodule "utils/python_recipe_parser"] + path = utils/python_recipe_parser url = https://github.com/karenyyng/spack_python_package_parser diff --git a/utils/python_recipe_parser b/utils/python_recipe_parser new file mode 160000 index 0000000000..3158074a2d --- /dev/null +++ b/utils/python_recipe_parser @@ -0,0 +1 @@ +Subproject commit 3158074a2d74f99aa72289dfcf093b1b0ba4474e diff --git a/var/spack/packages/asciidoc/package.py b/var/spack/packages/asciidoc/package.py new file mode 100644 index 0000000000..828f3b3f4f --- /dev/null +++ b/var/spack/packages/asciidoc/package.py @@ -0,0 +1,18 @@ +from spack import * + +class Asciidoc(Package): + """ A presentable text document format for writing articles, UNIX man + pages and other small to medium sized documents.""" + homepage = "http://asciidoc.org" + url = "http://downloads.sourceforge.net/project/asciidoc/asciidoc/8.6.9/asciidoc-8.6.9.tar.gz" + + version('8.6.9', 'c59018f105be8d022714b826b0be130a') + + depends_on('libxml2') + depends_on('libxslt') + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix) + + make() + make("install") diff --git a/var/spack/packages/boost/package.py b/var/spack/packages/boost/package.py index 254d1afda1..35824d53a2 100644 --- a/var/spack/packages/boost/package.py +++ b/var/spack/packages/boost/package.py @@ -14,6 +14,10 @@ class Boost(Package): list_url = "http://sourceforge.net/projects/boost/files/boost/" list_depth = 2 + version('1.59.0', '6aa9a5c6a4ca1016edd0ed1178e3cb87') + version('1.58.0', 'b8839650e61e9c1c0a89f371dd475546') + version('1.57.0', '1be49befbdd9a5ce9def2983ba3e7b76') + version('1.56.0', 'a744cf167b05d72335f27c88115f211d') version('1.55.0', 'd6eef4b4cacb2183f2bf265a5a03a354') version('1.54.0', '15cb8c0803064faef0c4ddf5bc5ca279') version('1.53.0', 'a00d22605d5dbcfb4c9936a9b35bc4c2') diff --git a/var/spack/packages/bowtie2/bowtie2-2.5.patch b/var/spack/packages/bowtie2/bowtie2-2.5.patch new file mode 100644 index 0000000000..290be39c73 --- /dev/null +++ b/var/spack/packages/bowtie2/bowtie2-2.5.patch @@ -0,0 +1,16 @@ +--- Makefile 2015-02-26 10:50:00.000000000 -0800 ++++ Makefile.new 2015-07-29 18:03:59.891357399 -0700 +@@ -22,10 +22,10 @@ + # + + INC = +-GCC_PREFIX = $(shell dirname `which gcc`) ++GCC_PREFIX = + GCC_SUFFIX = +-CC = $(GCC_PREFIX)/gcc$(GCC_SUFFIX) +-CPP = $(GCC_PREFIX)/g++$(GCC_SUFFIX) ++CC = cc ++CPP = c++ + CXX = $(CPP) + HEADERS = $(wildcard *.h) + BOWTIE_MM = 1 diff --git a/var/spack/packages/bowtie2/package.py b/var/spack/packages/bowtie2/package.py new file mode 100644 index 0000000000..339aab6598 --- /dev/null +++ b/var/spack/packages/bowtie2/package.py @@ -0,0 +1,24 @@ +from spack import * +from glob import glob +class Bowtie2(Package): + """Description""" + homepage = "bowtie-bio.sourceforge.net/bowtie2/index.shtml" + version('2.2.5','51fa97a862d248d7ee660efc1147c75f', url = "http://downloads.sourceforge.net/project/bowtie-bio/bowtie2/2.2.5/bowtie2-2.2.5-source.zip") + + patch('bowtie2-2.5.patch',when='@2.2.5', level=0) + + def install(self, spec, prefix): + make() + mkdirp(prefix.bin) + for bow in glob("bowtie2*"): + install(bow, prefix.bin) + # install('bowtie2',prefix.bin) + # install('bowtie2-align-l',prefix.bin) + # install('bowtie2-align-s',prefix.bin) + # install('bowtie2-build',prefix.bin) + # install('bowtie2-build-l',prefix.bin) + # install('bowtie2-build-s',prefix.bin) + # install('bowtie2-inspect',prefix.bin) + # install('bowtie2-inspect-l',prefix.bin) + # install('bowtie2-inspect-s',prefix.bin) + diff --git a/var/spack/packages/cscope/package.py b/var/spack/packages/cscope/package.py new file mode 100644 index 0000000000..9aac0f7304 --- /dev/null +++ b/var/spack/packages/cscope/package.py @@ -0,0 +1,17 @@ +from spack import * + +class Cscope(Package): + """Cscope is a developer's tool for browsing source code.""" + homepage = "http://http://cscope.sourceforge.net/" + url = "http://downloads.sourceforge.net/project/cscope/cscope/15.8b/cscope-15.8b.tar.gz" + + version('15.8b', '8f9409a238ee313a96f9f87fe0f3b176') + + # Can be configured to use flex (not necessary) + # ./configure --with-flex + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix) + + make() + make("install") diff --git a/var/spack/packages/czmq/package.py b/var/spack/packages/czmq/package.py new file mode 100644 index 0000000000..a2f1947554 --- /dev/null +++ b/var/spack/packages/czmq/package.py @@ -0,0 +1,19 @@ +from spack import * + +class Czmq(Package): + """ A C interface to the ZMQ library """ + homepage = "http://czmq.zeromq.org" + url = "https://github.com/zeromq/czmq/archive/v3.0.2.tar.gz" + + version('3.0.2', '23e9885f7ee3ce88d99d0425f52e9be1', url='https://github.com/zeromq/czmq/archive/v3.0.2.tar.gz') + + depends_on('zeromq') + + def install(self, spec, prefix): + bash = which("bash") + bash("./autogen.sh") + configure("--prefix=%s" % prefix) + + make() + make("install") + diff --git a/var/spack/packages/docbook-xml/package.py b/var/spack/packages/docbook-xml/package.py new file mode 100644 index 0000000000..fce1de7deb --- /dev/null +++ b/var/spack/packages/docbook-xml/package.py @@ -0,0 +1,19 @@ +import os +import glob +from spack import * + + +class DocbookXml(Package): + """Docbook DTD XML files.""" + homepage = "http://www.oasis-open.org/docbook" + url = "http://www.oasis-open.org/docbook/xml/4.5/docbook-xml-4.5.zip" + + version('4.5', '03083e288e87a7e829e437358da7ef9e') + + def install(self, spec, prefix): + cp = which('cp') + + install_args = ['-a', '-t', prefix] + install_args.extend(glob.glob('*')) + + cp(*install_args) diff --git a/var/spack/packages/doxygen/package.py b/var/spack/packages/doxygen/package.py new file mode 100644 index 0000000000..3d4a4e47a7 --- /dev/null +++ b/var/spack/packages/doxygen/package.py @@ -0,0 +1,25 @@ +#------------------------------------------------------------------------------ +# Author: Justin Too +# Date: September 11, 2015 +#------------------------------------------------------------------------------ + +from spack import * + +class Doxygen(Package): + """Doxygen is the de facto standard tool for generating documentation + from annotated C++ sources, but it also supports other popular programming + languages such as C, Objective-C, C#, PHP, Java, Python, IDL (Corba, + Microsoft, and UNO/OpenOffice flavors), Fortran, VHDL, Tcl, and to some extent D.. + """ + homepage = "http://www.stack.nl/~dimitri/doxygen/" + url = "http://ftp.stack.nl/pub/users/dimitri/doxygen-1.8.10.src.tar.gz" + + version('1.8.10', '79767ccd986f12a0f949015efb5f058f') + + depends_on("cmake@2.8.12:") + + def install(self, spec, prefix): + cmake('.', *std_cmake_args) + + make() + make("install") diff --git a/var/spack/packages/eigen/package.py b/var/spack/packages/eigen/package.py deleted file mode 100644 index b4d89ea3a0..0000000000 --- a/var/spack/packages/eigen/package.py +++ /dev/null @@ -1,17 +0,0 @@ -from spack import * - -class Eigen(Package): - """Eigen is a C++ template library for linear algebra: matrices, vectors, - numerical solvers, and related algorithms..""" - homepage = "http://eigen.tuxfamily.org/" - url = "http://bitbucket.org/eigen/eigen/get/3.2.5.tar.gz" - - version('3.2.5', '8cc513ac6ec687117acadddfcacf551b') - - def install(self, spec, prefix): - import os - os.system("mkdir ./build_dir && cd ./build_dir") - cmake('../', *std_cmake_args) - - make() - make("install") diff --git a/var/spack/packages/elfutils/package.py b/var/spack/packages/elfutils/package.py new file mode 100644 index 0000000000..926d234584 --- /dev/null +++ b/var/spack/packages/elfutils/package.py @@ -0,0 +1,26 @@ +from spack import * + +class Elfutils(Package): + """elfutils is a collection of various binary tools such as + eu-objdump, eu-readelf, and other utilities that allow you to + inspect and manipulate ELF files. Refer to Table 5.Tools Included + in elfutils for Red Hat Developer for a complete list of binary + tools that are distributed with the Red Hat Developer Toolset + version of elfutils.""" + + homepage = "https://fedorahosted.org/elfutils/" + + version('0.163', + git='git://git.fedorahosted.org/git/elfutils.git', + tag='elfutils-0.163') + + provides('elf') + + def install(self, spec, prefix): + autoreconf = which('autoreconf') + autoreconf('-if') + + configure('--prefix=%s' % prefix, '--enable-maintainer-mode') + make() + make("install") + diff --git a/var/spack/packages/fish/package.py b/var/spack/packages/fish/package.py new file mode 100644 index 0000000000..1225558705 --- /dev/null +++ b/var/spack/packages/fish/package.py @@ -0,0 +1,18 @@ +from spack import * + +class Fish(Package): + """fish is a smart and user-friendly command line shell for OS X, Linux, and + the rest of the family. + """ + + homepage = "http://fishshell.com/" + url = "http://fishshell.com/files/2.2.0/fish-2.2.0.tar.gz" + list_url = homepage + + version('2.2.0', 'a76339fd14ce2ec229283c53e805faac48c3e99d9e3ede9d82c0554acfc7b77a') + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix) + + make() + make("install") diff --git a/var/spack/packages/flux/package.py b/var/spack/packages/flux/package.py new file mode 100644 index 0000000000..c128f46be8 --- /dev/null +++ b/var/spack/packages/flux/package.py @@ -0,0 +1,36 @@ +from spack import * +import os + +class Flux(Package): + """ A next-generation resource manager (pre-alpha) """ + + homepage = "https://github.com/flux-framework/flux-core" + url = "https://github.com/flux-framework/flux-core" + + version('master', branch='master', git='https://github.com/flux-framework/flux-core') + + # Also needs autotools, but should use the system version if available + depends_on("zeromq@4.0.4:") + depends_on("czmq@2.2:") + depends_on("lua@5.1:5.1.99") + depends_on("munge") + depends_on("libjson-c") + depends_on("libxslt") + # TODO: This provides a catalog, hacked with environment below for now + depends_on("docbook-xml") + depends_on("asciidoc") + depends_on("python") + depends_on("py-cffi") + + def install(self, spec, prefix): + # Bootstrap with autotools + bash = which('bash') + bash('./autogen.sh') + + # Fix asciidoc dependency on xml style sheets and whatnot + os.environ['XML_CATALOG_FILES'] = os.path.join(spec['docbook-xml'].prefix, + 'catalog.xml') + # Configure, compile & install + configure("--prefix=" + prefix) + make("install", "V=1") + diff --git a/var/spack/packages/gflags/package.py b/var/spack/packages/gflags/package.py new file mode 100644 index 0000000000..62dd80a094 --- /dev/null +++ b/var/spack/packages/gflags/package.py @@ -0,0 +1,21 @@ +import os +from spack import * + +class Gflags(Package): + """The gflags package contains a C++ library that implements + commandline flags processing. It includes built-in support for + standard types such as string and the ability to define flags + in the source file in which they are used. Online documentation + available at: https://gflags.github.io/gflags/""" + + homepage = "https://gflags.github.io/gflags" + url = "https://github.com/gflags/gflags/archive/v2.1.2.tar.gz" + + version('2.1.2', 'ac432de923f9de1e9780b5254884599f') + + def install(self, spec, prefix): + cmake("-DCMAKE_INSTALL_PREFIX=" + prefix, + "-DBUILD_SHARED_LIBS=ON") + make() + make("test") + make("install") diff --git a/var/spack/packages/ghostscript/package.py b/var/spack/packages/ghostscript/package.py new file mode 100644 index 0000000000..0ab49d425f --- /dev/null +++ b/var/spack/packages/ghostscript/package.py @@ -0,0 +1,17 @@ +from spack import * + +class Ghostscript(Package): + """an interpreter for the PostScript language and for PDF. """ + homepage = "http://ghostscript.com/" + url = "http://downloads.ghostscript.com/public/ghostscript-9.16.tar.gz" + + version('9.16', '829319325bbdb83f5c81379a8f86f38f') + + parallel = False + + def install(self, spec, prefix): + configure("--prefix=%s" %prefix, "--enable-shared") + + make() + make("install") + diff --git a/var/spack/packages/glog/package.py b/var/spack/packages/glog/package.py new file mode 100644 index 0000000000..d73386b394 --- /dev/null +++ b/var/spack/packages/glog/package.py @@ -0,0 +1,15 @@ +import os +from spack import * + +class Glog(Package): + """C++ implementation of the Google logging module.""" + + homepage = "https://github.com/google/glog" + url = "https://github.com/google/glog/archive/v0.3.3.tar.gz" + + version('0.3.3', 'c1f86af27bd9c73186730aa957607ed0') + + def install(self, spec, prefix): + configure("--prefix=" + prefix) + make() + make("install") diff --git a/var/spack/packages/graphviz/package.py b/var/spack/packages/graphviz/package.py index 8244feccfd..7af7da1881 100644 --- a/var/spack/packages/graphviz/package.py +++ b/var/spack/packages/graphviz/package.py @@ -1,24 +1,21 @@ from spack import * class Graphviz(Package): - """graph visualization software.""" + """Graph Visualization Software""" homepage = "http://www.graphviz.org" url = "http://www.graphviz.org/pub/graphviz/stable/SOURCES/graphviz-2.38.0.tar.gz" version('2.38.0', '5b6a829b2ac94efcd5fa3c223ed6d3ae') - version('2.36.0', '1f41664dba0c93109ac8b71216bf2b57') - depends_on("cairo@1.1.10") - depends_on("freetype@2.1.10") - depends_on("fontconfig") - depends_on("zlib@1.2.3") - # depends_on("libpng@1.2.10") - # depends_on("expat@2.0.0") - # depends_on("gd@2.0.34") + parallel = False + + depends_on("swig") + depends_on("python") + depends_on("ghostscript") def install(self, spec, prefix): - configure('--prefix=%s' % prefix) + configure("--prefix=%s" %prefix) - # FIXME: Add logic to build and install here make() make("install") + diff --git a/var/spack/packages/leveldb/package.py b/var/spack/packages/leveldb/package.py new file mode 100644 index 0000000000..da68a9cbcb --- /dev/null +++ b/var/spack/packages/leveldb/package.py @@ -0,0 +1,29 @@ +import os +import glob +from spack import * + +class Leveldb(Package): + """LevelDB is a fast key-value storage library written at Google + that provides an ordered mapping from string keys to string values.""" + + homepage = "https://github.com/google/leveldb" + url = "https://github.com/google/leveldb/archive/v1.18.tar.gz" + + version('1.18', '73770de34a2a5ab34498d2e05b2b7fa0') + + depends_on("snappy") + + def install(self, spec, prefix): + make() + + mkdirp(prefix.include) + mkdirp(prefix.lib) + + cp = which('cp') + + # cp --preserve=links libleveldb.* prefix/lib + args = glob.glob('libleveldb.*') + args.append(prefix + '/lib') + cp('--preserve=links', *args) + + cp('-r', 'include/leveldb', prefix + '/include') diff --git a/var/spack/packages/libdrm/package.py b/var/spack/packages/libdrm/package.py index 1cab5372be..00736b7811 100644 --- a/var/spack/packages/libdrm/package.py +++ b/var/spack/packages/libdrm/package.py @@ -2,7 +2,7 @@ class Libdrm(Package): """A userspace library for accessing the DRM, direct - rendering manager, on Linux, BSD and other operating + rendering manager, on Linux, BSD and other operating systems that support the ioctl interface.""" homepage = "http://dri.freedesktop.org/libdrm/" # no real website... @@ -11,11 +11,8 @@ class Libdrm(Package): version('2.4.59', '105ac7af1afcd742d402ca7b4eb168b6') version('2.4.33', '86e4e3debe7087d5404461e0032231c8') - depends_on("libpciaccess") - def install(self, spec, prefix): configure("--prefix=%s" % prefix) - make("clean") make() make("install") diff --git a/var/spack/packages/libelf/package.py b/var/spack/packages/libelf/package.py index bf2fefabd5..9338b8f393 100644 --- a/var/spack/packages/libelf/package.py +++ b/var/spack/packages/libelf/package.py @@ -36,6 +36,8 @@ class Libelf(Package): version('0.8.13', '4136d7b4c04df68b686570afa26988ac') version('0.8.12', 'e21f8273d9f5f6d43a59878dc274fec7') + provides('elf') + def install(self, spec, prefix): configure("--prefix=" + prefix, "--enable-shared", diff --git a/var/spack/packages/libffi/package.py b/var/spack/packages/libffi/package.py index 2c1c4eed4d..18acabb00c 100644 --- a/var/spack/packages/libffi/package.py +++ b/var/spack/packages/libffi/package.py @@ -6,9 +6,9 @@ class Libffi(Package): to call any function specified by a call interface description at run time.""" homepage = "https://sourceware.org/libffi/" - url = "ftp://sourceware.org/pub/libffi/libffi-3.1.tar.gz" - - version('3.1', 'f5898b29bbfd70502831a212d9249d10') + + version('3.2.1','83b89587607e3eb65c70d361f13bab43',url = "ftp://sourceware.org/pub/libffi/libffi-3.2.1.tar.gz") + #version('3.1', 'f5898b29bbfd70502831a212d9249d10',url = "ftp://sourceware.org/pub/libffi/libffi-3.1.tar.gz") # Has a bug $(lib64) instead of ${lib64} in libffi.pc def install(self, spec, prefix): configure("--prefix=%s" % prefix) diff --git a/var/spack/packages/libjson-c/package.py b/var/spack/packages/libjson-c/package.py new file mode 100644 index 0000000000..c0801cce9c --- /dev/null +++ b/var/spack/packages/libjson-c/package.py @@ -0,0 +1,14 @@ +from spack import * + +class LibjsonC(Package): + """ A JSON implementation in C """ + homepage = "https://github.com/json-c/json-c/wiki" + url = "https://s3.amazonaws.com/json-c_releases/releases/json-c-0.11.tar.gz" + + version('0.11', 'aa02367d2f7a830bf1e3376f77881e98') + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix) + + make() + make("install") diff --git a/var/spack/packages/libpciaccess/package.py b/var/spack/packages/libpciaccess/package.py index 371fbb6685..6022fc34a3 100644 --- a/var/spack/packages/libpciaccess/package.py +++ b/var/spack/packages/libpciaccess/package.py @@ -1,16 +1,21 @@ from spack import * class Libpciaccess(Package): - """Generic PCI access library""" - homepage = "http://cgit.freedesktop.org/xorg/lib/libpciaccess/" - url = "http://pkgs.fedoraproject.org/repo/pkgs/libpciaccess/libpciaccess-0.13.2.tar.bz2/b7c0d3afce14eedca57312a3141ec13a/libpciaccess-0.13.2.tar.bz2" + """Generic PCI access library.""" - version('0.13.2', 'b7c0d3afce14eedca57312a3141ec13a') + homepage = "http://cgit.freedesktop.org/xorg/lib/libpciaccess/" + url = "http://cgit.freedesktop.org/xorg/lib/libpciaccess/" + + version('0.13.4', git='http://anongit.freedesktop.org/git/xorg/lib/libpciaccess.git', + tag='libpciaccess-0.13.4') + + depends_on('autoconf') + depends_on('libtool') def install(self, spec, prefix): - import os - os.system("autoconf") - configure('--prefix=%s' % prefix) + from subprocess import call + call(["./autogen.sh"]) + configure("--prefix=%s" % prefix) make() make("install") diff --git a/var/spack/packages/libsodium/package.py b/var/spack/packages/libsodium/package.py new file mode 100644 index 0000000000..1c8a16d998 --- /dev/null +++ b/var/spack/packages/libsodium/package.py @@ -0,0 +1,19 @@ +from spack import * + +class Libsodium(Package): + """Sodium is a modern, easy-to-use software library for encryption, + decryption, signatures, password hashing and more.""" + homepage = "https://download.libsodium.org/doc/" + url = "https://download.libsodium.org/libsodium/releases/libsodium-1.0.3.tar.gz" + + version('1.0.3', 'b3bcc98e34d3250f55ae196822307fab') + version('1.0.2', 'dc40eb23e293448c6fc908757738003f') + version('1.0.1', '9a221b49fba7281ceaaf5e278d0f4430') + version('1.0.0', '3093dabe4e038d09f0d150cef064b2f7') + version('0.7.1', 'c224fe3923d1dcfe418c65c8a7246316') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + + make() + make("install") diff --git a/var/spack/packages/libxml2/package.py b/var/spack/packages/libxml2/package.py index 72199d8def..3a0af6b368 100644 --- a/var/spack/packages/libxml2/package.py +++ b/var/spack/packages/libxml2/package.py @@ -9,11 +9,12 @@ class Libxml2(Package): version('2.9.2', '9e6a9aca9d155737868b3dc5fd82f788') + extends('python') depends_on('zlib') depends_on('xz') def install(self, spec, prefix): - configure("--prefix=%s" % prefix, - "--without-python") + configure("--prefix=%s" % prefix) + make() make("install") diff --git a/var/spack/packages/lmdb/package.py b/var/spack/packages/lmdb/package.py new file mode 100644 index 0000000000..875b8100c5 --- /dev/null +++ b/var/spack/packages/lmdb/package.py @@ -0,0 +1,39 @@ +import os +from spack import * + +class Lmdb(Package): + """Read-only mirror of official repo on openldap.org. Issues and + pull requests here are ignored. Use OpenLDAP ITS for issues. + http://www.openldap.org/software/repo.html""" + + + homepage = "http://www.openldap.org/software/repo.html" + url = "https://github.com/LMDB/lmdb/archive/LMDB_0.9.16.tar.gz" + + version('0.9.16', '0de89730b8f3f5711c2b3a4ba517b648') + + def install(self, spec, prefix): + os.chdir('libraries/liblmdb') + + make() + + mkdirp(prefix.bin) + mkdirp(prefix + '/man/man1') + mkdirp(prefix.lib) + mkdirp(prefix.include) + + bins = ['mdb_stat', 'mdb_copy', 'mdb_dump', 'mdb_load'] + for f in bins: + install(f, prefix.bin) + + mans = ['mdb_stat.1', 'mdb_copy.1', 'mdb_dump.1', 'mdb_load.1'] + for f in mans: + install(f, prefix + '/man/man1') + + libs = ['liblmdb.a', 'liblmdb.so'] + for f in libs: + install(f, prefix.lib) + + includes = ['lmdb.h'] + for f in includes: + install(f, prefix.include) diff --git a/var/spack/packages/lua/package.py b/var/spack/packages/lua/package.py new file mode 100644 index 0000000000..57c443cc2d --- /dev/null +++ b/var/spack/packages/lua/package.py @@ -0,0 +1,26 @@ +from spack import * +import os + +class Lua(Package): + """ The Lua programming language interpreter and library """ + homepage = "http://www.lua.org" + url = "http://www.lua.org/ftp/lua-5.1.5.tar.gz" + + version('5.3.1', '797adacada8d85761c079390ff1d9961') + version('5.3.0', 'a1b0a7e92d0c85bbff7a8d27bf29f8af') + version('5.2.4', '913fdb32207046b273fdb17aad70be13') + version('5.2.3', 'dc7f94ec6ff15c985d2d6ad0f1b35654') + version('5.2.2', 'efbb645e897eae37cad4344ce8b0a614') + version('5.2.1', 'ae08f641b45d737d12d30291a5e5f6e3') + version('5.2.0', 'f1ea831f397214bae8a265995ab1a93e') + version('5.1.5', '2e115fe26e435e33b0d5c022e4490567') + version('5.1.4', 'd0870f2de55d59c1c8419f36e8fac150') + version('5.1.3', 'a70a8dfaa150e047866dc01a46272599') + + depends_on('ncurses') + + def install(self, spec, prefix): + make('INSTALL_TOP=%s' % prefix, + 'MYLDFLAGS=-L%s/lib' % spec['ncurses'].prefix, + 'linux', + 'install') diff --git a/var/spack/packages/mesa/package.py b/var/spack/packages/mesa/package.py index 2dba878a77..2a04a8fd51 100644 --- a/var/spack/packages/mesa/package.py +++ b/var/spack/packages/mesa/package.py @@ -1,7 +1,7 @@ from spack import * class Mesa(Package): - """Mesa is an open-source implementation of the OpenGL + """Mesa is an open-source implementation of the OpenGL specification - a system for rendering interactive 3D graphics.""" homepage = "http://www.mesa3d.org" @@ -11,9 +11,10 @@ class Mesa(Package): # version('10.4.4', '8d863a3c209bf5116b2babfccccc68ce') version('8.0.5', 'cda5d101f43b8784fa60bdeaca4056f2') - # mesa 7.x, 8.x, 9.x + # mesa 7.x, 8.x, 9.x depends_on("libdrm@2.4.33") depends_on("llvm@3.0") + depends_on("libxml2") # patch("llvm-fixes.patch") # using newer llvm diff --git a/var/spack/packages/mpich/package.py b/var/spack/packages/mpich/package.py index ce07335bba..dfff22152d 100644 --- a/var/spack/packages/mpich/package.py +++ b/var/spack/packages/mpich/package.py @@ -38,7 +38,6 @@ class Mpich(Package): provides('mpi@:3', when='@3:') provides('mpi@:1', when='@1:') - def setup_dependent_environment(self, module, spec, dep_spec): """For dependencies, make mpicc's use spack wrapper.""" os.environ['MPICH_CC'] = 'cc' diff --git a/var/spack/packages/munge/package.py b/var/spack/packages/munge/package.py new file mode 100644 index 0000000000..c737ca0354 --- /dev/null +++ b/var/spack/packages/munge/package.py @@ -0,0 +1,20 @@ +from spack import * +import os + +class Munge(Package): + """ MUNGE Uid 'N' Gid Emporium """ + homepage = "https://code.google.com/p/munge/" + url = "https://github.com/dun/munge/releases/download/munge-0.5.11/munge-0.5.11.tar.bz2" + + version('0.5.11', 'bd8fca8d5f4c1fcbef1816482d49ee01', url='https://github.com/dun/munge/releases/download/munge-0.5.11/munge-0.5.11.tar.bz2') + + depends_on('openssl') + depends_on('libgcrypt') + + def install(self, spec, prefix): + os.makedirs(os.path.join(prefix, "lib/systemd/system")) + configure("--prefix=%s" % prefix) + + make() + make("install") + diff --git a/var/spack/packages/ncurses/package.py b/var/spack/packages/ncurses/package.py index a22e83503f..8f5763bfdd 100644 --- a/var/spack/packages/ncurses/package.py +++ b/var/spack/packages/ncurses/package.py @@ -21,3 +21,11 @@ def install(self, spec, prefix): make() make("install") + configure("--prefix=%s" % prefix, + "--with-shared", + "--disable-widec", + "--disable-pc-files", + "--without-ada") + make() + make("install") + diff --git a/var/spack/packages/netlib-blas/package.py b/var/spack/packages/netlib-blas/package.py index 0a6cdb0442..85e97323d3 100644 --- a/var/spack/packages/netlib-blas/package.py +++ b/var/spack/packages/netlib-blas/package.py @@ -9,6 +9,8 @@ class NetlibBlas(Package): version('3.5.0', 'b1d3e3e425b2e44a06760ff173104bdf') + variant('fpic', default=False, description="Build with -fpic compiler option") + # virtual dependency provides('blas') @@ -23,6 +25,10 @@ def patch(self): mf.filter('^LOADER.*', 'LOADER = f90') mf.filter('^CC =.*', 'CC = cc') + if '+fpic' in self.spec: + mf.filter('^OPTS.*=.*', 'OPTS = -O2 -frecursive -fpic') + mf.filter('^CFLAGS =.*', 'CFLAGS = -O3 -fpic') + def install(self, spec, prefix): make('blaslib') diff --git a/var/spack/packages/lapack/package.py b/var/spack/packages/netlib-lapack/package.py similarity index 77% rename from var/spack/packages/lapack/package.py rename to var/spack/packages/netlib-lapack/package.py index d9d37e3e4a..fb6b99e27c 100644 --- a/var/spack/packages/lapack/package.py +++ b/var/spack/packages/netlib-lapack/package.py @@ -1,6 +1,6 @@ from spack import * -class Lapack(Package): +class NetlibLapack(Package): """ LAPACK version 3.X is a comprehensive FORTRAN library that does linear algebra operations including matrix inversions, least @@ -18,9 +18,16 @@ class Lapack(Package): version('3.4.0', '02d5706ec03ba885fc246e5fa10d8c70') version('3.3.1', 'd0d533ec9a5b74933c2a1e84eedc58b4') + variant('shared', default=False, description="Build shared library version") + + # virtual dependency + provides('lapack') + # blas is a virtual dependency. depends_on('blas') + depends_on('cmake') + # Doesn't always build correctly in parallel parallel = False @@ -39,7 +46,14 @@ def get_blas_libs(self): def install(self, spec, prefix): blas_libs = ";".join(self.get_blas_libs()) - cmake(".", '-DBLAS_LIBRARIES=' + blas_libs, *std_cmake_args) + cmake_args = [".", '-DBLAS_LIBRARIES=' + blas_libs] + + if '+shared' in spec: + cmake_args.append('-DBUILD_SHARED_LIBS=ON') + + cmake_args += std_cmake_args + + cmake(*cmake_args) make() make("install") diff --git a/var/spack/packages/pidx/package.py b/var/spack/packages/pidx/package.py new file mode 100644 index 0000000000..81aed62fb1 --- /dev/null +++ b/var/spack/packages/pidx/package.py @@ -0,0 +1,21 @@ +from spack import * + +class Pidx(Package): + """PIDX Parallel I/O Library. + + PIDX is an efficient parallel I/O library that reads and writes + multiresolution IDX data files. + """ + + homepage = "http://www.cedmav.com/pidx" + + version('1.0', git='https://github.com/sci-visus/PIDX.git', + commit='6afa1cf71d1c41263296dc049c8fabaf73c296da') + + depends_on("mpi") + + def install(self, spec, prefix): + with working_dir('spack-build', create=True): + cmake('..', *std_cmake_args) + make() + make("install") diff --git a/var/spack/packages/pkg-config/package.py b/var/spack/packages/pkg-config/package.py new file mode 100644 index 0000000000..9964c6ce34 --- /dev/null +++ b/var/spack/packages/pkg-config/package.py @@ -0,0 +1,17 @@ +from spack import * + +class PkgConfig(Package): + """pkg-config is a helper tool used when compiling applications and libraries""" + homepage = "http://www.freedesktop.org/wiki/Software/pkg-config/" + url = "http://pkgconfig.freedesktop.org/releases/pkg-config-0.28.tar.gz" + + version('0.28', 'aa3c86e67551adc3ac865160e34a2a0d') + + parallel = False + + def install(self, spec, prefix): + configure("--prefix=%s" %prefix, "--enable-shared") + + make() + make("install") + diff --git a/var/spack/packages/protobuf/package.py b/var/spack/packages/protobuf/package.py new file mode 100644 index 0000000000..34085c7ce9 --- /dev/null +++ b/var/spack/packages/protobuf/package.py @@ -0,0 +1,16 @@ +import os +from spack import * + +class Protobuf(Package): + """Google's data interchange format.""" + + homepage = "https://developers.google.com/protocol-buffers" + url = "https://github.com/google/protobuf/releases/download/v2.5.0/protobuf-2.5.0.tar.bz2" + + version('2.5.0', 'a72001a9067a4c2c4e0e836d0f92ece4') + + def install(self, spec, prefix): + configure("--prefix=" + prefix) + make() + make("check") + make("install") diff --git a/var/spack/packages/py-Cython/package.py b/var/spack/packages/py-Cython/package.py deleted file mode 100644 index c250139114..0000000000 --- a/var/spack/packages/py-Cython/package.py +++ /dev/null @@ -1,12 +0,0 @@ -from spack import * - -class PyCython(Package): - """The Cython compiler for writing C extensions for the Python language.""" - homepage = "http://www.cython.org" - version("0.21.2", "d21adb870c75680dc857cd05d41046a4", - url="https://pypi.python.org/packages/source/C/Cython/Cython-0.21.2.tar.gz") - - extends("python") - - def install(self, spec, prefix): - python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-Distutils2/package.py b/var/spack/packages/py-Distutils2/package.py deleted file mode 100644 index 8690f569fe..0000000000 --- a/var/spack/packages/py-Distutils2/package.py +++ /dev/null @@ -1,15 +0,0 @@ -from spack import * - -class PyDistutils2(Package): - """Python Packaging Library.""" - - homepage = "https://hg.python.org/distutils2" - url = "https://pypi.python.org/packages/source/D/Distutils2/Distutils2-1.0a4.tar.gz" - - version('1.0a4', '52bc9dffb394970c27e02853ae3a3241') - - depends_on("python") - - def install(self, spec, prefix): - python('setup.py', 'install', '--prefix=%s' % prefix) - diff --git a/var/spack/packages/py-astroML/package.py b/var/spack/packages/py-astroML/package.py deleted file mode 100644 index 1583e249d2..0000000000 --- a/var/spack/packages/py-astroML/package.py +++ /dev/null @@ -1,12 +0,0 @@ -from spack import * - -class PyAstroml(Package): - """tools for machine learning and data mining in Astronomy""" - homepage = "http://astroML.github.com" - version("0.2", "85f558368546660564f20b30efafb024", - url="https://pypi.python.org/packages/source/a/astroML/astroML-0.2.tar.gz") - - extends("python") - - def install(self, spec, prefix): - python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-astropy/package.py b/var/spack/packages/py-astropy/package.py deleted file mode 100644 index 5d2d0619cc..0000000000 --- a/var/spack/packages/py-astropy/package.py +++ /dev/null @@ -1,12 +0,0 @@ -from spack import * - -class PyAstropy(Package): - """Community-developed python astronomy tools""" - homepage = "http://astropy.org" - version("0.4.2", "6dc4f643cde37ba0a8b4967dc8becee8", - url="https://pypi.python.org/packages/source/a/astropy/astropy-0.4.2.tar.gz") - - extends("python") - - def install(self, spec, prefix): - python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-autopep8/package.py b/var/spack/packages/py-autopep8/package.py deleted file mode 100644 index 39a2288ea0..0000000000 --- a/var/spack/packages/py-autopep8/package.py +++ /dev/null @@ -1,12 +0,0 @@ -from spack import * - -class PyAutopep8(Package): - """A tool that automatically formats Python code to conform to the PEP 8 style guide""" - homepage = "https://github.com/hhatto/autopep8" - version("1.1", "7998358d8f0efd77dcb2cc8e34e3cb5c", - url="https://pypi.python.org/packages/source/a/autopep8/autopep8-1.1.tar.gz") - - extends("python") - - def install(self, spec, prefix): - python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-certifi/package.py b/var/spack/packages/py-certifi/package.py deleted file mode 100644 index 442903e6a0..0000000000 --- a/var/spack/packages/py-certifi/package.py +++ /dev/null @@ -1,12 +0,0 @@ -from spack import * - -class PyCertifi(Package): - """Python package for providing Mozilla's CA Bundle.""" - homepage = "http://python-requests.org" - version("14.05.14", "315ea4e50673a16ab047099f816fd32a", - url="https://pypi.python.org/packages/source/c/certifi/certifi-14.05.14.tar.gz") - - extends("python") - - def install(self, spec, prefix): - python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-decorator/package.py b/var/spack/packages/py-decorator/package.py deleted file mode 100644 index 00c1c541d4..0000000000 --- a/var/spack/packages/py-decorator/package.py +++ /dev/null @@ -1,12 +0,0 @@ -from spack import * - -class PyDecorator(Package): - """Better living through Python with decorators""" - homepage = "https://github.com/micheles/decorator" - version("4.0.4", "dd3a0669e1e6f09699eefa2c7fbd9756", - url="https://pypi.python.org/packages/source/d/decorator/decorator-4.0.4.tar.gz") - - extends("python") - - def install(self, spec, prefix): - python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-emcee/package.py b/var/spack/packages/py-emcee/package.py deleted file mode 100644 index 544066b92d..0000000000 --- a/var/spack/packages/py-emcee/package.py +++ /dev/null @@ -1,12 +0,0 @@ -from spack import * - -class PyEmcee(Package): - """Kick ass affine-invariant ensemble MCMC sampling""" - homepage = "http://dan.iel.fm/emcee/" - version("2.1.0", "c6b6fad05c824d40671d4a4fc58dfff7", - url="https://pypi.python.org/packages/source/e/emcee/emcee-2.1.0.tar.gz") - - extends("python") - - def install(self, spec, prefix): - python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-fitsio/package.py b/var/spack/packages/py-fitsio/package.py deleted file mode 100644 index 046bc386d5..0000000000 --- a/var/spack/packages/py-fitsio/package.py +++ /dev/null @@ -1,13 +0,0 @@ -from spack import * - -class PyFitsio(Package): - """A full featured python library to read from and write to FITS files.""" - homepage = "https://github.com/esheldon/fitsio" - version("0.9.7", "75fa05f999ae8f55c8290bd78ada49e7", - url="https://pypi.python.org/packages/source/f/fitsio/fitsio-0.9.7.tar.gz") - - extends("python") - depends_on("py-numpy") - - def install(self, spec, prefix): - python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-gnureadline/package.py b/var/spack/packages/py-gnureadline/package.py deleted file mode 100644 index 51d1c6227e..0000000000 --- a/var/spack/packages/py-gnureadline/package.py +++ /dev/null @@ -1,12 +0,0 @@ -from spack import * - -class PyGnureadline(Package): - """The standard Python readline extension statically linked against the GNU readline library.""" - homepage = "http://github.com/ludwigschwardt/python-gnureadline" - version("6.3.3", "c4af83c9a3fbeac8f2da9b5a7c60e51c", - url="https://pypi.python.org/packages/source/g/gnureadline/gnureadline-6.3.3.tar.gz") - - extends("python") - - def install(self, spec, prefix): - python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-ipython/package.py b/var/spack/packages/py-ipython/package.py index 3b94b71f29..8d0e64a07f 100644 --- a/var/spack/packages/py-ipython/package.py +++ b/var/spack/packages/py-ipython/package.py @@ -3,29 +3,14 @@ class PyIpython(Package): """IPython provides a rich toolkit to help you make the most out of using Python interactively.""" homepage = "https://pypi.python.org/pypi/ipython" + url = "https://pypi.python.org/packages/source/i/ipython/ipython-2.3.1.tar.gz" - version('3.0.0','b3f00f3c0be036fafef3b0b9d663f27e', - url='https://pypi.python.org/packages/source/i/ipython/ipython-3.0.0.tar.gz') - version('4.0.0','c2fecbcf1c0fbdc82625c77a50733dd6', - url='https://pypi.python.org/packages/source/i/ipython/ipython-4.0.0.tar.gz') - version('2.3.1', '2b7085525dac11190bfb45bb8ec8dcbf', - url="https://pypi.python.org/packages/source/i/ipython/ipython-2.3.1.tar.gz") + version('2.3.1', '2b7085525dac11190bfb45bb8ec8dcbf') version('3.1.0', 'a749d90c16068687b0ec45a27e72ef8f') - version('3.2.1', 'f4c93d67ac4b2d4fc69df693b6f3c9e0', - url='https://github.com/ipython/ipython/archive/rel-3.2.1.tar.gz') extends('python') depends_on('py-pygments') - depends_on('py-setuptools@18.1') - depends_on('py-traitlets') - depends_on('py-pyzmq') - depends_on('py-pexpect') - depends_on('py-tornado') - depends_on('py-jinja2') + depends_on('py-setuptools') def install(self, spec, prefix): - import os - # os.system("spack activate py-setuptools@18.1") - # os.system("easy_install pip") - # os.system("pip install ipython[notebook]") python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-ipython/package.py.backup b/var/spack/packages/py-ipython/package.py.backup deleted file mode 100755 index 1ed8b57e60..0000000000 --- a/var/spack/packages/py-ipython/package.py.backup +++ /dev/null @@ -1,27 +0,0 @@ -from spack import * - -class PyIpython(Package): - """IPython provides a rich toolkit to help you make the most out of using Python interactively.""" - homepage = "https://pypi.python.org/pypi/ipython" - - version('2.3.1', '2b7085525dac11190bfb45bb8ec8dcbf', - url="https://pypi.python.org/packages/source/i/ipython/ipython-2.3.1.tar.gz") - version('3.1.0', 'a749d90c16068687b0ec45a27e72ef8f') - version('3.2.1', 'f4c93d67ac4b2d4fc69df693b6f3c9e0', - url='https://github.com/ipython/ipython/archive/rel-3.2.1.tar.gz') - - extends('python') - depends_on('py-pygments') - depends_on('py-setuptools@18.1') - depends_on('py-traitlets') - depends_on('py-pyzmq') - depends_on('py-pexpect') - depends_on('py-tornado') - depends_on('py-Jinja2') - - def install(self, spec, prefix): - import os - # os.system("spack activate py-setuptools@18.1") - # os.system("easy_install pip") - # os.system("pip install ipython[notebook]") - python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-ipython_genutils/package.py b/var/spack/packages/py-ipython_genutils/package.py deleted file mode 100644 index 39b15eec73..0000000000 --- a/var/spack/packages/py-ipython_genutils/package.py +++ /dev/null @@ -1,12 +0,0 @@ -from spack import * - -class PyIpythonGenutils(Package): - """Vestigial utilities from IPython""" - homepage = "http://ipython.org" - version("0.1.0", "9a8afbe0978adbcbfcb3b35b2d015a56", - url="https://pypi.python.org/packages/source/i/ipython_genutils/ipython_genutils-0.1.0.tar.gz") - - extends("python") - - def install(self, spec, prefix): - python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-joblib/package.py b/var/spack/packages/py-joblib/package.py deleted file mode 100644 index 10672314de..0000000000 --- a/var/spack/packages/py-joblib/package.py +++ /dev/null @@ -1,12 +0,0 @@ -from spack import * - -class PyJoblib(Package): - """Lightweight pipelining: using Python functions as pipeline jobs.""" - homepage = "http://packages.python.org/joblib/" - version("0.8.4", "90a1c25cc4dc4a8e3536093dbc35cff3", - url="https://pypi.python.org/packages/source/j/joblib/joblib-0.8.4.tar.gz") - - extends("python") - - def install(self, spec, prefix): - python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-jsonschema/package.py b/var/spack/packages/py-jsonschema/package.py deleted file mode 100644 index 9a7fc21862..0000000000 --- a/var/spack/packages/py-jsonschema/package.py +++ /dev/null @@ -1,16 +0,0 @@ -from spack import * - -class PyJsonschema(Package): - """implementation of JSON Schema validation for Python""" - homepage = "https://github.com/Julian/jsonschema" - url = "https://pypi.python.org/packages/source/j/jsonschema/jsonschema-2.5.1.tar.gz" - - version('2.4.0','661f85c3d23094afbb9ac3c0673840bf', - url='https://pypi.python.org/packages/source/j/jsonschema/jsonschema-2.4.0.tar.gz') - version('2.5.1', '374e848fdb69a3ce8b7e778b47c30640') - - extends("python") - depends_on("py-setuptools@18.1") - - def install(self, spec, prefix): - python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-matplotlib/package.py b/var/spack/packages/py-matplotlib/package.py index b9244daf56..e7ce3dfd24 100644 --- a/var/spack/packages/py-matplotlib/package.py +++ b/var/spack/packages/py-matplotlib/package.py @@ -12,6 +12,7 @@ class PyMatplotlib(Package): extends('python', ignore=r'bin/nosetests.*$') depends_on('py-pyside') + depends_on('py-ipython') depends_on('py-pyparsing') depends_on('py-six') depends_on('py-dateutil') diff --git a/var/spack/packages/py-mock/package.py b/var/spack/packages/py-mock/package.py index c43842c187..3b08428ba0 100644 --- a/var/spack/packages/py-mock/package.py +++ b/var/spack/packages/py-mock/package.py @@ -1,12 +1,17 @@ from spack import * class PyMock(Package): - """A Python Mocking and Patching Library for Testing""" - homepage = "http://www.voidspace.org.uk/python/mock/" - version("1.0.1", "c3971991738caa55ec7c356bbc154ee2", - url="https://pypi.python.org/packages/source/m/mock/mock-1.0.1.tar.gz") + """mock is a library for testing in Python. It allows you to replace parts + of your system under test with mock objects and make assertions about how + they have been used.""" - extends("python") + homepage = "https://github.com/testing-cabal/mock" + url = "https://pypi.python.org/packages/source/m/mock/mock-1.3.0.tar.gz" + + version('1.3.0', '73ee8a4afb3ff4da1b4afa287f39fdeb') + + extends('python') + depends_on('py-setuptools@17.1:') def install(self, spec, prefix): - python("setup.py", "install", "--prefix=%s" % prefix) + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-numexpr/package.py b/var/spack/packages/py-numexpr/package.py deleted file mode 100644 index 283ea18772..0000000000 --- a/var/spack/packages/py-numexpr/package.py +++ /dev/null @@ -1,12 +0,0 @@ -from spack import * - -class PyNumexpr(Package): - """Fast numerical expression evaluator for NumPy""" - homepage = "https://github.com/pydata/numexpr" - version("2.4", "df7e8d9e9dbb145b56d43c465c2bf854", - url="https://pypi.python.org/packages/source/n/numexpr/numexpr-2.4.tar.gz") - - extends("python") - - def install(self, spec, prefix): - python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-numpy/package.py b/var/spack/packages/py-numpy/package.py index 6534e063d2..efa109a3e9 100644 --- a/var/spack/packages/py-numpy/package.py +++ b/var/spack/packages/py-numpy/package.py @@ -7,9 +7,22 @@ class PyNumpy(Package): version('1.9.1', '78842b73560ec378142665e712ae4ad9') version('1.9.2', 'a1ed53432dbcd256398898d35bc8e645') - + extends('python') depends_on('py-nose') + depends_on('netlib-blas+fpic') + depends_on('netlib-lapack+shared') + + def patch(self): + filter_file( + "possible_executables = \['(gfortran|g77|ifort|efl)", + "possible_executables = ['fc", + "numpy/distutils/fcompiler/gnu.py", + "numpy/distutils/fcompiler/intel.py") def install(self, spec, prefix): + with open('site.cfg', 'w') as f: + f.write('[DEFAULT]\n') + f.write('libraries=lapack,blas\n') + f.write('library_dirs=%s/lib:%s/lib\n' % (spec['blas'].prefix, spec['lapack'].prefix)) python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-pandas/package.py b/var/spack/packages/py-pandas/package.py index 07ee12782f..5b9997faa9 100644 --- a/var/spack/packages/py-pandas/package.py +++ b/var/spack/packages/py-pandas/package.py @@ -6,8 +6,6 @@ class PyPandas(Package): homepage = "http://pandas.pydata.org/" url = "https://pypi.python.org/packages/source/p/pandas/pandas-0.16.0.tar.gz#md5=bfe311f05dc0c351f8955fbd1e296e73" - version('0.15.1','f6fd8d49fc801f6f3fa48f9117a6290b', - url='https://pypi.python.org/packages/source/p/pandas/pandas-0.15.1.tar.gz') version('0.16.0', 'bfe311f05dc0c351f8955fbd1e296e73') version('0.16.1', 'fac4f25748f9610a3e00e765474bdea8') diff --git a/var/spack/packages/py-pep8/package.py b/var/spack/packages/py-pep8/package.py deleted file mode 100644 index 0bcf416922..0000000000 --- a/var/spack/packages/py-pep8/package.py +++ /dev/null @@ -1,12 +0,0 @@ -from spack import * - -class PyPep8(Package): - """Python style guide checker""" - homepage = "http://pep8.readthedocs.org/" - version("1.6.1", "76cf60b245f8549cb458ffcd85710738", - url="https://pypi.python.org/packages/source/p/pep8/pep8-1.6.1.tar.gz") - - extends("python") - - def install(self, spec, prefix): - python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-pickleshare/package.py b/var/spack/packages/py-pickleshare/package.py deleted file mode 100644 index cf9ddb7db8..0000000000 --- a/var/spack/packages/py-pickleshare/package.py +++ /dev/null @@ -1,12 +0,0 @@ -from spack import * - -class PyPickleshare(Package): - """Tiny 'shelve'-like database with concurrency support""" - homepage = "https://github.com/vivainio/pickleshare" - version("0.5", "25337740507cb855ad58bfcf60f7710e", - url="https://pypi.python.org/packages/source/p/pickleshare/pickleshare-0.5.tar.gz") - - extends("python") - - def install(self, spec, prefix): - python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-pip/package.py b/var/spack/packages/py-pip/package.py deleted file mode 100644 index 89b8570fe4..0000000000 --- a/var/spack/packages/py-pip/package.py +++ /dev/null @@ -1,12 +0,0 @@ -from spack import * - -class PyPip(Package): - """The PyPA recommended tool for installing Python packages.""" - homepage = "https://pip.pypa.io/" - version("7.1.2", "3823d2343d9f3aaab21cf9c917710196", - url="https://pypi.python.org/packages/source/p/pip/pip-7.1.2.tar.gz") - - extends("python") - - def install(self, spec, prefix): - python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-py/package.py b/var/spack/packages/py-py/package.py deleted file mode 100644 index 0ef0403124..0000000000 --- a/var/spack/packages/py-py/package.py +++ /dev/null @@ -1,12 +0,0 @@ -from spack import * - -class PyPy(Package): - """library with cross-python path, ini-parsing, io, code, log facilities""" - homepage = "http://pylib.readthedocs.org/" - version("1.4.26", "30c3fd92a53f1a5ed6f3591c1fe75c0e", - url="https://pypi.python.org/packages/source/p/py/py-1.4.26.tar.gz") - - extends("python") - - def install(self, spec, prefix): - python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-py4j/package.py b/var/spack/packages/py-py4j/package.py deleted file mode 100644 index 4c31f57298..0000000000 --- a/var/spack/packages/py-py4j/package.py +++ /dev/null @@ -1,15 +0,0 @@ -from spack import * - - -class PyPy4j(Package): - """Enables Python programs to dynamically access arbitrary Java objects""" - homepage = "https://www.py4j.org/" - url = "https://pypi.python.org/packages/source/p/py4j/py4j-0.9.tar.gz" - - version('0.9', 'b6fed5faef81a5368e3d50a91a5c9a60') - - extends("python") - depends_on("py-setuptools@18.1") - - def install(self, spec, prefix): - python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-pyside/package.py b/var/spack/packages/py-pyside/package.py index 8d093efd2c..bb5da44d02 100644 --- a/var/spack/packages/py-pyside/package.py +++ b/var/spack/packages/py-pyside/package.py @@ -8,9 +8,12 @@ class PyPyside(Package): version('1.2.2', 'c45bc400c8a86d6b35f34c29e379e44d') + # TODO: make build dependency + # depends_on("cmake") + extends('python') depends_on('py-setuptools') - depends_on('qt@5.4.0') + depends_on('qt@:4') def patch(self): """Undo PySide RPATH handling and add Spack RPATH.""" diff --git a/var/spack/packages/py-pytest/package.py b/var/spack/packages/py-pytest/package.py deleted file mode 100644 index 3611b68862..0000000000 --- a/var/spack/packages/py-pytest/package.py +++ /dev/null @@ -1,12 +0,0 @@ -from spack import * - -class PyPytest(Package): - """pytest: simple powerful testing with Python""" - homepage = "http://pytest.org" - version("2.6.4", "14341e122f7e9031a0948eb6b01a2640", - url="https://pypi.python.org/packages/source/p/pytest/pytest-2.6.4.tar.gz") - - extends("python") - - def install(self, spec, prefix): - python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-python-dateutil/package.py b/var/spack/packages/py-python-dateutil/package.py deleted file mode 100644 index 8efa114db1..0000000000 --- a/var/spack/packages/py-python-dateutil/package.py +++ /dev/null @@ -1,12 +0,0 @@ -from spack import * - -class PyPythonDateutil(Package): - """Extensions to the standard Python datetime module""" - homepage = "http://labix.org/python-dateutil" - version("2.2", "c1f654d0ff7e33999380a8ba9783fd5c", - url="https://pypi.python.org/packages/source/p/python-dateutil/python-dateutil-2.2.tar.gz") - - extends("python") - - def install(self, spec, prefix): - python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-pytz/package.py b/var/spack/packages/py-pytz/package.py index f67bb42854..da6311a784 100644 --- a/var/spack/packages/py-pytz/package.py +++ b/var/spack/packages/py-pytz/package.py @@ -5,8 +5,6 @@ class PyPytz(Package): homepage = "https://pypi.python.org/pypi/pytz" url = "https://pypi.python.org/packages/source/p/pytz/pytz-2014.10.tar.gz" - version('2014.9','d42bda2f4c1e873e02fbd1e4acfd1b8c', - url='https://pypi.python.org/packages/source/p/pytz/pytz-2014.9.tar.gz') version('2014.10', 'eb1cb941a20c5b751352c52486aa1dd7') version('2015.4', '417a47b1c432d90333e42084a605d3d8') diff --git a/var/spack/packages/py-pyzmq/package.py b/var/spack/packages/py-pyzmq/package.py deleted file mode 100644 index f5551182b3..0000000000 --- a/var/spack/packages/py-pyzmq/package.py +++ /dev/null @@ -1,16 +0,0 @@ -from spack import * - -class PyPyzmq(Package): - """official Python binding for the ZeroMQ Messaging Library""" - homepage = "https://github.com/zeromq/pyzmq" - url = "https://pypi.python.org/packages/source/p/pyzmq/pyzmq-14.7.0.tar.gz" - - version('14.4.1','a6a7fb6950b1bf5333c6795f3edf196c', - url='https://pypi.python.org/packages/source/p/pyzmq/pyzmq-14.4.1.tar.gz') - version('14.7.0', '87e3abb33af5794db5ae85c667bbf324') - - extends('python') - depends_on('py-setuptools@18.1') - - def install(self, spec, prefix): - python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-scikit-learn/package.py b/var/spack/packages/py-scikit-learn/package.py index 744b37e7e0..5b078ce901 100644 --- a/var/spack/packages/py-scikit-learn/package.py +++ b/var/spack/packages/py-scikit-learn/package.py @@ -9,9 +9,6 @@ class PyScikitLearn(Package): version('0.16.1', '363ddda501e3b6b61726aa40b8dbdb7e') extends('python') - depends_on('python@2.7.10') - depends_on('py-numpy@1.9.2') - depends_on('py-scipy@0.15.1') def install(self, spec, prefix): python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-scipy/package.py b/var/spack/packages/py-scipy/package.py index bcbdb2100f..3a1124cc15 100644 --- a/var/spack/packages/py-scipy/package.py +++ b/var/spack/packages/py-scipy/package.py @@ -5,14 +5,14 @@ class PyScipy(Package): homepage = "https://pypi.python.org/pypi/scipy" url = "https://pypi.python.org/packages/source/s/scipy/scipy-0.15.0.tar.gz" - version('0.14.0','d7c7f4ccf8b07b08d6fe49d5cd51f85d', - url='https://pypi.python.org/packages/source/s/scipy/scipy-0.14.0.tar.gz') version('0.15.0', '639112f077f0aeb6d80718dc5019dc7a') version('0.15.1', 'be56cd8e60591d6332aac792a5880110') extends('python') depends_on('py-nose') depends_on('py-numpy') + depends_on('blas') + depends_on('lapack') def install(self, spec, prefix): python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-seaborn/package.py b/var/spack/packages/py-seaborn/package.py deleted file mode 100644 index 1e228b0622..0000000000 --- a/var/spack/packages/py-seaborn/package.py +++ /dev/null @@ -1,12 +0,0 @@ -from spack import * - -class PySeaborn(Package): - """Seaborn: statistical data visualization""" - homepage = "http://stanford.edu/~mwaskom/software/seaborn/" - version("0.5.1", "2ce6ea7d3c67858c0b1f5793fa2043b8", - url="https://pypi.python.org/packages/source/s/seaborn/seaborn-0.5.1.tar.gz") - - extends("python") - - def install(self, spec, prefix): - python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-setuptools/package.py b/var/spack/packages/py-setuptools/package.py index 053fe1ba80..760ad4d6db 100644 --- a/var/spack/packages/py-setuptools/package.py +++ b/var/spack/packages/py-setuptools/package.py @@ -3,12 +3,11 @@ class PySetuptools(Package): """Easily download, build, install, upgrade, and uninstall Python packages.""" homepage = "https://pypi.python.org/pypi/setuptools" + url = "https://pypi.python.org/packages/source/s/setuptools/setuptools-11.3.tar.gz" - version('18.1', 'f72e87f34fbf07f299f6cb46256a0b06', - url='https://pypi.python.org/packages/source/s/setuptools/setuptools-18.1.tar.gz') - version('11.3.1', '01f69212e019a2420c1693fb43593930', - url="https://pypi.python.org/packages/source/s/setuptools/setuptools-11.3.tar.gz") - version('16.0', '0ace0b96233516fc5f7c857d086aa3ad',) + version('11.3.1', '01f69212e019a2420c1693fb43593930') + version('16.0', '0ace0b96233516fc5f7c857d086aa3ad') + version('18.1', 'f72e87f34fbf07f299f6cb46256a0b06') extends('python') diff --git a/var/spack/packages/py-six/package.py b/var/spack/packages/py-six/package.py index e9946a1e56..05c5bd00a9 100644 --- a/var/spack/packages/py-six/package.py +++ b/var/spack/packages/py-six/package.py @@ -5,8 +5,6 @@ class PySix(Package): homepage = "https://pypi.python.org/pypi/six" url = "https://pypi.python.org/packages/source/s/six/six-1.9.0.tar.gz" - version('1.8.0','1626eb24cc889110c38f7e786ec69885', - url='https://pypi.python.org/packages/source/s/six/six-1.8.0.tar.gz') version('1.9.0', '476881ef4012262dfc8adc645ee786c4') extends('python') diff --git a/var/spack/packages/py-statsmodels/package.py b/var/spack/packages/py-statsmodels/package.py deleted file mode 100644 index 393c772738..0000000000 --- a/var/spack/packages/py-statsmodels/package.py +++ /dev/null @@ -1,12 +0,0 @@ -from spack import * - -class PyStatsmodels(Package): - """Statistical computations and models for use with SciPy""" - homepage = "http://statsmodels.sourceforge.net/" - version("0.6.1", "f7580ebf7d2a2c9b87abfad190dcb9a3", - url="https://pypi.python.org/packages/source/s/statsmodels/statsmodels-0.6.1.tar.gz") - - extends("python") - - def install(self, spec, prefix): - python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-tables/package.py b/var/spack/packages/py-tables/package.py deleted file mode 100644 index 637ccf34cd..0000000000 --- a/var/spack/packages/py-tables/package.py +++ /dev/null @@ -1,12 +0,0 @@ -from spack import * - -class PyTables(Package): - """Hierarchical datasets for Python""" - homepage = "http://www.pytables.org/" - version("3.1.1", "38d917f0c6dfb0bc28ce9ea0c3492524", - url="https://pypi.python.org/packages/source/t/tables/tables-3.1.1.tar.gz") - - extends("python") - - def install(self, spec, prefix): - python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-tornado/package.py b/var/spack/packages/py-tornado/package.py deleted file mode 100644 index 4fc2d42c7b..0000000000 --- a/var/spack/packages/py-tornado/package.py +++ /dev/null @@ -1,17 +0,0 @@ -from spack import * - -class PyTornado(Package): - """Python web framework and asynchronous networking library""" - homepage = "http://www.tornadoweb.org/" - url = "https://pypi.python.org/packages/source/t/tornado/tornado-4.2.1.tar.gz" - - version('4.0.2','985c0e704b765c33a6193d49d1935588', - url='https://pypi.python.org/packages/source/t/tornado/tornado-4.0.2.tar.gz') - version('4.2.1', 'd523204389cfb70121bb69709f551b20') - - extends('python') - depends_on('py-certifi') - depends_on('py-setuptools@18.1') - - def install(self, spec, prefix): - python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-traitlets/package.py b/var/spack/packages/py-traitlets/package.py deleted file mode 100644 index 7b7f5c64cc..0000000000 --- a/var/spack/packages/py-traitlets/package.py +++ /dev/null @@ -1,14 +0,0 @@ -from spack import * - -class PyTraitlets(Package): - """A lightweight Traits like module""" - homepage = "http://traitlets.readthedocs.org/" - url = "https://github.com/ipython/traitlets/archive/4.0.0.tar.gz" - - version('4.0.0', 'b5b95ea5941fd9619b4704dfd8201568') - - extends('python') - depends_on("py-setuptools@18.1") - - def install(self, spec, prefix): - python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-triangle-plot/package.py b/var/spack/packages/py-triangle-plot/package.py deleted file mode 100644 index 29c5f92857..0000000000 --- a/var/spack/packages/py-triangle-plot/package.py +++ /dev/null @@ -1,12 +0,0 @@ -from spack import * - -class PyTrianglePlot(Package): - """Make some beautiful corner plots of samples.""" - homepage = "https://github.com/dfm/triangle.py" - version("0.0.6", "970a35a9bde6002d673e3188fe39f0ed", - url="https://pypi.python.org/packages/source/t/triangle_plot/triangle_plot-0.0.6.tar.gz") - - extends("python") - - def install(self, spec, prefix): - python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/samtools/package.py b/var/spack/packages/samtools/package.py new file mode 100644 index 0000000000..72900398d8 --- /dev/null +++ b/var/spack/packages/samtools/package.py @@ -0,0 +1,18 @@ +from spack import * + +class Samtools(Package): + """SAM Tools provide various utilities for manipulating alignments in the SAM format, + including sorting, merging, indexing and generating + alignments in a per-position format""" + + homepage = "www.htslib.org" + version('1.2','988ec4c3058a6ceda36503eebecd4122',url = "https://github.com/samtools/samtools/releases/download/1.2/samtools-1.2.tar.bz2") + + depends_on("zlib") + depends_on("mpc") + parallel=False + patch("samtools1.2.patch",level=0) + + def install(self, spec, prefix): + make("prefix=%s" % prefix, "install") + diff --git a/var/spack/packages/samtools/samtools1.2.patch b/var/spack/packages/samtools/samtools1.2.patch new file mode 100644 index 0000000000..ead3ab4e2c --- /dev/null +++ b/var/spack/packages/samtools/samtools1.2.patch @@ -0,0 +1,20 @@ +--- Makefile 2015-02-03 08:27:34.000000000 -0800 ++++ Makefile.new 2015-07-21 10:38:27.881406892 -0700 +@@ -26,7 +26,7 @@ + CFLAGS = -g -Wall -O2 + LDFLAGS = + LDLIBS = +-DFLAGS= -D_FILE_OFFSET_BITS=64 -D_LARGEFILE64_SOURCE -D_CURSES_LIB=1 ++DFLAGS= -D_FILE_OFFSET_BITS=64 -D_LARGEFILE64_SOURCE -D_CURSES_LIB=0 + LOBJS= bam_aux.o bam.o bam_import.o sam.o \ + sam_header.o bam_plbuf.o + AOBJS= bam_index.o bam_plcmd.o sam_view.o \ +@@ -37,7 +37,7 @@ + faidx.o stats.o stats_isize.o bam_flags.o bam_split.o \ + bam_tview.o bam_tview_curses.o bam_tview_html.o bam_lpileup.o + INCLUDES= -I. -I$(HTSDIR) +-LIBCURSES= -lcurses # -lXCurses ++#LIBCURSES= -lcurses # -lXCurses + + prefix = /usr/local + exec_prefix = $(prefix) diff --git a/var/spack/packages/scorep/package.py b/var/spack/packages/scorep/package.py index 32a772e3db..f013bd1cbb 100644 --- a/var/spack/packages/scorep/package.py +++ b/var/spack/packages/scorep/package.py @@ -1,11 +1,10 @@ # FIXME: Add copyright statement from spack import * -from contextlib import closing class Scorep(Package): - """The Score-P measurement infrastructure is a highly scalable and - easy-to-use tool suite for profiling, event tracing, and online + """The Score-P measurement infrastructure is a highly scalable and + easy-to-use tool suite for profiling, event tracing, and online analysis of HPC applications.""" # FIXME: add a proper url for your package's homepage here. @@ -20,7 +19,7 @@ class Scorep(Package): depends_on("mpi") depends_on("papi") # depends_on("otf2@1.2:1.2.1") # only Score-P 1.2.x - depends_on("otf2") + depends_on("otf2") depends_on("opari2") depends_on("cube@4.2:4.2.3") @@ -53,12 +52,12 @@ def install(self, spec, prefix): # Use a custom compiler configuration, otherwise the score-p # build system messes with spack's compiler settings. # Create these three files in the build directory - with closing(open("platform-backend-user-provided", "w")) as backend_file: + with open("platform-backend-user-provided", "w") as backend_file: backend_file.write(self.backend_user_provided) - with closing(open("platform-frontend-user-provided", "w")) as frontend_file: + with open("platform-frontend-user-provided", "w") as frontend_file: frontend_file.write(self.frontend_user_provided) - with closing(open("platform-mpi-user-provided", "w")) as mpi_file: - mpi_file.write(self.mpi_user_provided) + with open("platform-mpi-user-provided", "w") as mpi_file: + mpi_file.write(self.mpi_user_provided) configure_args = ["--prefix=%s" % prefix, "--with-custom-compilers", diff --git a/var/spack/packages/snappy/package.py b/var/spack/packages/snappy/package.py new file mode 100644 index 0000000000..c8f9ceef7d --- /dev/null +++ b/var/spack/packages/snappy/package.py @@ -0,0 +1,15 @@ +import os +from spack import * + +class Snappy(Package): + """A fast compressor/decompressor: https://code.google.com/p/snappy""" + + homepage = "https://code.google.com/p/snappy" + url = "https://github.com/google/snappy/releases/download/1.1.3/snappy-1.1.3.tar.gz" + + version('1.1.3', '7358c82f133dc77798e4c2062a749b73') + + def install(self, spec, prefix): + configure("--prefix=" + prefix) + make() + make("install") diff --git a/var/spack/packages/stat/package.py b/var/spack/packages/stat/package.py index c40d65ae78..5d81e62731 100644 --- a/var/spack/packages/stat/package.py +++ b/var/spack/packages/stat/package.py @@ -9,17 +9,20 @@ class Stat(Package): version('2.1.0', 'ece26beaf057aa9134d62adcdda1ba91') version('2.0.0', 'c7494210b0ba26b577171b92838e1a9b') + variant('dysect', default=False, description="enable DySectAPI") + depends_on('libelf') depends_on('libdwarf') depends_on('dyninst') depends_on('graphlib') + depends_on('graphviz') depends_on('launchmon') depends_on('mrnet') patch('configure_mpicxx.patch', when='@2.1.0') def install(self, spec, prefix): - configure( + configure_args = [ "--enable-gui", "--prefix=%s" % prefix, "--disable-examples", # Examples require MPI: avoid this dependency. @@ -27,7 +30,11 @@ def install(self, spec, prefix): "--with-mrnet=%s" % spec['mrnet'].prefix, "--with-graphlib=%s" % spec['graphlib'].prefix, "--with-stackwalker=%s" % spec['dyninst'].prefix, - "--with-libdwarf=%s" % spec['libdwarf'].prefix) + "--with-libdwarf=%s" % spec['libdwarf'].prefix + ] + if '+dysect' in spec: + configure_args.append('--enable-dysectapi') + configure(*configure_args) make(parallel=False) make("install") diff --git a/var/spack/packages/vim/package.py b/var/spack/packages/vim/package.py index d0cf80b898..4099b3257f 100644 --- a/var/spack/packages/vim/package.py +++ b/var/spack/packages/vim/package.py @@ -35,15 +35,26 @@ class Vim(Package): variant('ruby', default=False, description="build with Ruby") depends_on('ruby', when='+ruby') + variant('cscope', default=False, description="build with cscope support") + depends_on('cscope', when='+cscope') + + variant('gui', default=False, description="build with gui (gvim)") + # virtual dependency? + def install(self, spec, prefix): feature_set = None for fs in self.feature_sets: if "+" + fs in spec: if feature_set is not None: - tty.error("Only one feature set allowed, both {} and {} specified".format( - feature_set, - fs)) + tty.error("Only one feature set allowed, both %s and %s specified" + % (feature_set, fs)) feature_set = fs + if '+gui' in spec: + if feature_set is not None: + if feature_set is not 'huge': + tty.error("+gui variant requires 'huge' feature set, %s was specified" + % feature_set) + feature_set = 'huge' if feature_set is None: feature_set = 'normal' @@ -60,6 +71,12 @@ def install(self, spec, prefix): else: configure_args.append("--enable-rubyinterp=dynamic") + if '+gui' in spec: + configure_args.append("--enable-gui=auto") + + if '+cscope' in spec: + configure_args.append("--enable-cscope") + configure("--prefix=%s" % prefix, *configure_args) make() diff --git a/var/spack/packages/zeromq/package.py b/var/spack/packages/zeromq/package.py new file mode 100644 index 0000000000..b5a1e3d4cd --- /dev/null +++ b/var/spack/packages/zeromq/package.py @@ -0,0 +1,20 @@ +from spack import * + +class Zeromq(Package): + """ The ZMQ networking/concurrency library and core API """ + homepage = "http://zguide.zeromq.org/" + url = "http://download.zeromq.org/zeromq-4.1.2.tar.gz" + + version('4.1.2', '159c0c56a895472f02668e692d122685') + version('4.1.1', '0a4b44aa085644f25c177f79dc13f253') + version('4.0.7', '9b46f7e7b0704b83638ef0d461fd59ab') + version('4.0.6', 'd47dd09ed7ae6e7fd6f9a816d7f5fdf6') + version('4.0.5', '73c39f5eb01b9d7eaf74a5d899f1d03d') + + depends_on("libsodium") + + def install(self, spec, prefix): + configure("--with-libsodium","--prefix=%s" % prefix) + + make() + make("install") From 6da1a105cb448ec3c4f8ca4910b5bc9f12044001 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 8 Jan 2016 15:03:34 -0800 Subject: [PATCH 093/481] Revert "adding python_recipe_parser as a submodule under the folder utils" This reverts commit 1501786fd9be0a01abe7f6bb917e416fb3256aa6. --- .gitmodules | 3 --- utils/python_recipe_parser | 1 - 2 files changed, 4 deletions(-) delete mode 100644 .gitmodules delete mode 160000 utils/python_recipe_parser diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index a097fcd8cf..0000000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "utils/python_recipe_parser"] - path = utils/python_recipe_parser - url = https://github.com/karenyyng/spack_python_package_parser diff --git a/utils/python_recipe_parser b/utils/python_recipe_parser deleted file mode 160000 index 3158074a2d..0000000000 --- a/utils/python_recipe_parser +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 3158074a2d74f99aa72289dfcf093b1b0ba4474e From aa28e4e81f80a1a388aabe589ca23955ebd9721b Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 8 Jan 2016 15:13:48 -0800 Subject: [PATCH 094/481] Improved error messages for compiler_for_spec when either zero or multiple compilers returned. --- lib/spack/spack/compilers/__init__.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 87106282cf..3a0a88b5e3 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -219,7 +219,10 @@ def compiler_for_spec(compiler_spec, target): compilers = [c for c in compilers if c.modules is None] elif target.compiler_strategy == "MODULES": compilers = [c for c in compilers if c.modules is not None] - assert(len(compilers) == 1) + if len(compilers) < 1: + raise NoCompilerForSpecError(compiler_spec, target) + if len(compilers) > 1: + raise CompilerSpecInsufficientlySpecificError(compiler_spec) return compilers[0] @@ -253,3 +256,13 @@ def __init__(self, compiler_spec): class NoCompilersError(spack.error.SpackError): def __init__(self): super(NoCompilersError, self).__init__("Spack could not find any compilers!") + +class NoCompilerForSpecError(spack.error.SpackError): + def __init__(self, compiler_spec, target): + super(NoCompilerForSpecError, self).__init__("No compilers for target %s satisfy spec %s", + compiler_spec, target) + +class CompilerSpecInsufficientlySpecificError(spack.error.SpackError): + def __init__(self, compiler_spec): + super(CompilerSpecInsufficientlySpecificError, self).__init__("Multiple compilers satisfy spec %s", + compiler_spec) From 2b4dd8b9af427e2fc3c4a743f4f058a26c7d583f Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 8 Jan 2016 16:30:27 -0800 Subject: [PATCH 095/481] Fixed target satisfaction and updated tests accordingly --- lib/spack/spack/concretize.py | 35 +--------------- lib/spack/spack/spec.py | 35 ++++++++++++++++ lib/spack/spack/test/multimethod.py | 20 ++++----- lib/spack/spack/test/spec_dag.py | 10 +++-- lib/spack/spack/test/spec_semantics.py | 41 ++++++++++++------- .../mock_packages/multimethod/package.py | 23 ++++------- 6 files changed, 85 insertions(+), 79 deletions(-) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index ce86786004..43637ed468 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -209,37 +209,6 @@ def concretize_version(self, spec): return True # Things changed - def class_from_platform_name(self, platform_name): - file_path = join_path(spack.platform_path, platform_name) - platform_mod = imp.load_source('spack.platforms', file_path + '.py') - cls = getattr(platform_mod, mod_to_class(platform_name)) - - return cls - - def spec_add_target_from_string(self, spec, target): - """If only a target is provided, spack will assume the default architecture. - A platform-target pair can be input delimited by a '-'. If either portion of - a platform-target pair is empty, spack will supply a default, in the case of - a blank target the default will be dependent on the platform. - E.g. x86_64 -> 64 bit x86 - bgq- -> default bgq target (back end/powerpc) - cray-hawswell -> haswell target on cray platform - """ - if '-' in target: - platform, target = target.split('-') - else: - platform = '' - - if platform != '': - cls = self.class_from_platform_name(platform) - platform = cls() - else: - platform = spack.architecture.sys_type() - if target != '': - spec.target = platform.target(target) - else: - spec.target = platform.target('default') - def concretize_target(self, spec): """If the spec already has an target and it is a an target type, return. Otherwise, if it has a target that is a string type, generate a @@ -251,14 +220,14 @@ def concretize_target(self, spec): if isinstance(spec.target,spack.architecture.Target): return False else: - self.spec_add_target_from_string(spec, spec.target) + spec.add_target_from_string(spec, spec.target) return True #changed if spec.root.target: if isinstance(spec.root.target,spack.architecture.Target): spec.target = spec.root.target else: - self.spec_add_target_from_string(spec, spec.root.target) + spec.add_target_from_string(spec, spec.root.target) else: platform = spack.architecture.sys_type() spec.target = platform.target('default') diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 9e011bfb9f..8f90cc0d7f 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1228,6 +1228,36 @@ def _autospec(self, spec_like): return parse_anonymous_spec(spec_like, self.name) + def add_target_from_string(self, target): + """If only a target is provided, spack will assume the default architecture. + A platform-target pair can be input delimited by a '-'. If either portion of + a platform-target pair is empty, spack will supply a default, in the case of + a blank target the default will be dependent on the platform. + E.g. x86_64 -> 64 bit x86 + bgq- -> default bgq target (back end/powerpc) + cray-hawswell -> haswell target on cray platform + """ + if target is None: + return + if '-' in target: + platform, target = target.split('-') + else: + platform = '' + + if platform != '': + # Find the class for the platform name given + file_path = join_path(spack.platform_path, platform_name) + platform_mod = imp.load_source('spack.platforms', file_path + '.py') + cls = getattr(platform_mod, mod_to_class(platform_name)) + platform = cls() + else: + platform = spack.architecture.sys_type() + if target != '': + self.target = platform.target(target) + else: + self.target = platform.target('default') + + def satisfies(self, other, deps=True, strict=False): """Determine if this spec satisfies all constraints of another. @@ -1275,6 +1305,11 @@ def satisfies(self, other, deps=True, strict=False): # Target satisfaction is currently just class equality. # If not strict, None means unconstrained. + if not isinstance(self.target, spack.architecture.Target): + self.add_target_from_string(self.target) + if not isinstance(other.target, spack.architecture.Target): + other.add_target_from_string(other.target) + if self.target and other.target: if self.target != other.target: return False diff --git a/lib/spack/spack/test/multimethod.py b/lib/spack/spack/test/multimethod.py index 741dc96072..c651b7ad4a 100644 --- a/lib/spack/spack/test/multimethod.py +++ b/lib/spack/spack/test/multimethod.py @@ -92,22 +92,16 @@ def test_default_works(self): def test_target_match(self): - pkg = spack.db.get('multimethod=x86_64') - self.assertEqual(pkg.different_by_target(), 'x86_64') + platform = spack.architecture.sys_type() + targets = platform.targets.values() + for target in targets[:-1]: + print target + pkg = spack.db.get('multimethod='+target.name) + self.assertEqual(pkg.different_by_target(), target.name) - pkg = spack.db.get('multimethod=ppc64') - self.assertEqual(pkg.different_by_target(), 'ppc64') - - pkg = spack.db.get('multimethod=ppc32') - self.assertEqual(pkg.different_by_target(), 'ppc32') - - pkg = spack.db.get('multimethod=arm64') - self.assertEqual(pkg.different_by_target(), 'arm64') - - pkg = spack.db.get('multimethod=macos') + pkg = spack.db.get('multimethod='+targets[-1].name) self.assertRaises(NoSuchMethodError, pkg.different_by_target) - def test_dependency_match(self): pkg = spack.db.get('multimethod^zmpi') self.assertEqual(pkg.different_by_dep(), 'zmpi') diff --git a/lib/spack/spack/test/spec_dag.py b/lib/spack/spack/test/spec_dag.py index 6a2dd6140f..d116454d5c 100644 --- a/lib/spack/spack/test/spec_dag.py +++ b/lib/spack/spack/test/spec_dag.py @@ -241,9 +241,13 @@ def test_unsatisfiable_compiler_version(self): def test_unsatisfiable_target(self): - set_pkg_dep('mpileaks', 'mpich=bgqos_0') - spec = Spec('mpileaks ^mpich=sles_10_ppc64 ^callpath ^dyninst ^libelf ^libdwarf') - self.assertRaises(spack.spec.UnsatisfiableTargetSpecError, spec.normalize) + platform = spack.architecture.sys_type() + if len(platform.targets) > 1: + first = platform.targets.values()[0].name + second = platform.targets.values()[1].name + set_pkg_dep('mpileaks', 'mpich='+first) + spec = Spec('mpileaks ^mpich='+ second +' ^callpath ^dyninst ^libelf ^libdwarf') + self.assertRaises(spack.spec.UnsatisfiableTargetSpecError, spec.normalize) def test_invalid_dep(self): diff --git a/lib/spack/spack/test/spec_semantics.py b/lib/spack/spack/test/spec_semantics.py index b8b4fb951c..ef4db3fe65 100644 --- a/lib/spack/spack/test/spec_semantics.py +++ b/lib/spack/spack/test/spec_semantics.py @@ -111,12 +111,13 @@ def test_satisfies_compiler_version(self): def test_satisfies_target(self): - self.check_satisfies('foo=chaos_5_x86_64_ib', '=chaos_5_x86_64_ib') - self.check_satisfies('foo=bgqos_0', '=bgqos_0') - - self.check_unsatisfiable('foo=bgqos_0', '=chaos_5_x86_64_ib') - self.check_unsatisfiable('foo=chaos_5_x86_64_ib', '=bgqos_0') + platform = spack.architecture.sys_type() + targets = platform.targets.values() + for target in targets: + self.check_satisfies('foo='+target.name, '='+target.name) + for i in range(1,len(targets)): + self.check_unsatisfiable('foo='+targets[i-1].name, '='+targets[i].name) def test_satisfies_dependencies(self): self.check_satisfies('mpileaks^mpich', '^mpich') @@ -267,13 +268,15 @@ def test_constrain_variants(self): def test_constrain_target(self): - self.check_constrain('libelf=bgqos_0', 'libelf=bgqos_0', 'libelf=bgqos_0') - self.check_constrain('libelf=bgqos_0', 'libelf', 'libelf=bgqos_0') + platform = spack.architecture.sys_type() + target = platform.target('default').name + self.check_constrain('libelf='+target, 'libelf='+target, 'libelf='+target) + self.check_constrain('libelf='+target, 'libelf', 'libelf='+target) def test_constrain_compiler(self): - self.check_constrain('libelf=bgqos_0', 'libelf=bgqos_0', 'libelf=bgqos_0') - self.check_constrain('libelf=bgqos_0', 'libelf', 'libelf=bgqos_0') + self.check_constrain('libelf%intel', 'libelf%intel', 'libelf%intel') + self.check_constrain('libelf%intel', 'libelf', 'libelf%intel') def test_invalid_constraint(self): @@ -283,7 +286,10 @@ def test_invalid_constraint(self): self.check_invalid_constraint('libelf+debug', 'libelf~debug') self.check_invalid_constraint('libelf+debug~foo', 'libelf+debug+foo') - self.check_invalid_constraint('libelf=bgqos_0', 'libelf=x86_54') + platform = spack.architecture.sys_type() + targets = platform.targets.values() + if len(targets) > 1: + self.check_invalid_constraint('libelf='+targets[0].name, 'libelf='+targets[1].name) def test_constrain_changed(self): @@ -293,7 +299,8 @@ def test_constrain_changed(self): self.check_constrain_changed('libelf%gcc', '%gcc@4.5') self.check_constrain_changed('libelf', '+debug') self.check_constrain_changed('libelf', '~debug') - self.check_constrain_changed('libelf', '=bgqos_0') + platform = spack.architecture.sys_type() + self.check_constrain_changed('libelf', '='+platform.target('default').name) def test_constrain_not_changed(self): @@ -304,7 +311,9 @@ def test_constrain_not_changed(self): self.check_constrain_not_changed('libelf%gcc@4.5', '%gcc@4.5') self.check_constrain_not_changed('libelf+debug', '+debug') self.check_constrain_not_changed('libelf~debug', '~debug') - self.check_constrain_not_changed('libelf=bgqos_0', '=bgqos_0') + platform = spack.architecture.sys_type() + default = platform.target('default').name + self.check_constrain_not_changed('libelf='+default, '='+default) self.check_constrain_not_changed('libelf^foo', 'libelf^foo') self.check_constrain_not_changed('libelf^foo^bar', 'libelf^foo^bar') @@ -316,7 +325,9 @@ def test_constrain_dependency_changed(self): self.check_constrain_changed('libelf^foo%gcc', 'libelf^foo%gcc@4.5') self.check_constrain_changed('libelf^foo', 'libelf^foo+debug') self.check_constrain_changed('libelf^foo', 'libelf^foo~debug') - self.check_constrain_changed('libelf^foo', 'libelf^foo=bgqos_0') + platform = spack.architecture.sys_type() + default = platform.target('default').name + self.check_constrain_changed('libelf^foo', 'libelf^foo='+default) def test_constrain_dependency_not_changed(self): @@ -326,5 +337,7 @@ def test_constrain_dependency_not_changed(self): self.check_constrain_not_changed('libelf^foo%gcc@4.5', 'libelf^foo%gcc@4.5') self.check_constrain_not_changed('libelf^foo+debug', 'libelf^foo+debug') self.check_constrain_not_changed('libelf^foo~debug', 'libelf^foo~debug') - self.check_constrain_not_changed('libelf^foo=bgqos_0', 'libelf^foo=bgqos_0') + platform = spack.architecture.sys_type() + default = platform.target('default').name + self.check_constrain_not_changed('libelf^foo='+default, 'libelf^foo='+default) diff --git a/var/spack/mock_packages/multimethod/package.py b/var/spack/mock_packages/multimethod/package.py index f78ef3bb3d..0a1e991d37 100644 --- a/var/spack/mock_packages/multimethod/package.py +++ b/var/spack/mock_packages/multimethod/package.py @@ -23,7 +23,7 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## from spack import * - +import spack.architecture class Multimethod(Package): """This package is designed for use with Spack's multimethod test. @@ -103,21 +103,12 @@ def has_a_default(self): # # Make sure we can switch methods on different target # - @when('=x86_64') - def different_by_target(self): - return 'x86_64' - - @when('=ppc64') - def different_by_target(self): - return 'ppc64' - - @when('=ppc32') - def different_by_target(self): - return 'ppc32' - - @when('=arm64') - def different_by_target(self): - return 'arm64' + platform = spack.architecture.sys_type() + targets = platform.targets.values() + for target in targets[:-1]: + @when('='+target.name) + def different_by_target(self): + return self.spec.target.name # From 2b2d4bae4e48151ae17c0f2f0514873b1247ff15 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 8 Jan 2016 17:38:37 -0800 Subject: [PATCH 096/481] Fixing multimethod test for new platforms --- lib/spack/spack/test/multimethod.py | 6 +++++- var/spack/mock_packages/multimethod/package.py | 10 ++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/test/multimethod.py b/lib/spack/spack/test/multimethod.py index c651b7ad4a..116cb2d2ff 100644 --- a/lib/spack/spack/test/multimethod.py +++ b/lib/spack/spack/test/multimethod.py @@ -100,7 +100,11 @@ def test_target_match(self): self.assertEqual(pkg.different_by_target(), target.name) pkg = spack.db.get('multimethod='+targets[-1].name) - self.assertRaises(NoSuchMethodError, pkg.different_by_target) + if len(targets) == 1: + self.assertEqual(pkg.different_by_target(), targets[-1].name) + else: + self.assertRaises(NoSuchMethodError, pkg.different_by_target) + def test_dependency_match(self): pkg = spack.db.get('multimethod^zmpi') diff --git a/var/spack/mock_packages/multimethod/package.py b/var/spack/mock_packages/multimethod/package.py index 0a1e991d37..4e9a2438a7 100644 --- a/var/spack/mock_packages/multimethod/package.py +++ b/var/spack/mock_packages/multimethod/package.py @@ -22,6 +22,9 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +import imp +from llnl.util.filesystem import join_path +from spack.util.naming import mod_to_class from spack import * import spack.architecture @@ -103,8 +106,15 @@ def has_a_default(self): # # Make sure we can switch methods on different target # +# for platform_name in ['cray_xc', 'darwin', 'linux']: +# file_path = join_path(spack.platform_path, platform_name) +# platform_mod = imp.load_source('spack.platforms', file_path + '.py') +# cls = getattr(platform_mod, mod_to_class(platform_name)) + +# platform = cls() platform = spack.architecture.sys_type() targets = platform.targets.values() + for target in targets[:-1]: @when('='+target.name) def different_by_target(self): From 6e5238d037aebe4e610a16f200d860cee1a1355d Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 8 Jan 2016 17:44:16 -0800 Subject: [PATCH 097/481] Fixing the fix and removing debug printing --- lib/spack/spack/test/multimethod.py | 1 - var/spack/mock_packages/multimethod/package.py | 4 +++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/test/multimethod.py b/lib/spack/spack/test/multimethod.py index 116cb2d2ff..9d9f39e87d 100644 --- a/lib/spack/spack/test/multimethod.py +++ b/lib/spack/spack/test/multimethod.py @@ -95,7 +95,6 @@ def test_target_match(self): platform = spack.architecture.sys_type() targets = platform.targets.values() for target in targets[:-1]: - print target pkg = spack.db.get('multimethod='+target.name) self.assertEqual(pkg.different_by_target(), target.name) diff --git a/var/spack/mock_packages/multimethod/package.py b/var/spack/mock_packages/multimethod/package.py index 4e9a2438a7..b37db9f1eb 100644 --- a/var/spack/mock_packages/multimethod/package.py +++ b/var/spack/mock_packages/multimethod/package.py @@ -114,8 +114,10 @@ def has_a_default(self): # platform = cls() platform = spack.architecture.sys_type() targets = platform.targets.values() + if len(targets) > 1: + targets = targets[:-1] - for target in targets[:-1]: + for target in targets: @when('='+target.name) def different_by_target(self): return self.spec.target.name From 41046499c69150a7a81518f2ffba3405de0d6577 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 8 Jan 2016 18:51:38 -0800 Subject: [PATCH 098/481] minor bug chasing --- lib/spack/env/cc | 16 ---------------- lib/spack/spack/compilers/__init__.py | 4 ++-- lib/spack/spack/concretize.py | 4 ++-- 3 files changed, 4 insertions(+), 20 deletions(-) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index b6c6e03e42..1704ae3280 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -71,22 +71,6 @@ for param in $parameters; do done -# -# Cray module environment-related stuff. -# -if [ ! -z "$SPACK_CRAYPE" ]; then - cur_pe=$(module list 2>&1 | grep PrgEnv | grep -o 'PrgEnv-[^/]*') - if [ ! -z "$cur_pe" ]; then - module swap $cur_pe $SPACK_CRAYPE - else - module load $SPACK_CRAYPE - fi -fi - -if [ ! -z "$SPACK_COMP_MODULE" ]; then - module load $SPACK_COMP_MODULE -fi - # # Figure out the type of compiler, the language, and the mode so that # the compiler script knows what to do. diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 3a0a88b5e3..e38ef949d7 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -259,8 +259,8 @@ def __init__(self): class NoCompilerForSpecError(spack.error.SpackError): def __init__(self, compiler_spec, target): - super(NoCompilerForSpecError, self).__init__("No compilers for target %s satisfy spec %s", - compiler_spec, target) + super(NoCompilerForSpecError, self).__init__("No compilers for target %s satisfy spec %s" % ( + target, compiler_spec)) class CompilerSpecInsufficientlySpecificError(spack.error.SpackError): def __init__(self, compiler_spec): diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 43637ed468..b8057cd46c 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -220,14 +220,14 @@ def concretize_target(self, spec): if isinstance(spec.target,spack.architecture.Target): return False else: - spec.add_target_from_string(spec, spec.target) + spec.add_target_from_string(spec.target) return True #changed if spec.root.target: if isinstance(spec.root.target,spack.architecture.Target): spec.target = spec.root.target else: - spec.add_target_from_string(spec, spec.root.target) + spec.add_target_from_string(spec.root.target) else: platform = spack.architecture.sys_type() spec.target = platform.target('default') From 9615efd940c425b049a142f8c22545c9a04124df Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 8 Jan 2016 18:53:19 -0800 Subject: [PATCH 099/481] cleanup: Removing an unnecessary line in an otherwise unchanged file --- lib/spack/env/cc | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index 1704ae3280..fa85bb595e 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -70,7 +70,6 @@ for param in $parameters; do fi done - # # Figure out the type of compiler, the language, and the mode so that # the compiler script knows what to do. From aab1a67d056bf9f0979f2cebfff607d05ac8b7e9 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Sat, 9 Jan 2016 09:42:36 -0800 Subject: [PATCH 100/481] Adding needed import modules. Changed platform_name to platform to prevent syntax errors --- lib/spack/spack/spec.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 8f90cc0d7f..264e0c0506 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -91,6 +91,7 @@ expansion when it is the first character in an id typed on the command line. """ import sys +import imp import itertools import hashlib import base64 @@ -100,6 +101,7 @@ from external.yaml.error import MarkedYAMLError import llnl.util.tty as tty +from llnl.util.filesystem import join_path from llnl.util.lang import * from llnl.util.tty.color import * @@ -109,6 +111,7 @@ import spack.compilers as compilers from spack.version import * +from spack.util.naming import mod_to_class from spack.util.string import * from spack.util.prefix import Prefix from spack.virtual import ProviderIndex @@ -1246,9 +1249,9 @@ def add_target_from_string(self, target): if platform != '': # Find the class for the platform name given - file_path = join_path(spack.platform_path, platform_name) + file_path = join_path(spack.platform_path, platform) platform_mod = imp.load_source('spack.platforms', file_path + '.py') - cls = getattr(platform_mod, mod_to_class(platform_name)) + cls = getattr(platform_mod, mod_to_class(platform)) platform = cls() else: platform = spack.architecture.sys_type() From 15713219e57a97048f4e3b085f9c75e188a67787 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 20 Jan 2016 10:32:56 -0800 Subject: [PATCH 101/481] Better regular expression searching. Tested on edison was finding test compilers and then spack was giving multiple compiler match errors --- lib/spack/spack/compiler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index 225c65917a..1a5be3c3fe 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -283,7 +283,7 @@ def find_in_modules(cls): modulecmd = which('modulecmd') modulecmd.add_default_arg('python') output = modulecmd('avail', cls.PrgEnv_compiler, return_oe=True) - matches = re.findall(r'(%s)/(\d+[\.\d]+)' % cls.PrgEnv_compiler, output) + matches = re.findall(r'(%s)/([\d\.]+[\d$])' % cls.PrgEnv_compiler, output) for name, version in matches: v = version From 840b41c450267764d021456875dbe87af52c2176 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 20 Jan 2016 10:36:15 -0800 Subject: [PATCH 102/481] Removed unneccessary $. Still learning regexp --- lib/spack/spack/compiler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index 1a5be3c3fe..46f2dfaec7 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -283,7 +283,7 @@ def find_in_modules(cls): modulecmd = which('modulecmd') modulecmd.add_default_arg('python') output = modulecmd('avail', cls.PrgEnv_compiler, return_oe=True) - matches = re.findall(r'(%s)/([\d\.]+[\d$])' % cls.PrgEnv_compiler, output) + matches = re.findall(r'(%s)/([\d\.]+[\d])' % cls.PrgEnv_compiler, output) for name, version in matches: v = version From 03585225338e2ccd61174800b42242c7f256f533 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Sat, 6 Feb 2016 15:41:22 -0800 Subject: [PATCH 103/481] new branch and also pushing some architecture changes where os is detected by linux and darwin and manually set by cray and bgq --- lib/spack/spack/architecture.py | 136 ++++++++++++++++++++------------ 1 file changed, 84 insertions(+), 52 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 6b9ae33b3e..4f36a3fa92 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -39,20 +39,25 @@ class InvalidSysTypeError(serr.SpackError): def __init__(self, sys_type): - super(InvalidSysTypeError, self).__init__("Invalid sys_type value for Spack: " + sys_type) + super(InvalidSysTypeError, self).__init__( + "Invalid sys_type value for Spack: " + sys_type) class NoSysTypeError(serr.SpackError): def __init__(self): - super(NoSysTypeError, self).__init__("Could not determine sys_type for this machine.") + super(NoSysTypeError, self).__init__( + "Could not determine sys_type for this machine.") @key_ordering class Target(object): - """ Target is the processor of the host machine. The host machine may have different front-end - and back-end targets, especially if it is a Cray machine. The target will have a name and - also the module_name (e.g craype-compiler). Targets will also recognize which platform - they came from using the set_platform method. Targets will have compiler finding strategies + """ Target is the processor of the host machine. + The host machine may have different front-end + and back-end targets, especially if it is a Cray machine. + The target will have a name and module_name (e.g craype-compiler). + Targets will also recognize which platform + they came from using the set_platform method. + Targets will have compiler finding strategies """ def __init__(self, name, compiler_strategy, module_name=None): @@ -60,10 +65,13 @@ def __init__(self, name, compiler_strategy, module_name=None): self.compiler_strategy = compiler_strategy self.module_name = module_name # craype-ivybridge - # Sets only the platform name to avoid recursiveness + # Sets only the platform name to avoid recursiveness def set_platform(self, platform): self.platform_name = platform.name + def set_operating_system(self, operating_sys): + self.platform_os = operating_sys + def to_dict(self): d = {} d['name'] = self.name @@ -71,6 +79,7 @@ def to_dict(self): d['module_name'] = self.module_name if self.platform_name: d['platform'] = self.platform_name + return d @staticmethod @@ -87,14 +96,16 @@ def from_dict(d): def _cmp_key(self): - return (self.name, self.compiler_strategy, self.module_name) + return (self.name, self.compiler_strategy, + self.module_name, self.platform_os) def __repr__(self): return self.__str__() def __str__(self): - if self.platform_name: - return self.platform_name + '-' + self.name + if self.platform_name and self.platform_os: + return (self.platform_name + '-' + + self.platform_os + '-' + self.name) return self.name @key_ordering @@ -105,26 +116,38 @@ class Platform(object): """ priority = None # Subclass needs to set this number. This controls order in which platform is detected. + front_end = None back_end = None default = None # The default back end target. On cray ivybridge + + front_os = None + back_os = None + default_os = None def __init__(self, name): self.targets = {} self.name = name def add_target(self, name, target): - """Used by the platform specific subclass to list available targets. Raises an error - if the platform specifies a name that is reserved by spack as an alias. + """Used by the platform specific subclass to list available targets. + Raises an error if the platform specifies a name + that is reserved by spack as an alias. """ if name in ['front_end', 'fe', 'back_end', 'be', 'default']: - raise ValueError("%s is a spack reserved alias and cannot be the name of a target" % name) + raise ValueError( + "%s is a spack reserved" \ + "alias and cannot be the name of a target" + % name) + + target.set_operating_system(self.platform_os()) target.set_platform(self) self.targets[name] = target def target(self, name): - """This is a getter method for the target dictionary that handles defaulting based - on the values provided by default, front-end, and back-end. This can be overwritten + """This is a getter method for the target dictionary that + handles defaulting based on the values provided by default, + front-end, and back-end. This can be overwritten by a subclass for which we want to provide further aliasing options. """ if name == 'default': @@ -135,6 +158,51 @@ def target(self, name): name = self.back_end return self.targets[name] + + def _detect_linux_os(self): + """ If it is one a linux machine use the python method platform.dist() + """ + os_name = py_platform.dist()[0] + version = py_platform.dist()[1] + a return os_name + version + + def _detect_mac_os(self): + """If it is on a mac machine then use the python method platform.mac_ver + """ + mac_releases = {'10.6' : 'snowleopard', '10.7' : 'lion', + '10.8' : 'mountainlion', '10.9' : 'mavericks', + '10.10' : 'yosemite', '10.11' : 'elcapitan'} + + mac_ver = py_platform.mac_ver() + try: + os_name = mac_releases[mac_ver] + mac_ver = Version(mac_ver) + + except KeyError: + os_name = 'mac_os' + + return os_name + + def set_os(self): + """ Set the OS according to the platform it is on. Darwin and Linux + will simply be an auto-detected linux distro or mac release. The + special cases will be for Cray and BGQ machines which have two + different OS for login and compute nodes. The client should provide + the name and major version of the operating system + """ + if self.name == 'darwin': + self.default_os = self._detect_mac_os() + else: + self.default_os = self._detect_linux_os() + + def platform_os(self, name=None): + """ Get the platform operating system from the platform """ + if name == 'front_os': + return self.front_os + elif name == 'back_os': + return self.back_os + else: + return self.default_os @classmethod def detect(self): @@ -144,6 +212,7 @@ def detect(self): """ raise NotImplementedError() + def __repr__(self): return self.__str__() @@ -153,43 +222,6 @@ def __str__(self): def _cmp_key(self): return (self.name, (_cmp_key(t) for t in self.targets.values())) -def get_sys_type_from_spack_globals(): - """Return the SYS_TYPE from spack globals, or None if it isn't set.""" - if not hasattr(spack, "sys_type"): - return None - elif hasattr(spack.sys_type, "__call__"): - return spack.sys_type() #If in __init__.py there is a sys_type() then call that - else: - return spack.sys_type # Else use the attributed which defaults to None - - -# This is livermore dependent. Hard coded for livermore -#def get_sys_type_from_environment(): -# """Return $SYS_TYPE or None if it's not defined.""" -# return os.environ.get('SYS_TYPE') - - -def get_mac_sys_type(): - """Return a Mac OS SYS_TYPE or None if this isn't a mac. - Front-end config - """ - mac_ver = py_platform.mac_ver()[0] - if not mac_ver: - return None - return "macosx_%s_%s" % (Version(mac_ver).up_to(2), py_platform.machine()) - - -def get_sys_type_from_uname(): - """ Returns a sys_type from the uname argument - Front-end config - """ - try: - platform_proc = subprocess.Popen(['uname', '-i'], stdout = subprocess.PIPE) - platform, _ = platform_proc.communicate() - return platform.strip() - except: - return None - @memoized def all_platforms(): modules = [] From 21a5a3404134c1676a7aa498ec201ed911e80029 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 10 Feb 2016 10:59:16 -0800 Subject: [PATCH 104/481] Revert "new branch and also pushing some architecture changes where os is detected by linux and darwin and manually set by cray and bgq" This reverts commit 70088be24b2ed34076f7f5292a2a465a8655a886. --- lib/spack/spack/architecture.py | 136 ++++++++++++-------------------- 1 file changed, 52 insertions(+), 84 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 4f36a3fa92..6b9ae33b3e 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -39,25 +39,20 @@ class InvalidSysTypeError(serr.SpackError): def __init__(self, sys_type): - super(InvalidSysTypeError, self).__init__( - "Invalid sys_type value for Spack: " + sys_type) + super(InvalidSysTypeError, self).__init__("Invalid sys_type value for Spack: " + sys_type) class NoSysTypeError(serr.SpackError): def __init__(self): - super(NoSysTypeError, self).__init__( - "Could not determine sys_type for this machine.") + super(NoSysTypeError, self).__init__("Could not determine sys_type for this machine.") @key_ordering class Target(object): - """ Target is the processor of the host machine. - The host machine may have different front-end - and back-end targets, especially if it is a Cray machine. - The target will have a name and module_name (e.g craype-compiler). - Targets will also recognize which platform - they came from using the set_platform method. - Targets will have compiler finding strategies + """ Target is the processor of the host machine. The host machine may have different front-end + and back-end targets, especially if it is a Cray machine. The target will have a name and + also the module_name (e.g craype-compiler). Targets will also recognize which platform + they came from using the set_platform method. Targets will have compiler finding strategies """ def __init__(self, name, compiler_strategy, module_name=None): @@ -65,13 +60,10 @@ def __init__(self, name, compiler_strategy, module_name=None): self.compiler_strategy = compiler_strategy self.module_name = module_name # craype-ivybridge - # Sets only the platform name to avoid recursiveness + # Sets only the platform name to avoid recursiveness def set_platform(self, platform): self.platform_name = platform.name - def set_operating_system(self, operating_sys): - self.platform_os = operating_sys - def to_dict(self): d = {} d['name'] = self.name @@ -79,7 +71,6 @@ def to_dict(self): d['module_name'] = self.module_name if self.platform_name: d['platform'] = self.platform_name - return d @staticmethod @@ -96,16 +87,14 @@ def from_dict(d): def _cmp_key(self): - return (self.name, self.compiler_strategy, - self.module_name, self.platform_os) + return (self.name, self.compiler_strategy, self.module_name) def __repr__(self): return self.__str__() def __str__(self): - if self.platform_name and self.platform_os: - return (self.platform_name + '-' + - self.platform_os + '-' + self.name) + if self.platform_name: + return self.platform_name + '-' + self.name return self.name @key_ordering @@ -116,38 +105,26 @@ class Platform(object): """ priority = None # Subclass needs to set this number. This controls order in which platform is detected. - front_end = None back_end = None default = None # The default back end target. On cray ivybridge - - front_os = None - back_os = None - default_os = None def __init__(self, name): self.targets = {} self.name = name def add_target(self, name, target): - """Used by the platform specific subclass to list available targets. - Raises an error if the platform specifies a name - that is reserved by spack as an alias. + """Used by the platform specific subclass to list available targets. Raises an error + if the platform specifies a name that is reserved by spack as an alias. """ if name in ['front_end', 'fe', 'back_end', 'be', 'default']: - raise ValueError( - "%s is a spack reserved" \ - "alias and cannot be the name of a target" - % name) - - target.set_operating_system(self.platform_os()) + raise ValueError("%s is a spack reserved alias and cannot be the name of a target" % name) target.set_platform(self) self.targets[name] = target def target(self, name): - """This is a getter method for the target dictionary that - handles defaulting based on the values provided by default, - front-end, and back-end. This can be overwritten + """This is a getter method for the target dictionary that handles defaulting based + on the values provided by default, front-end, and back-end. This can be overwritten by a subclass for which we want to provide further aliasing options. """ if name == 'default': @@ -158,51 +135,6 @@ def target(self, name): name = self.back_end return self.targets[name] - - def _detect_linux_os(self): - """ If it is one a linux machine use the python method platform.dist() - """ - os_name = py_platform.dist()[0] - version = py_platform.dist()[1] - a return os_name + version - - def _detect_mac_os(self): - """If it is on a mac machine then use the python method platform.mac_ver - """ - mac_releases = {'10.6' : 'snowleopard', '10.7' : 'lion', - '10.8' : 'mountainlion', '10.9' : 'mavericks', - '10.10' : 'yosemite', '10.11' : 'elcapitan'} - - mac_ver = py_platform.mac_ver() - try: - os_name = mac_releases[mac_ver] - mac_ver = Version(mac_ver) - - except KeyError: - os_name = 'mac_os' - - return os_name - - def set_os(self): - """ Set the OS according to the platform it is on. Darwin and Linux - will simply be an auto-detected linux distro or mac release. The - special cases will be for Cray and BGQ machines which have two - different OS for login and compute nodes. The client should provide - the name and major version of the operating system - """ - if self.name == 'darwin': - self.default_os = self._detect_mac_os() - else: - self.default_os = self._detect_linux_os() - - def platform_os(self, name=None): - """ Get the platform operating system from the platform """ - if name == 'front_os': - return self.front_os - elif name == 'back_os': - return self.back_os - else: - return self.default_os @classmethod def detect(self): @@ -212,7 +144,6 @@ def detect(self): """ raise NotImplementedError() - def __repr__(self): return self.__str__() @@ -222,6 +153,43 @@ def __str__(self): def _cmp_key(self): return (self.name, (_cmp_key(t) for t in self.targets.values())) +def get_sys_type_from_spack_globals(): + """Return the SYS_TYPE from spack globals, or None if it isn't set.""" + if not hasattr(spack, "sys_type"): + return None + elif hasattr(spack.sys_type, "__call__"): + return spack.sys_type() #If in __init__.py there is a sys_type() then call that + else: + return spack.sys_type # Else use the attributed which defaults to None + + +# This is livermore dependent. Hard coded for livermore +#def get_sys_type_from_environment(): +# """Return $SYS_TYPE or None if it's not defined.""" +# return os.environ.get('SYS_TYPE') + + +def get_mac_sys_type(): + """Return a Mac OS SYS_TYPE or None if this isn't a mac. + Front-end config + """ + mac_ver = py_platform.mac_ver()[0] + if not mac_ver: + return None + return "macosx_%s_%s" % (Version(mac_ver).up_to(2), py_platform.machine()) + + +def get_sys_type_from_uname(): + """ Returns a sys_type from the uname argument + Front-end config + """ + try: + platform_proc = subprocess.Popen(['uname', '-i'], stdout = subprocess.PIPE) + platform, _ = platform_proc.communicate() + return platform.strip() + except: + return None + @memoized def all_platforms(): modules = [] From 2650c60374a0857679d52f98fc5f94b23c7d6660 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 10 Feb 2016 16:14:50 -0800 Subject: [PATCH 105/481] Added operating system class prototype and some autodetect features for operating system --- lib/spack/spack/architecture.py | 61 ++++++++++++++++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 6b9ae33b3e..2914d36ba2 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -109,8 +109,13 @@ class Platform(object): back_end = None default = None # The default back end target. On cray ivybridge + front_os = None + back_os = None + default_os = None + def __init__(self, name): self.targets = {} + self.operating_sys = {} self.name = name def add_target(self, name, target): @@ -136,10 +141,49 @@ def target(self, name): return self.targets[name] + def _detect_linux_os(self): + return OperatingSystem(py_platform.dist()[0], py_platform.dist()[1]) + + def _detect_mac_os(self): + mac_releases = {'10.6': "snowleopard", + "10.7": "lion", + "10.8": "mountainlion", + "10.9": "mavericks", + "10.10": "yosemite", + "10.11": "elcapitan"} + mac_ver = py_platform.mac_ver()[:-2] + try: + os_name = mac_releases[mac_ver] + return OperatingSystem(os_name, mac_ver) + except KeyError: + os_name = "mac_os" + return OperatingSystem(os_name, mac_ver) + + def add_operating_system(self, name=None, operating_system=None): + if self.name == 'linux': + linux_os = self._detect_linux_os() + self.operating_sys[linux_os.name] = linux_os + elif self.name == 'darwin': + mac_os = self._detect_mac_os() + self.operating_sys[mac_os.name] = mac_os + else: + self.operating_sys[name] = operating_system + + def operating_system(self, name): + if name == 'default': + name = self.default_os + if name == 'front_os': + name = self.front_os + if name == 'back_os': + name = self.back_os + + return self.operating_sys[name] + @classmethod def detect(self): """ Subclass is responsible for implementing this method. - Returns True if the Platform class detects that it is the current platform + Returns True if the Platform class detects that + it is the current platform and False if it's not. """ raise NotImplementedError() @@ -153,6 +197,21 @@ def __str__(self): def _cmp_key(self): return (self.name, (_cmp_key(t) for t in self.targets.values())) + +class OperatingSystem(object): + """ The operating system will contain a name and version. It will + also represent itself simple with it's name + """ + def __init__(self, name, version): + self.name = name + self.version = version + + def __str__(self): + return self.name + + def __repr__(self): + return self.__str__() + def get_sys_type_from_spack_globals(): """Return the SYS_TYPE from spack globals, or None if it isn't set.""" if not hasattr(spack, "sys_type"): From d0ae6dd40159c1e3b043db254f56c8b7ca22294c Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 10 Feb 2016 16:15:25 -0800 Subject: [PATCH 106/481] Changed add_target_from_string to include methods to concretize operating_system --- lib/spack/spack/spec.py | 64 ++++++++++++++++++++--------------------- 1 file changed, 31 insertions(+), 33 deletions(-) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 264e0c0506..503f80e1e6 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -90,6 +90,7 @@ specs to avoid ambiguity. Both are provided because ~ can cause shell expansion when it is the first character in an id typed on the command line. """ +from collections import namedtuple import sys import imp import itertools @@ -459,7 +460,7 @@ def _set_target(self, target): """Called by the parser to set the target.""" if self.target: raise DuplicateTargetError( "Spec for '%s' cannot have two targets." % self.name) - self.target = target + self.target = target # a string can be set def _add_dependency(self, spec): @@ -1231,7 +1232,7 @@ def _autospec(self, spec_like): return parse_anonymous_spec(spec_like, self.name) - def add_target_from_string(self, target): + def add_target_from_string(self, arch): """If only a target is provided, spack will assume the default architecture. A platform-target pair can be input delimited by a '-'. If either portion of a platform-target pair is empty, spack will supply a default, in the case of @@ -1240,31 +1241,28 @@ def add_target_from_string(self, target): bgq- -> default bgq target (back end/powerpc) cray-hawswell -> haswell target on cray platform """ - if target is None: + Arch = namedtuple("Arch", "arch_os target") + platform = spack.architecture.sys_type() + + if arch is None: return - if '-' in target: - platform, target = target.split('-') - else: - platform = '' - - if platform != '': - # Find the class for the platform name given - file_path = join_path(spack.platform_path, platform) - platform_mod = imp.load_source('spack.platforms', file_path + '.py') - cls = getattr(platform_mod, mod_to_class(platform)) - platform = cls() - else: - platform = spack.architecture.sys_type() - if target != '': - self.target = platform.target(target) - else: - self.target = platform.target('default') + if '-' in arch: + os_name, target = arch.split('-') + self.target = Arch(arch_os=platform.operating_system(os_name), + target = platform.target(target)) + elif arch in platform.targets: + self.target = Arch(arch_os=platform.operating_system('default'), + target=platform.target(target)) + else: + os_name = arch # Odd naming here, definitely need to change + self.target = Arch(arch_os=platform.operating_system(os_name), + target=platform.target('default')) def satisfies(self, other, deps=True, strict=False): - """Determine if this spec satisfies all constraints of another. + """determine if this spec satisfies all constraints of another. - There are two senses for satisfies: + there are two senses for satisfies: * `loose` (default): the absence of a constraint in self implies that it *could* be satisfied by other, so we only @@ -1276,32 +1274,32 @@ def satisfies(self, other, deps=True, strict=False): """ other = self._autospec(other) - # A concrete provider can satisfy a virtual dependency. + # a concrete provider can satisfy a virtual dependency. if not self.virtual and other.virtual: pkg = spack.db.get(self.name) if pkg.provides(other.name): for provided, when_spec in pkg.provided.items(): - if self.satisfies(when_spec, deps=False, strict=strict): + if self.satisfies(when_spec, deps=false, strict=strict): if provided.satisfies(other): - return True - return False + return True + return False - # Otherwise, first thing we care about is whether the name matches + # otherwise, first thing we care about is whether the name matches if self.name != other.name: - return False + return False if self.versions and other.versions: if not self.versions.satisfies(other.versions, strict=strict): - return False + return False elif strict and (self.versions or other.versions): - return False + return False - # None indicates no constraints when not strict. + # none indicates no constraints when not strict. if self.compiler and other.compiler: if not self.compiler.satisfies(other.compiler, strict=strict): - return False + return False elif strict and (other.compiler and not self.compiler): - return False + return True if not self.variants.satisfies(other.variants, strict=strict): return False From fb234205c210930112f7fcd2c7c360e0c950d042 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 10 Feb 2016 16:16:11 -0800 Subject: [PATCH 107/481] Added method to concretize target and os pair --- lib/spack/spack/concretize.py | 75 ++++++++++++++++++++++++++++------- 1 file changed, 61 insertions(+), 14 deletions(-) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index b8057cd46c..d78c102d93 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -33,6 +33,7 @@ TODO: make this customizable and allow users to configure concretization policies. """ +import collections from llnl.util.filesystem import join_path import spack import spack.spec @@ -209,6 +210,22 @@ def concretize_version(self, spec): return True # Things changed + def _concretize_operating_system(self, arch, platform): + """ Future method for concretizing operating system """ + if isinstance(arch.arch_os, OperatingSystem): + return False + else: + arch.arch_os = platform.operating_system('default') + return True + + + def _concretize_arch_target(self, arch, platform): + if isinstance(arch.target, spack.architecture.Target): + return False + else: + arch.target = platform.target('default') + return True + def concretize_target(self, spec): """If the spec already has an target and it is a an target type, return. Otherwise, if it has a target that is a string type, generate a @@ -216,23 +233,53 @@ def concretize_target(self, spec): DAG has an target, then use that. Otherwise, take the system's default target. """ - if spec.target is not None: - if isinstance(spec.target,spack.architecture.Target): - return False - else: - spec.add_target_from_string(spec.target) - return True #changed + platform = spack.architecture.sys_type() - if spec.root.target: - if isinstance(spec.root.target,spack.architecture.Target): - spec.target = spec.root.target - else: - spec.add_target_from_string(spec.root.target) + if spec.target is None: + # Create an empty tuple + Arch = collections.namedtuple("Arch", "arch_os target") + spec.target = Arch(arch_os=platform.operating_system('default'), + target=platform.target('default')) + + return True + # If there is a target and it is a tuple and has both filled return + # False + if not isinstance(spec.target, basestring): + return any((self._concretize_operating_system(spec.target, platform), + self._concretize_arch_target(spec.target, plarform))) else: - platform = spack.architecture.sys_type() - spec.target = platform.target('default') + spec.add_target_from_string(spec.target) - return True #changed + if spec.root.target and \ + not isinstance(spec.root.target, basestring): + bool_flag = any( + (self._concretize_operating_system(spec.root.target, platform), + self._concretize_arch_target(spec.root.target, platform))) + spec.target =spec.root.target + return bool_flag + else: + spec.add_target_from_string(spec.root.target) + + + # if there is no target specified used the defaults + + #if spec.target is not None: + # if isinstance(spec.target,spack.architecture.Target): + # return False + # else: + # spec.add_target_from_string(spec.target) + # return True #changed + + #if spec.root.target: + # if isinstance(spec.root.target,spack.architecture.Target): + # spec.target = spec.root.target + # else: + # spec.add_target_from_string(spec.root.target) + #else: + # platform = spack.architecture.sys_type() + # spec.target = platform.target('default') + + #return True #changed def concretize_variants(self, spec): From cb8d5ab1837cfddefa31bd09592ca4ac999692dc Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 10 Feb 2016 16:16:35 -0800 Subject: [PATCH 108/481] Added new add_os method --- lib/spack/spack/platforms/cray_xc.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/platforms/cray_xc.py b/lib/spack/spack/platforms/cray_xc.py index e3adb182ea..b37d36a62c 100644 --- a/lib/spack/spack/platforms/cray_xc.py +++ b/lib/spack/spack/platforms/cray_xc.py @@ -1,6 +1,6 @@ import os -from spack.architecture import Platform, Target +from spack.architecture import Platform, Target, OperatingSystem class CrayXc(Platform): priority = 20 @@ -8,6 +8,10 @@ class CrayXc(Platform): back_end = 'ivybridge' default = 'ivybridge' + front_os = "SuSE11" + back_os = "GNU/Linux" + default_os = "GNU/Linux" + def __init__(self): ''' Since cori doesn't have ivybridge as a front end it's better if we use CRAY_CPU_TARGET as the default. This will ensure @@ -30,6 +34,9 @@ def __init__(self): self.add_target('sandybridge', Target('sandybridge', 'PATH')) self.add_target('ivybridge', Target('ivybridge', 'MODULES', 'craype-ivybridge')) self.add_target('haswell', Target('haswell', 'MODULES', 'craype-haswell')) + + self.add_operating_system('SuSE11', OperatingSystem('SuSE11', '11')) + self.add_operating_system('GNU/Linux', OperatingSystem('GNU/Linux', '10')) # self.add_target(self.front_end, Target(self.front_end, 'PATH')) # Back End compiler needs the proper target module loaded. From 77e93e1b791f1c264a999de3b59f2e6396e13391 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 10 Feb 2016 16:16:58 -0800 Subject: [PATCH 109/481] Added add_os method to platform subclasses --- lib/spack/spack/platforms/darwin.py | 1 + lib/spack/spack/platforms/linux.py | 1 + 2 files changed, 2 insertions(+) diff --git a/lib/spack/spack/platforms/darwin.py b/lib/spack/spack/platforms/darwin.py index 52284826b1..cfa1d957a1 100644 --- a/lib/spack/spack/platforms/darwin.py +++ b/lib/spack/spack/platforms/darwin.py @@ -10,6 +10,7 @@ class Darwin(Platform): def __init__(self): super(Darwin, self).__init__('darwin') self.add_target(self.default, Target(self.default, 'PATH')) + self.add_operating_system() @classmethod def detect(self): diff --git a/lib/spack/spack/platforms/linux.py b/lib/spack/spack/platforms/linux.py index 7f94d80c34..dcb08569a8 100644 --- a/lib/spack/spack/platforms/linux.py +++ b/lib/spack/spack/platforms/linux.py @@ -10,6 +10,7 @@ class Linux(Platform): def __init__(self): super(Linux, self).__init__('linux') self.add_target(self.default, Target(self.default, 'PATH')) + self.add_operating_system() @classmethod def detect(self): From b9d09202c9856bf173c4bbabd4b73573d355d408 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 11 Feb 2016 11:15:19 -0800 Subject: [PATCH 110/481] changed some variables to account for the fact that target is now a tuple --- lib/spack/spack/concretize.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index d78c102d93..8886b3cd93 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -212,7 +212,7 @@ def concretize_version(self, spec): def _concretize_operating_system(self, arch, platform): """ Future method for concretizing operating system """ - if isinstance(arch.arch_os, OperatingSystem): + if isinstance(arch.arch_os, spack.architecture.OperatingSystem): return False else: arch.arch_os = platform.operating_system('default') @@ -240,13 +240,12 @@ def concretize_target(self, spec): Arch = collections.namedtuple("Arch", "arch_os target") spec.target = Arch(arch_os=platform.operating_system('default'), target=platform.target('default')) - - return True + return True # If there is a target and it is a tuple and has both filled return # False if not isinstance(spec.target, basestring): return any((self._concretize_operating_system(spec.target, platform), - self._concretize_arch_target(spec.target, plarform))) + self._concretize_arch_target(spec.target, platform))) else: spec.add_target_from_string(spec.target) @@ -316,9 +315,9 @@ def concretize_compiler(self, spec): # Should think whether this can be more efficient def _proper_compiler_style(cspec, target): compilers = spack.compilers.compilers_for_spec(cspec) - if target.compiler_strategy == 'PATH': + if target.target.compiler_strategy == 'PATH': filter(lambda c: not c.modules, compilers) - if target.compiler_strategy == 'MODULES': + if target.target.compiler_strategy == 'MODULES': filter(lambda c: c.modules, compilers) return compilers From 3e1be63b0f8e745a04e041288a7583333d31673c Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 11 Feb 2016 11:15:50 -0800 Subject: [PATCH 111/481] Changed some variables to account for the fact that target is now a tuple --- lib/spack/spack/spec.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 503f80e1e6..3b2d54d047 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -566,7 +566,7 @@ def traverse(self, visited=None, d=0, **kwargs): in the traversal. root [=True] - If false, this won't yield the root node, just its descendents. + If False, this won't yield the root node, just its descendents. direction [=children|parents] If 'children', does a traversal of this spec's children. If @@ -661,7 +661,7 @@ def to_node_dict(self): for d in sorted(self.dependencies)) } if self.target: - d['target'] = self.target.to_dict() + d['target'] = self.target.target.to_dict() else: d['target'] = None if self.compiler: @@ -755,7 +755,6 @@ def _concretize_helper(self, presets=None, visited=None): if self.name in presets: changed |= self.constrain(presets[self.name]) - else: # Concretize virtual dependencies last. Because they're added # to presets below, their constraints will all be merged, but we'll @@ -830,6 +829,7 @@ def concretize(self): changed = True force = False + # Loops forever here in my implementation while changed: changes = (self.normalize(force=force), self._expand_virtual_packages(), @@ -1279,7 +1279,7 @@ def satisfies(self, other, deps=True, strict=False): pkg = spack.db.get(self.name) if pkg.provides(other.name): for provided, when_spec in pkg.provided.items(): - if self.satisfies(when_spec, deps=false, strict=strict): + if self.satisfies(when_spec, deps=False, strict=strict): if provided.satisfies(other): return True return False @@ -1306,9 +1306,9 @@ def satisfies(self, other, deps=True, strict=False): # Target satisfaction is currently just class equality. # If not strict, None means unconstrained. - if not isinstance(self.target, spack.architecture.Target): + if isinstance(self.target, basestring): self.add_target_from_string(self.target) - if not isinstance(other.target, spack.architecture.Target): + if isinstance(other.target, basestring): other.add_target_from_string(other.target) if self.target and other.target: @@ -1380,7 +1380,7 @@ def _dup(self, other, **kwargs): Options: dependencies[=True] - Whether deps should be copied too. Set to false to copy a + Whether deps should be copied too. Set to False to copy a spec but not its dependencies. """ From 8e8c63bd679b57a969274287d402ea9843ecd1ae Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 11 Feb 2016 11:47:39 -0800 Subject: [PATCH 112/481] Using pylint, fixed some of the indentation and spacing errors --- lib/spack/spack/architecture.py | 94 ++++++++++++--------------------- 1 file changed, 33 insertions(+), 61 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 2914d36ba2..c4ef0805ac 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -22,7 +22,6 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import os import imp import platform as py_platform import inspect @@ -34,31 +33,34 @@ import spack from spack.util.naming import mod_to_class import spack.error as serr -from spack.version import Version -from external import yaml + class InvalidSysTypeError(serr.SpackError): def __init__(self, sys_type): - super(InvalidSysTypeError, self).__init__("Invalid sys_type value for Spack: " + sys_type) + super(InvalidSysTypeError, self).__init__( + "Invalid sys_type value for Spack: " + sys_type) class NoSysTypeError(serr.SpackError): def __init__(self): - super(NoSysTypeError, self).__init__("Could not determine sys_type for this machine.") + super(NoSysTypeError, self).__init__( + "Could not determine sys_type for this machine.") @key_ordering class Target(object): - """ Target is the processor of the host machine. The host machine may have different front-end - and back-end targets, especially if it is a Cray machine. The target will have a name and - also the module_name (e.g craype-compiler). Targets will also recognize which platform - they came from using the set_platform method. Targets will have compiler finding strategies - """ + """ Target is the processor of the host machine. + The host machine may have different front-end and back-end targets, + especially if it is a Cray machine. The target will have a name and + also the module_name (e.g craype-compiler). Targets will also + recognize which platform they came from using the set_platform method. + Targets will have compiler finding strategies + """ def __init__(self, name, compiler_strategy, module_name=None): - self.name = name # case of cray "ivybridge" but if it's x86_64 + self.name = name # case of cray "ivybridge" but if it's x86_64 self.compiler_strategy = compiler_strategy - self.module_name = module_name # craype-ivybridge + self.module_name = module_name # craype-ivybridge # Sets only the platform name to avoid recursiveness def set_platform(self, platform): @@ -85,7 +87,6 @@ def from_dict(d): target.platform_name = d['platform'] return target - def _cmp_key(self): return (self.name, self.compiler_strategy, self.module_name) @@ -97,6 +98,7 @@ def __str__(self): return self.platform_name + '-' + self.name return self.name + @key_ordering class Platform(object): """ Abstract class that each type of Platform will subclass. @@ -104,10 +106,10 @@ class Platform(object): is returned """ - priority = None # Subclass needs to set this number. This controls order in which platform is detected. + priority = None # Subclass needs to set this number. This controls order in which platform is detected. front_end = None back_end = None - default = None # The default back end target. On cray ivybridge + default = None # The default back end target. On cray ivybridge front_os = None back_os = None @@ -119,17 +121,21 @@ def __init__(self, name): self.name = name def add_target(self, name, target): - """Used by the platform specific subclass to list available targets. Raises an error - if the platform specifies a name that is reserved by spack as an alias. + """Used by the platform specific subclass to list available targets. + Raises an error if the platform specifies a name + that is reserved by spack as an alias. """ if name in ['front_end', 'fe', 'back_end', 'be', 'default']: - raise ValueError("%s is a spack reserved alias and cannot be the name of a target" % name) + raise ValueError( + "%s is a spack reserved alias " + "and cannot be the name of a target" % name) target.set_platform(self) self.targets[name] = target def target(self, name): - """This is a getter method for the target dictionary that handles defaulting based - on the values provided by default, front-end, and back-end. This can be overwritten + """This is a getter method for the target dictionary + that handles defaulting based on the values provided by default, + front-end, and back-end. This can be overwritten by a subclass for which we want to provide further aliasing options. """ if name == 'default': @@ -150,7 +156,7 @@ def _detect_mac_os(self): "10.8": "mountainlion", "10.9": "mavericks", "10.10": "yosemite", - "10.11": "elcapitan"} + "10.11": "elcapitan"} mac_ver = py_platform.mac_ver()[:-2] try: os_name = mac_releases[mac_ver] @@ -199,8 +205,10 @@ def _cmp_key(self): class OperatingSystem(object): - """ The operating system will contain a name and version. It will - also represent itself simple with it's name + """ Operating System class. It will include the name and version. + The biggest attribute to this class will be the compiler finding + strategy. At the moment the compiler finding strategy is going to + be a string. """ def __init__(self, name, version): self.name = name @@ -212,42 +220,6 @@ def __str__(self): def __repr__(self): return self.__str__() -def get_sys_type_from_spack_globals(): - """Return the SYS_TYPE from spack globals, or None if it isn't set.""" - if not hasattr(spack, "sys_type"): - return None - elif hasattr(spack.sys_type, "__call__"): - return spack.sys_type() #If in __init__.py there is a sys_type() then call that - else: - return spack.sys_type # Else use the attributed which defaults to None - - -# This is livermore dependent. Hard coded for livermore -#def get_sys_type_from_environment(): -# """Return $SYS_TYPE or None if it's not defined.""" -# return os.environ.get('SYS_TYPE') - - -def get_mac_sys_type(): - """Return a Mac OS SYS_TYPE or None if this isn't a mac. - Front-end config - """ - mac_ver = py_platform.mac_ver()[0] - if not mac_ver: - return None - return "macosx_%s_%s" % (Version(mac_ver).up_to(2), py_platform.machine()) - - -def get_sys_type_from_uname(): - """ Returns a sys_type from the uname argument - Front-end config - """ - try: - platform_proc = subprocess.Popen(['uname', '-i'], stdout = subprocess.PIPE) - platform, _ = platform_proc.communicate() - return platform.strip() - except: - return None @memoized def all_platforms(): @@ -267,6 +239,7 @@ def all_platforms(): return modules + @memoized def sys_type(): """ Gather a list of all available subclasses of platforms. @@ -276,9 +249,8 @@ def sys_type(): """ # Try to create a Platform object using the config file FIRST platform_list = all_platforms() - platform_list.sort(key = lambda a: a.priority) + platform_list.sort(key=lambda a: a.priority) for platform in platform_list: if platform.detect(): return platform() - From 7732f375d24c7089d571cbb1659cbd1d60250888 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Sat, 13 Feb 2016 14:36:01 -0800 Subject: [PATCH 113/481] Changed operating system getter so that linux and darwin autodetected operating systems are set as class default_os --- lib/spack/spack/architecture.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index c4ef0805ac..0e006bdf40 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -168,9 +168,11 @@ def _detect_mac_os(self): def add_operating_system(self, name=None, operating_system=None): if self.name == 'linux': linux_os = self._detect_linux_os() + self.default_os = linux_os.name self.operating_sys[linux_os.name] = linux_os elif self.name == 'darwin': mac_os = self._detect_mac_os() + self.default_os = mac_os.name self.operating_sys[mac_os.name] = mac_os else: self.operating_sys[name] = operating_system From c7993010116e02426c73dc32d8cfb74b8d542868 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Sat, 13 Feb 2016 14:37:07 -0800 Subject: [PATCH 114/481] Changed the method in which architecture is converted from string to namedtuple. Also added some TODO comments to later consider changing how architecture is handled --- lib/spack/spack/spec.py | 166 +++++++++++++++++++++++++--------------- 1 file changed, 104 insertions(+), 62 deletions(-) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 3b2d54d047..9b5df9a825 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -63,7 +63,7 @@ if it comes immediately after the compiler name. Otherwise it will be associated with the current package spec. -6. The target to build with. This is needed on machines where +6. The architecture to build with. This is needed on machines where cross-compilation is required Here is the EBNF grammar for a spec:: @@ -72,9 +72,9 @@ dep_list = { ^ spec } spec = id [ options ] options = { @version-list | +variant | -variant | ~variant | - %compiler | =target } + %compiler | =architecture } variant = id - target = id + architecture = id compiler = id [ version-list ] version-list = version [ { , version } ] version = id | id: | :id | id:id @@ -107,6 +107,7 @@ from llnl.util.tty.color import * import spack +import spack.architecture import spack.parse import spack.error import spack.compilers as compilers @@ -123,7 +124,7 @@ # Convenient names for color formats so that other things can use them compiler_color = '@g' version_color = '@c' -target_color = '@m' +architecture_color = '@m' enabled_variant_color = '@B' disabled_variant_color = '@r' dependency_color = '@.' @@ -134,7 +135,7 @@ See spack.color for descriptions of the color codes. """ color_formats = {'%' : compiler_color, '@' : version_color, - '=' : target_color, + '=' : architecture_color, '+' : enabled_variant_color, '~' : disabled_variant_color, '^' : dependency_color, @@ -411,7 +412,7 @@ def __init__(self, spec_like, *dep_like, **kwargs): self.name = other.name self.dependents = other.dependents self.versions = other.versions - self.target = other.target + self.architecture = other.architecture self.compiler = other.compiler self.dependencies = other.dependencies self.variants = other.variants @@ -456,11 +457,11 @@ def _set_compiler(self, compiler): self.compiler = compiler - def _set_target(self, target): - """Called by the parser to set the target.""" - if self.target: raise DuplicateTargetError( - "Spec for '%s' cannot have two targets." % self.name) - self.target = target # a string can be set + def _set_architecture(self, architecture): + """Called by the parser to set the architecture.""" + if self.architecture: raise DuplicateArchitectureError( + "Spec for '%s' cannot have two architectures." % self.name) + self.architecture = architecture # a string can be set def _add_dependency(self, spec): @@ -516,7 +517,7 @@ def is_virtual(name): @property def concrete(self): """A spec is concrete if it can describe only ONE build of a package. - If any of the name, version, target, compiler, + If any of the name, version, architecture, compiler, variants, or depdenencies are ambiguous,then it is not concrete. """ if self._concrete: @@ -525,7 +526,7 @@ def concrete(self): self._concrete = bool(not self.virtual and self.versions.concrete and self.variants.concrete - and self.target + and self.architecture and self.compiler and self.compiler.concrete and self.dependencies.concrete) return self._concrete @@ -660,10 +661,12 @@ def to_node_dict(self): 'dependencies' : dict((d, self.dependencies[d].dag_hash()) for d in sorted(self.dependencies)) } - if self.target: - d['target'] = self.target.target.to_dict() + if self.architecture: + # TODO: Fix the target.to_dict to account for the tuple + # Want it to be a dict of dicts + d['architecture'] = self.architecture.target.to_dict() else: - d['target'] = None + d['architecture'] = None if self.compiler: d.update(self.compiler.to_dict()) else: @@ -689,7 +692,9 @@ def from_node_dict(node): spec = Spec(name) spec.versions = VersionList.from_dict(node) - spec.target = spack.architecture.Target.from_dict(node['target']) + # TODO: Need to fix the architecture.Target.from_dict + spec.architecture = spack.architecture.Target.from_dict( + node['architecture']) if node['compiler'] is None: spec.compiler = None @@ -760,7 +765,7 @@ def _concretize_helper(self, presets=None, visited=None): # to presets below, their constraints will all be merged, but we'll # still need to select a concrete package later. changed |= any( - (spack.concretizer.concretize_target(self), + (spack.concretizer.concretize_architecture(self), spack.concretizer.concretize_compiler(self), spack.concretizer.concretize_version(self), spack.concretizer.concretize_variants(self))) @@ -1149,10 +1154,11 @@ def constrain(self, other, deps=True): raise UnsatisfiableVariantSpecError(self.variants[v], other.variants[v]) - if self.target is not None and other.target is not None: - if self.target != other.target: - raise UnsatisfiableTargetSpecError(self.target, - other.target) + # TODO: Check out the logic here + if self.architecture is not None and other.architecture is not None: + if self.architecture != other.architecture: + raise UnsatisfiableTargetSpecError(self.architecture, + other.architecture) changed = False if self.compiler is not None and other.compiler is not None: @@ -1164,9 +1170,9 @@ def constrain(self, other, deps=True): changed |= self.versions.intersect(other.versions) changed |= self.variants.constrain(other.variants) - old = self.target - self.target = self.target or other.target - changed |= (self.target != old) + old = self.architecture + self.architecture = self.architecture or other.architecture + changed |= (self.architecture != old) if deps: changed |= self._constrain_dependencies(other) @@ -1231,34 +1237,67 @@ def _autospec(self, spec_like): except SpecError: return parse_anonymous_spec(spec_like, self.name) + def _is_valid_platform(self, platform, platform_list): + if platform in platform_names: + return True + return False - def add_target_from_string(self, arch): - """If only a target is provided, spack will assume the default architecture. - A platform-target pair can be input delimited by a '-'. If either portion of - a platform-target pair is empty, spack will supply a default, in the case of - a blank target the default will be dependent on the platform. - E.g. x86_64 -> 64 bit x86 - bgq- -> default bgq target (back end/powerpc) - cray-hawswell -> haswell target on cray platform + def _is_valid_target(self, target, platform): + if target in platform.targets: + return True + return False + + def add_architecture_from_string(self, arch): + """ The user is able to provide a architecture string of the form + platform-os-target. This string will be parsed by this function + and turn the architecture string into an architecture tuple of + platform, operating system and target processor classes. + The platform-os-target triplet can be delimited by a '-'. If any + portion of the architecture triplet is empty, spack will supply + the default. If the architecture is blank then defaults will + be provided based off of the platform + + e.g + =linux-ubuntu10-x84_64 -> (linux, ubuntu10, x86_64) + + =cray_xc-SuSE11-haswell -> (cray_xc, SuSE11, haswell) + + =bgq -> (bgq, + default_os, + default_target) + + =elcapitan -> (darwin, elcapitan, x86_64) + + =x86_64 -> (autodetected platform, + default_os, + x86_64) """ - Arch = namedtuple("Arch", "arch_os target") - platform = spack.architecture.sys_type() - + + platform_list = spack.architecture.all_platforms() + platform_names = [plat.__name__.lower() for plat in platform_list] if arch is None: return + # Get all the platforms to check whether string is valid + Arch = namedtuple("Arch", "platform platform_os target") + arch_list = arch.split("-") + platform = spack.architecture.sys_type() + for entry in arch_list: + if _is_valid_platform(entry, platform_names): + platform = entry() + elif _is_valid_target(entry, platform): + target = entry + else: + platform_os = entry - if '-' in arch: - os_name, target = arch.split('-') - self.target = Arch(arch_os=platform.operating_system(os_name), - target = platform.target(target)) - elif arch in platform.targets: - self.target = Arch(arch_os=platform.operating_system('default'), - target=platform.target(target)) - else: - os_name = arch # Odd naming here, definitely need to change - self.target = Arch(arch_os=platform.operating_system(os_name), - target=platform.target('default')) + if target is None: + target = platform.target('default') + if platform_os is None: + platform_os = platform.operating_system('default') + self.architecture = Arch(platform=platform, + platform_os=platform_os, + target=target) + def satisfies(self, other, deps=True, strict=False): """determine if this spec satisfies all constraints of another. @@ -1306,15 +1345,16 @@ def satisfies(self, other, deps=True, strict=False): # Target satisfaction is currently just class equality. # If not strict, None means unconstrained. - if isinstance(self.target, basestring): - self.add_target_from_string(self.target) - if isinstance(other.target, basestring): - other.add_target_from_string(other.target) + if isinstance(self.architecture, basestring): + self.add_architecture_from_string(self.architecture) + if isinstance(other.architecture, basestring): + other.add_architecture_from_string(other.architecture) - if self.target and other.target: - if self.target != other.target: + # TODO: Need to make sure that comparisons can be made via classes + if self.architecture and other.architecture: + if self.architecture != other.architecture: return False - elif strict and (other.target and not self.target): + elif strict and (other.architecture and not self.architecture): return False # If we need to descend into dependencies, do it, otherwise we're done. @@ -1384,11 +1424,12 @@ def _dup(self, other, **kwargs): spec but not its dependencies. """ + # TODO: Check if comparisons for tuple are valid # We don't count dependencies as changes here changed = True if hasattr(self, 'name'): changed = (self.name != other.name and self.versions != other.versions and \ - self.target != other.target and self.compiler != other.compiler and \ + self.architecture != other.architecture and self.compiler != other.compiler and \ self.variants != other.variants and self._normal != other._normal and \ self.concrete != other.concrete and self.external != other.external and \ self.external_module != other.external_module) @@ -1396,7 +1437,7 @@ def _dup(self, other, **kwargs): # Local node attributes get copied first. self.name = other.name self.versions = other.versions.copy() - self.target = other.target + self.architecture = other.architecture self.compiler = other.compiler.copy() if other.compiler else None if kwargs.get('cleardeps', True): self.dependents = DependencyMap() @@ -1526,7 +1567,7 @@ def ne_dag(self, other): def _cmp_node(self): """Comparison key for just *this node* and not its deps.""" return (self.name, self.versions, self.variants, - self.target, self.compiler) + self.architecture, self.compiler) def eq_node(self, other): @@ -1585,7 +1626,7 @@ def format(self, format_string='$_$@$%@$+$=', **kwargs): Anything else is copied verbatim into the output stream. *Example:* ``$_$@$+`` translates to the name, version, and options - of the package, but no dependencies, target, or compiler. + of the package, but no dependencies, architecture, or compiler. TODO: allow, e.g., $6# to customize short hash length TODO: allow, e.g., $## for full hash. @@ -1628,9 +1669,10 @@ def write(s, c): elif c == '+': if self.variants: write(fmt % str(self.variants), c) + # TODO: Check string methods here elif c == '=': - if self.target: - write(fmt % (c + str(self.target)), c) + if self.architecture: + write(fmt % (c + str(self.architecture)), c) elif c == '#': out.write('-' + fmt % (self.dag_hash(7))) elif c == '$': @@ -2036,10 +2078,10 @@ def __init__(self, pkg, variant): "Package %s has no variant %s!" % (pkg, variant)) -class DuplicateTargetError(SpecError): +class DuplicateArchitectureError(SpecError): """Raised when the same target occurs in a spec twice.""" def __init__(self, message): - super(DuplicateTargetError, self).__init__(message) + super(DuplicateArchitectureError, self).__init__(message) class InconsistentSpecError(SpecError): From 35a602baaf39fb5056f3dbb7eccbd638e126159f Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Tue, 16 Feb 2016 15:17:57 -0800 Subject: [PATCH 115/481] Changed target checking to architecture checking for abi compatible --- lib/spack/spack/abi.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/abi.py b/lib/spack/spack/abi.py index 1dd49d6b2c..0f07feda5f 100644 --- a/lib/spack/spack/abi.py +++ b/lib/spack/spack/abi.py @@ -34,9 +34,10 @@ class ABI(object): """This class provides methods to test ABI compatibility between specs. The current implementation is rather rough and could be improved.""" - def target_compatible(self, parent, child): + def architecture_compatible(self, parent, child): """Returns true iff the parent and child specs have ABI compatible targets.""" - return not parent.target or not child.target or parent.target == child.target + return not parent.architecture or not \ + child.achitecture or parent.architecture == child.architecture @memoized @@ -123,6 +124,6 @@ def compiler_compatible(self, parent, child, **kwargs): def compatible(self, parent, child, **kwargs): """Returns true iff a parent and child spec are ABI compatible""" loosematch = kwargs.get('loose', False) - return self.target_compatible(parent, child) and \ + return self.architecture_compatible(parent, child) and \ self.compiler_compatible(parent, child, loose=loosematch) From 19dca4bcc9dcdd902fb5490b4ee49b9dc92aa5ba Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Tue, 16 Feb 2016 15:21:36 -0800 Subject: [PATCH 116/481] Added some _cmp_key methods to OperatingSystem class, and also changed the way Platforms are compared. Created a mini inherited class named Arch that inherits from namedtuple. The reason for this is to override the namedtuple __str__ method to make arch tuples look prettier when printed out. Rather than Arch(platform= ... ) it prints to platform-os-target which makes directory paths to packages look cleaner. --- lib/spack/spack/architecture.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 0e006bdf40..3cbce5dc52 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -22,6 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +from collections import namedtuple import imp import platform as py_platform import inspect @@ -178,7 +179,7 @@ def add_operating_system(self, name=None, operating_system=None): self.operating_sys[name] = operating_system def operating_system(self, name): - if name == 'default': + if name == 'default_os': name = self.default_os if name == 'front_os': name = self.front_os @@ -203,9 +204,10 @@ def __str__(self): return self.name def _cmp_key(self): - return (self.name, (_cmp_key(t) for t in self.targets.values())) - + return (self.name, (_cmp_key(t) for t in self.targets.values()), + (_cmp_key(o) for o in self.operating_sys.values())) +@key_ordering class OperatingSystem(object): """ Operating System class. It will include the name and version. The biggest attribute to this class will be the compiler finding @@ -222,6 +224,19 @@ def __str__(self): def __repr__(self): return self.__str__() + def _cmp_key(self): + return (self.name, self.version) + + +class Arch(namedtuple("Arch", "platform platform_os target")): + """ namedtuple for Architecture. Will have it's own __str__ method + to make printing out the tuple easier and also won't make directory + paths look odd """ + __slots__ = () + + def __str__(self): + return (self.platform.name +"-"+ + self.platform_os.name + "-" + self.target.name) @memoized def all_platforms(): From 31ab2383067830e47d54788a2cac01f949e77798 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Tue, 16 Feb 2016 15:22:23 -0800 Subject: [PATCH 117/481] Beginning attemps to fix concretization method to account for the new tuple architecture --- lib/spack/spack/concretize.py | 73 ++++++++++++++++++++++------------- 1 file changed, 46 insertions(+), 27 deletions(-) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 8886b3cd93..1c373679a2 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -210,55 +210,74 @@ def concretize_version(self, spec): return True # Things changed + def _concretize_platform(self, arch, platform): + if issubclass(arch.platform.__class__, spack.architecture.Platform): + return True + else: + arch.platform = platform + def _concretize_operating_system(self, arch, platform): """ Future method for concretizing operating system """ - if isinstance(arch.arch_os, spack.architecture.OperatingSystem): + if isinstance(arch.platform_os, spack.architecture.OperatingSystem): return False else: - arch.arch_os = platform.operating_system('default') + arch.arch_os = platform.operating_system('default_os') return True - def _concretize_arch_target(self, arch, platform): + def _concretize_target(self, arch, platform): if isinstance(arch.target, spack.architecture.Target): return False else: arch.target = platform.target('default') return True - def concretize_target(self, spec): - """If the spec already has an target and it is a an target type, - return. Otherwise, if it has a target that is a string type, generate a - target based on that type. If it has no target and the root of the - DAG has an target, then use that. Otherwise, take the system's default - target. + def concretize_architecture(self, spec): + """If the spec is empty provide the defaults of the platform. If the + architecture is not a basestring, then check if either the platform, + target or operating system are concretized. If any of the fields are + changed then return True. If everything is concretized (i.e the + architecture attribute is a namedtuple of classes) then return False. + If the target is a string type, then convert the string into a + concretized architecture. If it has no architecture and the root of the + DAG has an architecture, then use the root otherwise use the defaults + on the platform. """ - platform = spack.architecture.sys_type() - if spec.target is None: + platform = spack.architecture.sys_type() + import ipdb;ipdb.set_trace() + if spec.architecture is None: # Create an empty tuple - Arch = collections.namedtuple("Arch", "arch_os target") - spec.target = Arch(arch_os=platform.operating_system('default'), - target=platform.target('default')) - return True - # If there is a target and it is a tuple and has both filled return - # False - if not isinstance(spec.target, basestring): - return any((self._concretize_operating_system(spec.target, platform), - self._concretize_arch_target(spec.target, platform))) + Arch = spack.architecture.Arch + # Set the architecture to all defaults + spec.architecture = Arch(platform=platform, + platform_os=platform.operating_system('default_os'), + target=platform.target('default')) + return True + #If there is a target and it is a tuple and has both filled return + #False + if not isinstance(spec.architecture, basestring): + return any(( + self._concretize_platform(spec.architecture, platform), + self._concretize_operating_system(spec.architecture, platform), + self._concretize_arch_target(spec.architecture, platform))) else: spec.add_target_from_string(spec.target) - if spec.root.target and \ - not isinstance(spec.root.target, basestring): - bool_flag = any( - (self._concretize_operating_system(spec.root.target, platform), - self._concretize_arch_target(spec.root.target, platform))) - spec.target =spec.root.target + # Does not look pretty at all!!! + if spec.root.architecture and \ + not isinstance(spec.root.architecture, basestring): + bool_flag = any(( + self._concretize_platform(spec.root.architecture, platform), + self._concretize_operating_system(spec.root.architecture, + platform), + self._concretize_target(spec.root.target, platform))) + spec.architecture =spec.root.architecture return bool_flag else: - spec.add_target_from_string(spec.root.target) + spec.add_architecture_from_string(spec.root.architecture) + return True # if there is no target specified used the defaults From 62b0293963bd21ff734993e5ac577650af0faa3f Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Tue, 16 Feb 2016 15:23:38 -0800 Subject: [PATCH 118/481] Cleaned up the file. Got rid of commented out sections of code since they weren't going to be used anyway --- lib/spack/spack/platforms/cray_xc.py | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/lib/spack/spack/platforms/cray_xc.py b/lib/spack/spack/platforms/cray_xc.py index b37d36a62c..d48b6ffcfa 100644 --- a/lib/spack/spack/platforms/cray_xc.py +++ b/lib/spack/spack/platforms/cray_xc.py @@ -17,7 +17,7 @@ def __init__(self): if we use CRAY_CPU_TARGET as the default. This will ensure that if we're on a XC-40 or XC-30 then we can detect the target ''' - super(CrayXc, self).__init__('cray_xc') + super(CrayXc, self).__init__('crayxc') # Handle the default here so we can check for a key error if 'CRAY_CPU_TARGET' in os.environ: @@ -28,23 +28,21 @@ def __init__(self): self.front_end = self.default self.back_end = self.default - # Could switch to use modules and fe targets for front end # Currently using compilers by path for front end. self.add_target('sandybridge', Target('sandybridge', 'PATH')) - self.add_target('ivybridge', Target('ivybridge', 'MODULES', 'craype-ivybridge')) - self.add_target('haswell', Target('haswell', 'MODULES', 'craype-haswell')) - - self.add_operating_system('SuSE11', OperatingSystem('SuSE11', '11')) - self.add_operating_system('GNU/Linux', OperatingSystem('GNU/Linux', '10')) -# self.add_target(self.front_end, Target(self.front_end, 'PATH')) - # Back End compiler needs the proper target module loaded. -# self.add_target(self.back_end, Target(self.front_end, 'MODULES', 'craype-'+ self.back_end)) -# self.add_target(self.default, Target(self.default, 'MODULES', 'craype-' + self.default)) - # This is kludgy and the order matters when the targets are all haswell - # This is because the last one overwrites the others when they have the - # same name. + self.add_target('ivybridge', + Target('ivybridge', 'MODULES', 'craype-ivybridge')) + + self.add_target('haswell', + Target('haswell', 'MODULES', 'craype-haswell')) + + self.add_operating_system('SuSE11', + OperatingSystem('SuSE11', '11')) + + self.add_operating_system('GNU/Linux', + OperatingSystem('GNU/Linux', '10')) @classmethod def detect(self): From a3039c4c67b376d2e8577c2227cffac8873bdc74 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Tue, 16 Feb 2016 15:26:07 -0800 Subject: [PATCH 119/481] Changed add_architecture_from_string, it now loops through the string and checks whether each piece of string is a valid platform, operating system and target. If the operating system or target are none it will use the defaults. Updated the documentation for that method. One thing that bothers me is how to account for the multitude of operating systems when cross compiling. If someone wants to compile with another operating system not found on current platform. How can spack check to see if it is valid? --- lib/spack/spack/spec.py | 64 +++++++++++++++++++++++++++-------------- 1 file changed, 43 insertions(+), 21 deletions(-) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 9b5df9a825..91e9e432eb 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1238,14 +1238,18 @@ def _autospec(self, spec_like): return parse_anonymous_spec(spec_like, self.name) def _is_valid_platform(self, platform, platform_list): - if platform in platform_names: + if platform in platform_list: return True return False def _is_valid_target(self, target, platform): if target in platform.targets: return True - return False + return False + def _is_valid_os(self, os_string, platform): + if os_string in platform.operating_sys: + return True + return False def add_architecture_from_string(self, arch): """ The user is able to provide a architecture string of the form @@ -1254,8 +1258,9 @@ def add_architecture_from_string(self, arch): platform, operating system and target processor classes. The platform-os-target triplet can be delimited by a '-'. If any portion of the architecture triplet is empty, spack will supply - the default. If the architecture is blank then defaults will - be provided based off of the platform + the default. If the entire architecture field is blank then + defaults will be provided based off of the platform. + This happens in the concretize_architecture method in concretize.py e.g =linux-ubuntu10-x84_64 -> (linux, ubuntu10, x86_64) @@ -1272,27 +1277,38 @@ def add_architecture_from_string(self, arch): default_os, x86_64) """ - + if arch is None: return + platform_list = spack.architecture.all_platforms() - platform_names = [plat.__name__.lower() for plat in platform_list] - if arch is None: - return - # Get all the platforms to check whether string is valid - Arch = namedtuple("Arch", "platform platform_os target") + platform_names = {plat.__name__.lower():plat for plat in platform_list} + Arch = spack.architecture.Arch arch_list = arch.split("-") - platform = spack.architecture.sys_type() + + # Create instance of current platform, gets overwritten if user + # provided a platform spec. + platform = spack.architecture.sys_type() + target = None + platform_os = None + for entry in arch_list: - if _is_valid_platform(entry, platform_names): - platform = entry() - elif _is_valid_target(entry, platform): - target = entry + if self._is_valid_platform(entry, platform_names): + if entry != platform.name: + platform = platform_dict[entry]() # Create instance of platform + elif self._is_valid_target(entry, platform): + target = platform.target(entry) + # Need to figure out if we're supporting arbitrary os's and how + # to account for them + # Not really a good implementation since the user can add + # gibberish and spack will see it as an os + elif self._is_valid_os(entry, platform): + platform_os = platform.operating_system(entry) else: - platform_os = entry + raise UnknownArchitectureSpecError(entry) if target is None: target = platform.target('default') if platform_os is None: - platform_os = platform.operating_system('default') + platform_os = platform.operating_system('default_os') self.architecture = Arch(platform=platform, platform_os=platform_os, @@ -1854,7 +1870,6 @@ def __init__(self): def do_parse(self): specs = [] - try: while self.next: if self.accept(ID): @@ -1889,7 +1904,7 @@ def spec(self): spec.name = self.token.value spec.versions = VersionList() spec.variants = VariantMap(spec) - spec.target = None + spec.architecture = None spec.compiler = None spec.external = None spec.external_module = None @@ -1920,7 +1935,7 @@ def spec(self): spec._set_compiler(self.compiler()) elif self.accept(EQ): - spec._set_target(self.target()) + spec._set_architecture(self.architecture()) else: break @@ -1938,7 +1953,7 @@ def variant(self): return self.token.value - def target(self): + def architecture(self): self.expect(ID) return self.token.value @@ -2077,6 +2092,13 @@ def __init__(self, pkg, variant): super(UnknownVariantError, self).__init__( "Package %s has no variant %s!" % (pkg, variant)) +class UnknownArchitectureSpecError(SpecError): + """ Raised when an entry in a string field is neither a platform, + operating system or a target. """ + def __init__(self, architecture_spec_entry): + super(UnknownArchitectureSpecError, self).__init__( + "Architecture spec %s is not a valid spec entry" % ( + architecture_spec_entry)) class DuplicateArchitectureError(SpecError): """Raised when the same target occurs in a spec twice.""" From 2a79537ba1120620f79d39bd4081a6a11b7862c4 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 17 Feb 2016 14:44:38 -0800 Subject: [PATCH 120/481] Fixed silly typo that gave errors in tests --- lib/spack/spack/abi.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/abi.py b/lib/spack/spack/abi.py index 0f07feda5f..bd9bc30fd1 100644 --- a/lib/spack/spack/abi.py +++ b/lib/spack/spack/abi.py @@ -36,8 +36,8 @@ class ABI(object): def architecture_compatible(self, parent, child): """Returns true iff the parent and child specs have ABI compatible targets.""" - return not parent.architecture or not \ - child.achitecture or parent.architecture == child.architecture + return not parent.architecture or not child.architecture \ + or parent.architecture == child.architecture @memoized From 2c20fc2ebf8e9b360f3935a1867d9b95c69513ac Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 17 Feb 2016 14:45:53 -0800 Subject: [PATCH 121/481] Changed module loading to use arch tuple instead of regular target.module --- lib/spack/spack/build_environment.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index d849b4b56c..6e03394777 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -226,8 +226,8 @@ def set_build_environment_variables(pkg): pkg_config_dirs.append(pcdir) path_put_first("PKG_CONFIG_PATH", pkg_config_dirs) - if pkg.spec.target.module_name: - load_module(pkg.spec.target.module_name) + if pkg.spec.architecture.target.module_name: + load_module(pkg.spec.architecture.target.module_name) def set_module_variables_for_package(pkg): """Populate the module scope of install() with some useful functions. From 524e9b372ee85508abad1163368c7a9fdc09fcc3 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 17 Feb 2016 14:47:29 -0800 Subject: [PATCH 122/481] From now on all targets are now architecture. Architecture is a named tuple so any time we need access to target we do architecture.target. --- lib/spack/spack/concretize.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 1c373679a2..6c2aaeda01 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -180,7 +180,7 @@ def concretize_version(self, spec): # If there are known available versions, return the most recent # version that satisfies the spec - pkg = spec.package + pkg = spec.package # Gives error here with dynist cmp_versions = partial(spack.pkgsort.version_compare, spec.name) valid_versions = sorted( [v for v in pkg.versions @@ -212,9 +212,10 @@ def concretize_version(self, spec): def _concretize_platform(self, arch, platform): if issubclass(arch.platform.__class__, spack.architecture.Platform): - return True + return False else: arch.platform = platform + return True def _concretize_operating_system(self, arch, platform): """ Future method for concretizing operating system """ @@ -245,7 +246,6 @@ def concretize_architecture(self, spec): """ platform = spack.architecture.sys_type() - import ipdb;ipdb.set_trace() if spec.architecture is None: # Create an empty tuple Arch = spack.architecture.Arch @@ -260,9 +260,9 @@ def concretize_architecture(self, spec): return any(( self._concretize_platform(spec.architecture, platform), self._concretize_operating_system(spec.architecture, platform), - self._concretize_arch_target(spec.architecture, platform))) + self._concretize_target(spec.architecture, platform))) else: - spec.add_target_from_string(spec.target) + spec.add_architecture_from_string(spec.target) # Does not look pretty at all!!! if spec.root.architecture and \ @@ -325,18 +325,18 @@ def concretize_compiler(self, spec): link to this one, to maximize compatibility. """ # Pass on concretizing the compiler if the target is not yet determined - if not spec.target: + if not spec.architecture.target: #Although this usually means changed, this means awaiting other changes return True # Only use a matching compiler if it is of the proper style # Takes advantage of the proper logic already existing in compiler_for_spec # Should think whether this can be more efficient - def _proper_compiler_style(cspec, target): + def _proper_compiler_style(cspec, architecture): compilers = spack.compilers.compilers_for_spec(cspec) - if target.target.compiler_strategy == 'PATH': + if architecture.target.compiler_strategy == 'PATH': filter(lambda c: not c.modules, compilers) - if target.target.compiler_strategy == 'MODULES': + if architecture.target.compiler_strategy == 'MODULES': filter(lambda c: c.modules, compilers) return compilers @@ -369,7 +369,7 @@ def _proper_compiler_style(cspec, target): # copy concrete version into other_compiler index = len(matches)-1 - while not _proper_compiler_style(matches[index], spec.target): + while not _proper_compiler_style(matches[index], spec.architecture): index -= 1 if index == 0: raise NoValidVersionError(spec) From ab4006ee2c4e7ecbf256a9de56e54e8573b37e01 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 17 Feb 2016 14:48:53 -0800 Subject: [PATCH 123/481] Changed so that directory layout uses the platform-os-target string version of the arch tuple --- lib/spack/spack/directory_layout.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index da3441fa13..435ecfdf4d 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -200,8 +200,7 @@ def relative_path_for_spec(self, spec): spec.version, spec.dag_hash(self.hash_len)) - path = join_path( - spec.target, + path = join_path(spec.architecture, "%s-%s" % (spec.compiler.name, spec.compiler.version), dir_name) From 1d484dbe5fb70885875dcbb2e795c4d517646e77 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 17 Feb 2016 14:49:36 -0800 Subject: [PATCH 124/481] Changed so that modules use correct file path with spec.architecture instead of spec.target --- lib/spack/spack/modules.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index f84ac75c77..56a61adefb 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -193,7 +193,7 @@ class Dotkit(EnvModule): @property def file_name(self): - return join_path(Dotkit.path, self.spec.target, + return join_path(Dotkit.path, self.spec.architecture, self.spec.format('$_$@$%@$+$#.dk')) @property @@ -230,7 +230,7 @@ class TclModule(EnvModule): @property def file_name(self): - return join_path(TclModule.path, self.spec.target, self.use_name) + return join_path(TclModule.path, self.spec.architecture, self.use_name) @property From 54042e399b5cfb4344643d8447c4274d34842643 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 17 Feb 2016 14:50:10 -0800 Subject: [PATCH 125/481] Changed target to architecture.target --- lib/spack/spack/package.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 1148e4ac3d..b0d84863de 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -588,11 +588,13 @@ def prefix(self): @property + #TODO: Change this to architecture def compiler(self): """Get the spack.compiler.Compiler object used to build this package.""" if not self.spec.concrete: raise ValueError("Can only get a compiler for a concrete package.") - return spack.compilers.compiler_for_spec(self.spec.compiler, self.spec.target) + return spack.compilers.compiler_for_spec(self.spec.compiler, + self.spec.architecture.target) def url_version(self, version): From d9e8bf18078bc4df381b05820b14365f6e0ae01f Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 17 Feb 2016 14:51:14 -0800 Subject: [PATCH 126/481] Added ARCHITECTURE field when showing specs --- lib/spack/spack/spec.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 91e9e432eb..0b42498910 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1735,9 +1735,9 @@ def write(s, c): elif named_str == 'OPTIONS': if self.variants: write(fmt % str(self.variants), '+') - elif named_str == 'TARGET': - if self.target: - write(fmt % str(self.target), '=') + elif named_str == 'ARCHITECTURE': + if self.architecture: + write(fmt % self.architecture, '=') elif named_str == 'SHA1': if self.dependencies: out.write(fmt % str(self.dep_hash(8))) From 93a92a00eeaf6e3e721b2150fbc44bd1b388ee66 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 17 Feb 2016 14:53:09 -0800 Subject: [PATCH 127/481] Changed abstract.target to abstract.architecture.target and abstract.architecture.target.concrete --- lib/spack/spack/test/concretize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index 2403719134..c12aada076 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -46,7 +46,7 @@ def check_spec(self, abstract, concrete): if abstract.compiler and abstract.compiler.concrete: self.assertEqual(abstract.compiler, concrete.compiler) - if abstract.target and abstract.target.concrete: + if abstract.architecture and abstract.architecture.target.concrete: self.assertEqual(abstract.target, concrete.target) From 0a6b54a8c40045dac7d763880de496cd01984c0c Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 17 Feb 2016 14:53:56 -0800 Subject: [PATCH 128/481] Added module compilers so that no CompilerSpecError is thrown --- var/spack/mock_configs/site_spackconfig/compilers.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/var/spack/mock_configs/site_spackconfig/compilers.yaml b/var/spack/mock_configs/site_spackconfig/compilers.yaml index 0a2dc893e2..fcbf7a53f1 100644 --- a/var/spack/mock_configs/site_spackconfig/compilers.yaml +++ b/var/spack/mock_configs/site_spackconfig/compilers.yaml @@ -5,8 +5,18 @@ compilers: cxx: /path/to/clang++ f77: None fc: None + modules: None gcc@4.5.0: cc: /path/to/gcc cxx: /path/to/g++ f77: /path/to/gfortran fc: /path/to/gfortran + modules: None + gcc@5.2.0: + cc: cc + cxx: CC + f77: ftn + fc: ftn + modules: + - PrgEnv-gnu + - gcc/5.2.0 From 3558404c037cdd0730cd4672c8522b9bb3bd6bd2 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 17 Feb 2016 14:54:39 -0800 Subject: [PATCH 129/481] Check difference by using arch tuple rather than .target --- var/spack/mock_packages/multimethod/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/mock_packages/multimethod/package.py b/var/spack/mock_packages/multimethod/package.py index b37db9f1eb..b926d42d7b 100644 --- a/var/spack/mock_packages/multimethod/package.py +++ b/var/spack/mock_packages/multimethod/package.py @@ -120,7 +120,7 @@ def has_a_default(self): for target in targets: @when('='+target.name) def different_by_target(self): - return self.spec.target.name + return self.spec.architecture.target.name # From 28218755a5b11dd02c3c7e4c4a503a110ade94e2 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 18 Feb 2016 13:06:55 -0800 Subject: [PATCH 130/481] Changed spec.target to spec.architecture --- lib/spack/spack/concretize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 6c2aaeda01..0e7bf53b44 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -262,7 +262,7 @@ def concretize_architecture(self, spec): self._concretize_operating_system(spec.architecture, platform), self._concretize_target(spec.architecture, platform))) else: - spec.add_architecture_from_string(spec.target) + spec.add_architecture_from_string(spec.architecture) # Does not look pretty at all!!! if spec.root.architecture and \ From 18ddbae60e0fa91cbbc73a1b501c489c3f7fc8d2 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 22 Feb 2016 14:49:53 -0800 Subject: [PATCH 131/481] Added new module path to operating system file --- lib/spack/spack/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index e1ef094f17..6ad9b87ca2 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -39,6 +39,7 @@ module_path = join_path(lib_path, "spack") platform_path = join_path(module_path, 'platforms') compilers_path = join_path(module_path, "compilers") +operating_system_path = join_path(module_path, 'operating_system') test_path = join_path(module_path, "test") hooks_path = join_path(module_path, "hooks") var_path = join_path(prefix, "var", "spack") From a8de45ce28b8277c0ca1740aa53e1e4e334596d1 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 22 Feb 2016 14:50:51 -0800 Subject: [PATCH 132/481] Got rid of methods used to detect mac osx and linux osx. Now those methods are used my operating system subclasses --- lib/spack/spack/architecture.py | 88 ++++++++++++++++++++------------- 1 file changed, 54 insertions(+), 34 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 3cbce5dc52..3ba45f3d82 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -148,35 +148,29 @@ def target(self, name): return self.targets[name] - def _detect_linux_os(self): - return OperatingSystem(py_platform.dist()[0], py_platform.dist()[1]) + #def _detect_linux_os(self): + # return OperatingSystem(py_platform.dist()[0], py_platform.dist()[1]) - def _detect_mac_os(self): - mac_releases = {'10.6': "snowleopard", - "10.7": "lion", - "10.8": "mountainlion", - "10.9": "mavericks", - "10.10": "yosemite", - "10.11": "elcapitan"} - mac_ver = py_platform.mac_ver()[:-2] - try: - os_name = mac_releases[mac_ver] - return OperatingSystem(os_name, mac_ver) - except KeyError: - os_name = "mac_os" - return OperatingSystem(os_name, mac_ver) - - def add_operating_system(self, name=None, operating_system=None): - if self.name == 'linux': - linux_os = self._detect_linux_os() - self.default_os = linux_os.name - self.operating_sys[linux_os.name] = linux_os - elif self.name == 'darwin': - mac_os = self._detect_mac_os() - self.default_os = mac_os.name - self.operating_sys[mac_os.name] = mac_os - else: - self.operating_sys[name] = operating_system + #def _detect_mac_os(self): + # mac_releases = {'10.6': "snowleopard", + # "10.7": "lion", + # "10.8": "mountainlion", + # "10.9": "mavericks", + # "10.10": "yosemite", + # "10.11": "elcapitan"} + # mac_ver = py_platform.mac_ver()[:-2] + # try: + # os_name = mac_releases[mac_ver] + # return OperatingSystem(os_name, mac_ver) + # except KeyError: + # os_name = "mac_os" + # return OperatingSystem(os_name, mac_ver) + + def add_operating_system(self, name, os_class): + """ Add the operating_system class object into the + platform.operating_sys dictionary + """ + self.operating_sys[name] = os_class def operating_system(self, name): if name == 'default_os': @@ -207,27 +201,37 @@ def _cmp_key(self): return (self.name, (_cmp_key(t) for t in self.targets.values()), (_cmp_key(o) for o in self.operating_sys.values())) + @key_ordering class OperatingSystem(object): - """ Operating System class. It will include the name and version. - The biggest attribute to this class will be the compiler finding - strategy. At the moment the compiler finding strategy is going to - be a string. + """ Operating System will be like a class similar to platform extended + by subclasses for the specifics. Operating System will contain the + compiler finding logic. Instead of calling two separate methods to + find compilers we call find_compilers method for each operating system """ def __init__(self, name, version): self.name = name self.version = version def __str__(self): - return self.name + return self.name + self.version def __repr__(self): return self.__str__() + + def compiler_strategy(self): + """ The compiler strategy will be overwritten by the subclass. + Depending on where it comes from it will either use compilers + based off of MODULES search method or the PATH search method + """ + raise NotImplementedError() def _cmp_key(self): return (self.name, self.version) + - +#NOTE: Key error caused because Architecture has no comparison method +@key_ordering class Arch(namedtuple("Arch", "platform platform_os target")): """ namedtuple for Architecture. Will have it's own __str__ method to make printing out the tuple easier and also won't make directory @@ -238,6 +242,22 @@ def __str__(self): return (self.platform.name +"-"+ self.platform_os.name + "-" + self.target.name) + def _cmp_key(self): + return (self.platform.name, self.platform_os.name, self.target.name) + + @staticmethod + def to_dict(platform, os_name, target): + """ This function will handle all the converting of dictionaries + for each of the architecture classes. This is better than having + each class have a to_dict method when creating the dict of dicts + arch class + """ + d = {} + d['platform'] = platform + d['operating_system'] = os_name + d['target'] = target.to_dict() + return d + @memoized def all_platforms(): modules = [] From 725d6d5fce56aa7d0ae401ec71d1565835248250 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 22 Feb 2016 14:52:09 -0800 Subject: [PATCH 133/481] import linux distro operating system subclass, set front-end os to LinuxDistro() --- lib/spack/spack/platforms/cray_xc.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/lib/spack/spack/platforms/cray_xc.py b/lib/spack/spack/platforms/cray_xc.py index d48b6ffcfa..b58c84d3eb 100644 --- a/lib/spack/spack/platforms/cray_xc.py +++ b/lib/spack/spack/platforms/cray_xc.py @@ -1,6 +1,7 @@ import os -from spack.architecture import Platform, Target, OperatingSystem +from spack.architecture import Platform, Target +from spack.operating_system.linux_distro import LinuxDistro class CrayXc(Platform): priority = 20 @@ -9,8 +10,8 @@ class CrayXc(Platform): default = 'ivybridge' front_os = "SuSE11" - back_os = "GNU/Linux" - default_os = "GNU/Linux" + back_os = "CNL" + default_os = "CNL" def __init__(self): ''' Since cori doesn't have ivybridge as a front end it's better @@ -31,18 +32,12 @@ def __init__(self): # Could switch to use modules and fe targets for front end # Currently using compilers by path for front end. self.add_target('sandybridge', Target('sandybridge', 'PATH')) - self.add_target('ivybridge', Target('ivybridge', 'MODULES', 'craype-ivybridge')) - self.add_target('haswell', - Target('haswell', 'MODULES', 'craype-haswell')) + Target('haswell', 'MODULES', 'craype-haswell')) - self.add_operating_system('SuSE11', - OperatingSystem('SuSE11', '11')) - - self.add_operating_system('GNU/Linux', - OperatingSystem('GNU/Linux', '10')) + self.add_operating_system('SuSE11', LinuxDistro()) @classmethod def detect(self): From 1a7d6ed49a8a3fd18fd3deb8ddc035ce0063766d Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 22 Feb 2016 14:52:52 -0800 Subject: [PATCH 134/481] import MacOSX subclass and added add_operating_system method to use subclass --- lib/spack/spack/platforms/darwin.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/platforms/darwin.py b/lib/spack/spack/platforms/darwin.py index cfa1d957a1..5b797a78db 100644 --- a/lib/spack/spack/platforms/darwin.py +++ b/lib/spack/spack/platforms/darwin.py @@ -1,5 +1,6 @@ import subprocess from spack.architecture import Platform, Target +from spack.operating_system.mac_osx import MacOSX class Darwin(Platform): priority = 89 @@ -10,7 +11,9 @@ class Darwin(Platform): def __init__(self): super(Darwin, self).__init__('darwin') self.add_target(self.default, Target(self.default, 'PATH')) - self.add_operating_system() + mac_os = MacOSX() + self.default_os = mac_os.name + self.add_operating_system(mac_os.name, mac_os) @classmethod def detect(self): From 22bf4bc08020898106ef2c8043f102ceeb83ec5c Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 22 Feb 2016 14:53:16 -0800 Subject: [PATCH 135/481] import linux distro subclass. Added method to add operating system to platform dictionary --- lib/spack/spack/platforms/linux.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/platforms/linux.py b/lib/spack/spack/platforms/linux.py index dcb08569a8..220f1bbaf4 100644 --- a/lib/spack/spack/platforms/linux.py +++ b/lib/spack/spack/platforms/linux.py @@ -1,5 +1,6 @@ import subprocess from spack.architecture import Platform, Target +from spack.operating_system.linux_distro import LinuxDistro class Linux(Platform): priority = 90 @@ -10,7 +11,9 @@ class Linux(Platform): def __init__(self): super(Linux, self).__init__('linux') self.add_target(self.default, Target(self.default, 'PATH')) - self.add_operating_system() + linux_dist = LinuxDistro() + self.default_os = str(linux_dist) + self.add_operating_system(str(linux_dist), linux_dist) @classmethod def detect(self): From 1367ccab93e6263e11477abe7ea05f1ad74d88a2 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 22 Feb 2016 14:53:55 -0800 Subject: [PATCH 136/481] New folder that will hold operating system subclasses --- lib/spack/spack/operating_system/__init__.py | 0 .../spack/operating_system/linux_distro.py | 25 +++++++++++ lib/spack/spack/operating_system/mac_osx.py | 45 +++++++++++++++++++ 3 files changed, 70 insertions(+) create mode 100644 lib/spack/spack/operating_system/__init__.py create mode 100644 lib/spack/spack/operating_system/linux_distro.py create mode 100644 lib/spack/spack/operating_system/mac_osx.py diff --git a/lib/spack/spack/operating_system/__init__.py b/lib/spack/spack/operating_system/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/lib/spack/spack/operating_system/linux_distro.py b/lib/spack/spack/operating_system/linux_distro.py new file mode 100644 index 0000000000..30a85fe61c --- /dev/null +++ b/lib/spack/spack/operating_system/linux_distro.py @@ -0,0 +1,25 @@ +import platform as py_platform +import spack +from spack.architecture import Platform, OperatingSystem + +class LinuxDistro(OperatingSystem): + """ This class will represent the autodetected operating system + for a Linux System. Since there are many different flavors of + Linux, this class will attempt to encompass them all through + autodetection using the python module platform and the method + platform.dist() + """ + def __init__(self): + def detect_operating_system(): + name = py_platform.dist()[0] + version = py_platform.dist()[1] + return name, version + + name, version = detect_operating_system() + + super(LinuxDistro, self).__init__(name, version) + + @property + def compiler_strategy(self): + return "PATH" + diff --git a/lib/spack/spack/operating_system/mac_osx.py b/lib/spack/spack/operating_system/mac_osx.py new file mode 100644 index 0000000000..0b939a5546 --- /dev/null +++ b/lib/spack/spack/operating_system/mac_osx.py @@ -0,0 +1,45 @@ +""" This class represents the MAC_OSX operating system. This will be auto + detected using the python platform.mac_ver. The MAC_OSX platform + will be represented using the major version operating system name, i.e + el capitan, yosemite...etc. +""" + +import spack +import os +import platform as py_platform +from spack.architecture import Platform, OperatingSystem + +class MacOSX(OperatingSystem): + def __init__(self): + """ Autodetects the mac version from a dictionary. Goes back as + far as 10.6 snowleopard. If the user has an older mac then + the version will just be a generic mac_os. + """ + + def get_mac_release(): + mac_releases = {'10.6': "snowleopard", + "10.7": "lion", + "10.8": "mountainlion", + "10.9": "mavericks", + "10.10": "yosemite", + "10.11": "elcapitan"} + + mac_ver = py_platform.mac_ver()[0][:-2] + try: + name = mac_releases[mac_ver] + return name, mac_ver + except KeyError: + name = "mac_os" + return name, mac_ver + + name, version = get_mac_release() + + super(MacOSX, self).__init__(name, version) + + @property + def compiler_strategy(self): + return "PATH" + + + + From a385dae1aee6b7624707f42af8a6c52b9d9889bd Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 22 Feb 2016 14:54:30 -0800 Subject: [PATCH 137/481] Unit tests to test operating system subclass creation and whether compiler strategy is set correctly --- lib/spack/spack/test/operating_system.py | 41 ++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 lib/spack/spack/test/operating_system.py diff --git a/lib/spack/spack/test/operating_system.py b/lib/spack/spack/test/operating_system.py new file mode 100644 index 0000000000..9d6850bfa6 --- /dev/null +++ b/lib/spack/spack/test/operating_system.py @@ -0,0 +1,41 @@ +""" Test checks if the operating_system class is created correctly and that +the functions are using the correct operating_system. Also checks whether +the operating_system correctly uses the compiler_strategy +""" + +import unittest +import os +import platform +from spack.platforms.cray_xc import CrayXc +from spack.platforms.linux import Linux +from spack.platforms.darwin import Darwin +from spack.operating_system.linux_distro import LinuxDistro +from spack.operating_system.mac_osx import MacOSX + +class TestOperatingSystem(unittest.TestCase): + + def setUp(self): + cray_xc = CrayXc() + linux = Linux() + darwin = Darwin() + self.cray_operating_sys = cray_xc.operating_system('front_os') + self.darwin_operating_sys = darwin.operating_system('default_os') + self.linux_operating_sys = linux.operating_system('default_os') + + def test_cray_front_end_operating_system(self): + self.assertIsInstance(self.cray_operating_sys, LinuxDistro) + + def test_cray_front_end_compiler_strategy(self): + self.assertEquals(self.cray_operating_sys.compiler_strategy, "PATH") + + def test_linux_operating_system(self): + print self.linux_operating_sys + self.assertIsInstance(self.linux_operating_sys, LinuxDistro) + + def test_linux_compiler_strategy(self): + self.assertEquals(self.linux_operating_sys.compiler_strategy, "PATH") + + + + + From ebc5e26c2d95971f414378631089fee270beeb51 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 22 Feb 2016 16:16:20 -0800 Subject: [PATCH 138/481] Added compiler strategy field to Operating System class --- lib/spack/spack/architecture.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 3ba45f3d82..574076bae4 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -209,26 +209,28 @@ class OperatingSystem(object): compiler finding logic. Instead of calling two separate methods to find compilers we call find_compilers method for each operating system """ - def __init__(self, name, version): + + def __init__(self, name, version, compiler_strategy): self.name = name self.version = version + self.compiler_strategy = compiler_strategy def __str__(self): return self.name + self.version def __repr__(self): return self.__str__() - - def compiler_strategy(self): - """ The compiler strategy will be overwritten by the subclass. - Depending on where it comes from it will either use compilers - based off of MODULES search method or the PATH search method - """ - raise NotImplementedError() def _cmp_key(self): return (self.name, self.version) + def compiler_strategy(self): + """ Operating Systems will now be in charge of the compiler finding + strategy. They will each have their own find_compilers method + which depending on their compiler strategy will find the compilers + using a specific method (i.e PATH vs MODULES) + """ + raise NotImplementedError() #NOTE: Key error caused because Architecture has no comparison method @key_ordering From 3c87d137a3f0823289c52ffff87e58b8b1d7e271 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 22 Feb 2016 16:17:33 -0800 Subject: [PATCH 139/481] Added compiler strategy entry and also added new operating system subclass called CNL for compute node linux (name will probably change because I can't find docs on the EXACT name) --- lib/spack/spack/operating_system/cnl.py | 20 +++++++++++++++++++ .../spack/operating_system/linux_distro.py | 10 +++++++--- lib/spack/spack/operating_system/mac_osx.py | 9 ++++++--- 3 files changed, 33 insertions(+), 6 deletions(-) create mode 100644 lib/spack/spack/operating_system/cnl.py diff --git a/lib/spack/spack/operating_system/cnl.py b/lib/spack/spack/operating_system/cnl.py new file mode 100644 index 0000000000..e52052dfa8 --- /dev/null +++ b/lib/spack/spack/operating_system/cnl.py @@ -0,0 +1,20 @@ +import spack +from spack.architecture import OperatingSystem + +class ComputeNodeLinux(OperatingSystem): + """ Compute Node Linux (CNL) is the operating system used for the Cray XC + series super computers. It is a very stripped down version of GNU/Linux. + Any compilers found through this operating system will be used with + modules. If updated, user must make sure that version and name are + updated to indicate that OS has been upgraded (or downgraded) + """ + def __init__(self): + name = 'CNL' + version = '10' + super(ComputeNodeLinux, self).__init__(name, version, "MODULES") + + def compiler_strategy(self): + return self.compiler_strategy + + def find_compilers(self): + pass diff --git a/lib/spack/spack/operating_system/linux_distro.py b/lib/spack/spack/operating_system/linux_distro.py index 30a85fe61c..b11c7a88fa 100644 --- a/lib/spack/spack/operating_system/linux_distro.py +++ b/lib/spack/spack/operating_system/linux_distro.py @@ -17,9 +17,13 @@ def detect_operating_system(): name, version = detect_operating_system() - super(LinuxDistro, self).__init__(name, version) + super(LinuxDistro, self).__init__(name, version, "PATH") - @property def compiler_strategy(self): - return "PATH" + return self.compiler_strategy + + def find_compilers(self): + pass + + diff --git a/lib/spack/spack/operating_system/mac_osx.py b/lib/spack/spack/operating_system/mac_osx.py index 0b939a5546..f52cdd6bc7 100644 --- a/lib/spack/spack/operating_system/mac_osx.py +++ b/lib/spack/spack/operating_system/mac_osx.py @@ -34,11 +34,14 @@ def get_mac_release(): name, version = get_mac_release() - super(MacOSX, self).__init__(name, version) + super(MacOSX, self).__init__(name, version, "PATH") - @property def compiler_strategy(self): - return "PATH" + return self.compiler_strategy + + def find_compilers(self): + pass + From 083b7b46d92605269df05a9267657acf5e4b4677 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 22 Feb 2016 16:17:54 -0800 Subject: [PATCH 140/481] Added more tests --- lib/spack/spack/test/operating_system.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/test/operating_system.py b/lib/spack/spack/test/operating_system.py index 9d6850bfa6..a4327dde83 100644 --- a/lib/spack/spack/test/operating_system.py +++ b/lib/spack/spack/test/operating_system.py @@ -11,6 +11,7 @@ from spack.platforms.darwin import Darwin from spack.operating_system.linux_distro import LinuxDistro from spack.operating_system.mac_osx import MacOSX +from spack.operating_system.cnl import ComputeNodeLinux class TestOperatingSystem(unittest.TestCase): @@ -19,6 +20,8 @@ def setUp(self): linux = Linux() darwin = Darwin() self.cray_operating_sys = cray_xc.operating_system('front_os') + self.cray_default_os = cray_xc.operating_system('default_os') + self.cray_back_os = cray_xc.operating_system('back_os') self.darwin_operating_sys = darwin.operating_system('default_os') self.linux_operating_sys = linux.operating_system('default_os') @@ -28,6 +31,12 @@ def test_cray_front_end_operating_system(self): def test_cray_front_end_compiler_strategy(self): self.assertEquals(self.cray_operating_sys.compiler_strategy, "PATH") + def test_cray_back_end_operating_system(self): + self.assertIsInstance(self.cray_back_os,ComputeNodeLinux) + + def test_cray_back_end_compiler_strategy(self): + self.assertEquals(self.cray_back_os.compiler_strategy, "MODULES") + def test_linux_operating_system(self): print self.linux_operating_sys self.assertIsInstance(self.linux_operating_sys, LinuxDistro) @@ -35,7 +44,16 @@ def test_linux_operating_system(self): def test_linux_compiler_strategy(self): self.assertEquals(self.linux_operating_sys.compiler_strategy, "PATH") - + + def test_cray_front_end_compiler_list(self): + """ Operating systems will now be in charge of finding compilers. + So, depending on which operating system you want to build for + or which operating system you are on, then you could detect + compilers in a certain way. Cray linux environment on the front + end is just a regular linux distro whereas the Cray linux compute + node is a stripped down version which modules are important + """ + self.assertEquals(True, False) From cfa7c4feb8419f82a864f9d1afcab69965e9c996 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 22 Feb 2016 16:18:25 -0800 Subject: [PATCH 141/481] Added CNL10 as back_os and default_os --- lib/spack/spack/platforms/cray_xc.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/platforms/cray_xc.py b/lib/spack/spack/platforms/cray_xc.py index b58c84d3eb..dffa2e4031 100644 --- a/lib/spack/spack/platforms/cray_xc.py +++ b/lib/spack/spack/platforms/cray_xc.py @@ -2,6 +2,7 @@ from spack.architecture import Platform, Target from spack.operating_system.linux_distro import LinuxDistro +from spack.operating_system.cnl import ComputeNodeLinux class CrayXc(Platform): priority = 20 @@ -10,8 +11,8 @@ class CrayXc(Platform): default = 'ivybridge' front_os = "SuSE11" - back_os = "CNL" - default_os = "CNL" + back_os = "CNL10" + default_os = "CNL10" def __init__(self): ''' Since cori doesn't have ivybridge as a front end it's better @@ -38,6 +39,7 @@ def __init__(self): Target('haswell', 'MODULES', 'craype-haswell')) self.add_operating_system('SuSE11', LinuxDistro()) + self.add_operating_system('CNL10', ComputeNodeLinux()) @classmethod def detect(self): From 5d5d3c5858a8882b615fb97fb07e335dc6c17f14 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 24 Feb 2016 12:15:52 -0800 Subject: [PATCH 142/481] Added to dictionary method --- lib/spack/spack/architecture.py | 95 +++++++++++++++++++++++++-------- 1 file changed, 74 insertions(+), 21 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 574076bae4..109e599470 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -89,7 +89,7 @@ def from_dict(d): return target def _cmp_key(self): - return (self.name, self.compiler_strategy, self.module_name) + return (self.name, self.module_name) def __repr__(self): return self.__str__() @@ -200,7 +200,6 @@ def __str__(self): def _cmp_key(self): return (self.name, (_cmp_key(t) for t in self.targets.values()), (_cmp_key(o) for o in self.operating_sys.values())) - @key_ordering class OperatingSystem(object): @@ -221,17 +220,19 @@ def __str__(self): def __repr__(self): return self.__str__() - def _cmp_key(self): - return (self.name, self.version) def compiler_strategy(self): """ Operating Systems will now be in charge of the compiler finding strategy. They will each have their own find_compilers method which depending on their compiler strategy will find the compilers - using a specific method (i.e PATH vs MODULES) + using a specific method (i.e PATH vs MODULES). """ raise NotImplementedError() + def _cmp_key(self): + return (self.name, self.version, self.compiler_strategy) + + #NOTE: Key error caused because Architecture has no comparison method @key_ordering class Arch(namedtuple("Arch", "platform platform_os target")): @@ -241,24 +242,77 @@ class Arch(namedtuple("Arch", "platform platform_os target")): __slots__ = () def __str__(self): - return (self.platform.name +"-"+ - self.platform_os.name + "-" + self.target.name) - + return (self.platform.name +"-"+ self.platform_os.name + "-" + self.target.name) + def _cmp_key(self): return (self.platform.name, self.platform_os.name, self.target.name) - @staticmethod - def to_dict(platform, os_name, target): - """ This function will handle all the converting of dictionaries - for each of the architecture classes. This is better than having - each class have a to_dict method when creating the dict of dicts - arch class - """ - d = {} - d['platform'] = platform - d['operating_system'] = os_name - d['target'] = target.to_dict() - return d + +def _helper_to_dict(arch_field_dict, arch_field_name, *args): + """ General method to turn each class in architecture into a + dictionary. Takes as argument the class dictionary, + """ + d = {} + d[arch_field_name] = {} + for items in args: + d[arch_field_name][items] = arch_field_dict[items] + return d + +def to_dict(arch): + d = {} + d['architecture'] = {} + + platform = arch.platform.__dict__ + platform_os = arch.platform_os.__dict__ + target = arch.target.__dict__ + + platform_dict = _helper_to_dict(platform,'platform','name') + os_dict = _helper_to_dict(platform_os, 'platform_os', 'name','version', + 'compiler_strategy') + target_dict = _helper_to_dict(target,'target', 'name', + 'module_name','platform_name') + + d['architecture'].update(platform_dict) + d['architecture'].update(os_dict) + d['architecture'].update(target_dict) + + return d + +def _platform_from_dict(platform): + platform_list = all_platforms() + platform_names = {plat.__name__.lower():plat for plat in platform_list} + return platform_names[platform['name']]() + + +def _target_from_dict(target): + target = Target.__new__(Target) + target.name = d['name'] + target.compiler_strategy = d['compiler_strategy'] + target.module_name = d['module_name'] + if 'platform' in d: + target.platform_name = d['platform'] + return target + +def _operating_system_from_dict(os_dict): + pass + +def arch_from_dict(d): + if d is None: + return None + arch = Arch + platform_dict = d['platform'] + platform_os_dict = d['platform_os'] + target_dict = d['target'] + + platform = _platform_from_dict(platform_dict) + platform_os = _operating_system_from_dict(platform_os_dict) + target = _target_from_dict(target_dict) + + arch.platform = platform + arch.platform_os = platform_os + arch.target = target + + return arch @memoized def all_platforms(): @@ -278,7 +332,6 @@ def all_platforms(): return modules - @memoized def sys_type(): """ Gather a list of all available subclasses of platforms. From 339f4bfd23b097d009009b6dabd524f8bc831449 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 24 Feb 2016 12:16:09 -0800 Subject: [PATCH 143/481] Added unit testing for to_dict method --- lib/spack/spack/test/architecture.py | 43 ++++++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/test/architecture.py b/lib/spack/spack/test/architecture.py index cf7938f5d6..5bc78d1b65 100644 --- a/lib/spack/spack/test/architecture.py +++ b/lib/spack/spack/test/architecture.py @@ -6,19 +6,58 @@ import platform import spack from spack.architecture import * +import spack.spec from spack.platforms.cray_xc import CrayXc from spack.platforms.linux import Linux from spack.platforms.bgq import Bgq from spack.platforms.darwin import Darwin class ArchitectureTest(unittest.TestCase): + + def setUp(self): + zlib = spack.spec.Spec("zlib") + zlib.concretize() + self.architecture = zlib.architecture + self.platform = sys_type() + self.platform_os = self.platform.operating_system('default_os') + self.target = self.platform.target('default') + + #def test_to_dict_function_with_target(self): + # d = spack.architecture.to_dict(self.architecture) + # print d['target'] + # self.assertEquals(d['target'], {'name': self.target.name, + # 'module_name' : self.target.module_name, + # 'platform_name' : self.target.platform_name, + # 'compiler_strategy': 'MODULES' + # }) + + def test_to_dict_function_with_architecture(self): + d = spack.architecture.to_dict(self.architecture) + self.assertEquals(d, {'architecture': + {'platform' : {'name': 'crayxc'}, + 'platform_os': { + 'compiler_strategy': 'MODULES', + 'name':'CNL', + 'version':'10'}, + 'target' : {'platform_name' :'crayxc', + 'module_name': 'craype-haswell', + 'name':'haswell'}}}) + + #def test_to_dict_function_with_operating_system(self): + # d = spack.architecture.to_dict(self.architecture) + # self.assertEquals(d['platform_os'], {'name': self.platform_os.name, + # 'version': self.platform_os.version, + # 'compiler_strategy': self.platform_os.compiler_strategy}) + + def test_architecture_from_dict(self): + pass def test_platform_class_and_compiler_strategies(self): a = CrayXc() - t = a.target('default') + t = a.operating_system('default_os') self.assertEquals(t.compiler_strategy, 'MODULES') b = Linux() - s = b.target('default') + s = b.operating_system('default_os') self.assertEquals(s.compiler_strategy, 'PATH') def test_sys_type(self): From 77799741430796974809fce0f08d099303ec6da6 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 24 Feb 2016 15:02:40 -0800 Subject: [PATCH 144/481] arch_from_dict worked on --- lib/spack/spack/architecture.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 109e599470..f0fc00242c 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -294,7 +294,11 @@ def _target_from_dict(target): return target def _operating_system_from_dict(os_dict): - pass + name = os_dict['name'] + os_list = all_platforms(True) + os_classes = {o.__name__.lower():o for o in os_list} + return os_classes[name]() + def arch_from_dict(d): if d is None: @@ -315,11 +319,19 @@ def arch_from_dict(d): return arch @memoized -def all_platforms(): +def all_platforms(operating_system=False): modules = [] - for name in list_modules(spack.platform_path): - mod_name = 'spack.platformss' + name - path = join_path(spack.platform_path, name) + ".py" + + if operating_system: + mod_path = spack.operating_system_path + mod_string = "spack.operating_system." + else: + mod_path = spack.platform_path + mod_string = "spack.platformss" + + for name in list_modules(mod_path): + mod_name = mod_string + name + path = join_path(mod_path, name) + ".py" mod = imp.load_source(mod_name, path) class_name = mod_to_class(name) if not hasattr(mod, class_name): From 9e844d974c2bce42b3e6e647c16b10aa122e54ce Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 24 Feb 2016 15:30:20 -0800 Subject: [PATCH 145/481] Added comments. Need to figure out how to make arch_tuple from_dict --- lib/spack/spack/architecture.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index f0fc00242c..4b24680501 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -250,7 +250,8 @@ def _cmp_key(self): def _helper_to_dict(arch_field_dict, arch_field_name, *args): """ General method to turn each class in architecture into a - dictionary. Takes as argument the class dictionary, + dictionary. Takes as argument the class dictionary, the field name + (platform, platform_os, target) and then any attribute args """ d = {} d[arch_field_name] = {} @@ -294,10 +295,11 @@ def _target_from_dict(target): return target def _operating_system_from_dict(os_dict): + #TODO: Have a way to recognize the OS subclasses name = os_dict['name'] os_list = all_platforms(True) - os_classes = {o.__name__.lower():o for o in os_list} - return os_classes[name]() + os_classes = {o.__name__:o for o in os_list} + def arch_from_dict(d): @@ -318,6 +320,7 @@ def arch_from_dict(d): return arch +#TODO: Haven't changed name here but all_platforms should now pull os class list @memoized def all_platforms(operating_system=False): modules = [] From 2de81cfc62ca456b1f303d64198590eb003fdcd3 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 24 Feb 2016 15:31:21 -0800 Subject: [PATCH 146/481] Changed name to appropriate camelcase --- lib/spack/spack/operating_system/cnl.py | 4 +- .../spack/operating_system/linux_distro.py | 10 ++--- lib/spack/spack/operating_system/mac_osx.py | 38 ++++++++----------- 3 files changed, 20 insertions(+), 32 deletions(-) diff --git a/lib/spack/spack/operating_system/cnl.py b/lib/spack/spack/operating_system/cnl.py index e52052dfa8..07221630f2 100644 --- a/lib/spack/spack/operating_system/cnl.py +++ b/lib/spack/spack/operating_system/cnl.py @@ -1,7 +1,7 @@ import spack from spack.architecture import OperatingSystem -class ComputeNodeLinux(OperatingSystem): +class Cnl(OperatingSystem): """ Compute Node Linux (CNL) is the operating system used for the Cray XC series super computers. It is a very stripped down version of GNU/Linux. Any compilers found through this operating system will be used with @@ -11,7 +11,7 @@ class ComputeNodeLinux(OperatingSystem): def __init__(self): name = 'CNL' version = '10' - super(ComputeNodeLinux, self).__init__(name, version, "MODULES") + super(Cnl, self).__init__(name, version, "MODULES") def compiler_strategy(self): return self.compiler_strategy diff --git a/lib/spack/spack/operating_system/linux_distro.py b/lib/spack/spack/operating_system/linux_distro.py index b11c7a88fa..1345b7d618 100644 --- a/lib/spack/spack/operating_system/linux_distro.py +++ b/lib/spack/spack/operating_system/linux_distro.py @@ -1,6 +1,6 @@ import platform as py_platform import spack -from spack.architecture import Platform, OperatingSystem +from spack.architecture import OperatingSystem class LinuxDistro(OperatingSystem): """ This class will represent the autodetected operating system @@ -10,13 +10,9 @@ class LinuxDistro(OperatingSystem): platform.dist() """ def __init__(self): - def detect_operating_system(): - name = py_platform.dist()[0] - version = py_platform.dist()[1] - return name, version + name = py_platform.dist()[0] + version = py_platform.dist()[1] - name, version = detect_operating_system() - super(LinuxDistro, self).__init__(name, version, "PATH") def compiler_strategy(self): diff --git a/lib/spack/spack/operating_system/mac_osx.py b/lib/spack/spack/operating_system/mac_osx.py index f52cdd6bc7..6fb6fef138 100644 --- a/lib/spack/spack/operating_system/mac_osx.py +++ b/lib/spack/spack/operating_system/mac_osx.py @@ -3,38 +3,30 @@ will be represented using the major version operating system name, i.e el capitan, yosemite...etc. """ - -import spack -import os import platform as py_platform -from spack.architecture import Platform, OperatingSystem +import spack +from spack.architecture import OperatingSystem -class MacOSX(OperatingSystem): +class MacOsx(OperatingSystem): def __init__(self): """ Autodetects the mac version from a dictionary. Goes back as far as 10.6 snowleopard. If the user has an older mac then the version will just be a generic mac_os. """ - - def get_mac_release(): - mac_releases = {'10.6': "snowleopard", - "10.7": "lion", - "10.8": "mountainlion", - "10.9": "mavericks", - "10.10": "yosemite", - "10.11": "elcapitan"} + mac_releases = {'10.6': "snowleopard", + "10.7": "lion", + "10.8": "mountainlion", + "10.9": "mavericks", + "10.10": "yosemite", + "10.11": "elcapitan"} - mac_ver = py_platform.mac_ver()[0][:-2] - try: - name = mac_releases[mac_ver] - return name, mac_ver - except KeyError: - name = "mac_os" - return name, mac_ver + mac_ver = py_platform.mac_ver()[0][:-2] + try: + name = mac_releases[mac_ver] + except KeyError: + name = "mac_os" - name, version = get_mac_release() - - super(MacOSX, self).__init__(name, version, "PATH") + super(MacOsx, self).__init__(name, mac_ver, "PATH") def compiler_strategy(self): return self.compiler_strategy From 234681306cdfb9aea9772781fcb68c4045bdc315 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 24 Feb 2016 15:32:23 -0800 Subject: [PATCH 147/481] Changed to appropriate class names for operating system --- lib/spack/spack/platforms/cray_xc.py | 5 ++--- lib/spack/spack/platforms/darwin.py | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/lib/spack/spack/platforms/cray_xc.py b/lib/spack/spack/platforms/cray_xc.py index dffa2e4031..da90726fef 100644 --- a/lib/spack/spack/platforms/cray_xc.py +++ b/lib/spack/spack/platforms/cray_xc.py @@ -1,8 +1,7 @@ import os - from spack.architecture import Platform, Target from spack.operating_system.linux_distro import LinuxDistro -from spack.operating_system.cnl import ComputeNodeLinux +from spack.operating_system.cnl import Cnl class CrayXc(Platform): priority = 20 @@ -39,7 +38,7 @@ def __init__(self): Target('haswell', 'MODULES', 'craype-haswell')) self.add_operating_system('SuSE11', LinuxDistro()) - self.add_operating_system('CNL10', ComputeNodeLinux()) + self.add_operating_system('CNL10', Cnl()) @classmethod def detect(self): diff --git a/lib/spack/spack/platforms/darwin.py b/lib/spack/spack/platforms/darwin.py index 5b797a78db..4512857371 100644 --- a/lib/spack/spack/platforms/darwin.py +++ b/lib/spack/spack/platforms/darwin.py @@ -1,6 +1,6 @@ import subprocess from spack.architecture import Platform, Target -from spack.operating_system.mac_osx import MacOSX +from spack.operating_system.mac_osx import MacOsx class Darwin(Platform): priority = 89 @@ -11,7 +11,7 @@ class Darwin(Platform): def __init__(self): super(Darwin, self).__init__('darwin') self.add_target(self.default, Target(self.default, 'PATH')) - mac_os = MacOSX() + mac_os = MacOsx() self.default_os = mac_os.name self.add_operating_system(mac_os.name, mac_os) From 5c3c6e7f066a3f95f9113c46640546516d7a8257 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 24 Feb 2016 15:34:32 -0800 Subject: [PATCH 148/481] Got rid of print statements --- lib/spack/spack/test/operating_system.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/spack/spack/test/operating_system.py b/lib/spack/spack/test/operating_system.py index a4327dde83..205408db3f 100644 --- a/lib/spack/spack/test/operating_system.py +++ b/lib/spack/spack/test/operating_system.py @@ -38,7 +38,6 @@ def test_cray_back_end_compiler_strategy(self): self.assertEquals(self.cray_back_os.compiler_strategy, "MODULES") def test_linux_operating_system(self): - print self.linux_operating_sys self.assertIsInstance(self.linux_operating_sys, LinuxDistro) def test_linux_compiler_strategy(self): From bd7c189c123154e88e092eadbbd93b00ce43cc6a Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 24 Feb 2016 15:37:50 -0800 Subject: [PATCH 149/481] Added more comments --- lib/spack/spack/spec.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 0b42498910..c8c114a8b1 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -661,12 +661,14 @@ def to_node_dict(self): 'dependencies' : dict((d, self.dependencies[d].dag_hash()) for d in sorted(self.dependencies)) } + if self.architecture: # TODO: Fix the target.to_dict to account for the tuple # Want it to be a dict of dicts - d['architecture'] = self.architecture.target.to_dict() + d['architecture'] = spack.architecture.to_dict(self.architecture) else: d['architecture'] = None + if self.compiler: d.update(self.compiler.to_dict()) else: @@ -693,8 +695,7 @@ def from_node_dict(node): spec = Spec(name) spec.versions = VersionList.from_dict(node) # TODO: Need to fix the architecture.Target.from_dict - spec.architecture = spack.architecture.Target.from_dict( - node['architecture']) + spec.architecture = spack.architecture.from_dict(node['architecture']) if node['compiler'] is None: spec.compiler = None @@ -1292,17 +1293,17 @@ def add_architecture_from_string(self, arch): for entry in arch_list: if self._is_valid_platform(entry, platform_names): + # If entry is different from platform name then create it. + # else just keep the already instantiated platform class if entry != platform.name: platform = platform_dict[entry]() # Create instance of platform elif self._is_valid_target(entry, platform): target = platform.target(entry) - # Need to figure out if we're supporting arbitrary os's and how - # to account for them - # Not really a good implementation since the user can add - # gibberish and spack will see it as an os + # check if os is valid by checking platform operating sys dict elif self._is_valid_os(entry, platform): platform_os = platform.operating_system(entry) else: + # throw error since entry is unknown raise UnknownArchitectureSpecError(entry) if target is None: From 8d1c06d14125d86c24584c7e6f0a9ab0d300d10f Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 24 Feb 2016 15:38:33 -0800 Subject: [PATCH 150/481] Changed to dict method. Didn't realize this was being assigned to a dictionary already so got rid of d['architecture'] --- lib/spack/spack/architecture.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 4b24680501..e76e411ae0 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -261,8 +261,7 @@ def _helper_to_dict(arch_field_dict, arch_field_name, *args): def to_dict(arch): d = {} - d['architecture'] = {} - + platform = arch.platform.__dict__ platform_os = arch.platform_os.__dict__ target = arch.target.__dict__ @@ -273,9 +272,9 @@ def to_dict(arch): target_dict = _helper_to_dict(target,'target', 'name', 'module_name','platform_name') - d['architecture'].update(platform_dict) - d['architecture'].update(os_dict) - d['architecture'].update(target_dict) + d.update(platform_dict) + d.update(os_dict) + d.update(target_dict) return d From 863a5bc4fa7d6b880769192e4fcb7a461a6a0063 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 25 Feb 2016 10:34:55 -0800 Subject: [PATCH 151/481] Fix for 'Parent module spack.operating_systems not found while handling absolute import' --- lib/spack/spack/architecture.py | 40 ++++++++++++++++----------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index e76e411ae0..5827427003 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -67,26 +67,26 @@ def __init__(self, name, compiler_strategy, module_name=None): def set_platform(self, platform): self.platform_name = platform.name - def to_dict(self): - d = {} - d['name'] = self.name - d['compiler_strategy'] = self.compiler_strategy - d['module_name'] = self.module_name - if self.platform_name: - d['platform'] = self.platform_name - return d + #def to_dict(self): + # d = {} + # d['name'] = self.name + # d['compiler_strategy'] = self.compiler_strategy + # d['module_name'] = self.module_name + # if self.platform_name: + # d['platform'] = self.platform_name + # return d - @staticmethod - def from_dict(d): - if d is None: - return None - target = Target.__new__(Target) - target.name = d['name'] - target.compiler_strategy = d['compiler_strategy'] - target.module_name = d['module_name'] - if 'platform' in d: - target.platform_name = d['platform'] - return target + #@staticmethod + #def from_dict(d): + # if d is None: + # return None + # target = Target.__new__(Target) + # target.name = d['name'] + # target.compiler_strategy = d['compiler_strategy'] + # target.module_name = d['module_name'] + # if 'platform' in d: + # target.platform_name = d['platform'] + # return target def _cmp_key(self): return (self.name, self.module_name) @@ -326,7 +326,7 @@ def all_platforms(operating_system=False): if operating_system: mod_path = spack.operating_system_path - mod_string = "spack.operating_system." + mod_string = "spack.operating_systems" else: mod_path = spack.platform_path mod_string = "spack.platformss" From 34be473b7cfbb23d4deb5c3f9774bc1acea22574 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 25 Feb 2016 10:35:58 -0800 Subject: [PATCH 152/481] Deleted files --- lib/spack/spack/operating_systems/cnl.py | 19 +++++++++ .../spack/operating_systems/linux_distro.py | 24 +++++++++++ lib/spack/spack/operating_systems/mac_osx.py | 40 +++++++++++++++++++ 3 files changed, 83 insertions(+) create mode 100644 lib/spack/spack/operating_systems/cnl.py create mode 100644 lib/spack/spack/operating_systems/linux_distro.py create mode 100644 lib/spack/spack/operating_systems/mac_osx.py diff --git a/lib/spack/spack/operating_systems/cnl.py b/lib/spack/spack/operating_systems/cnl.py new file mode 100644 index 0000000000..20cb4c215b --- /dev/null +++ b/lib/spack/spack/operating_systems/cnl.py @@ -0,0 +1,19 @@ +from spack.architecture import OperatingSystem + +class Cnl(OperatingSystem): + """ Compute Node Linux (CNL) is the operating system used for the Cray XC + series super computers. It is a very stripped down version of GNU/Linux. + Any compilers found through this operating system will be used with + modules. If updated, user must make sure that version and name are + updated to indicate that OS has been upgraded (or downgraded) + """ + def __init__(self): + name = 'CNL' + version = '10' + super(Cnl, self).__init__(name, version, "MODULES") + + def compiler_strategy(self): + return self.compiler_strategy + + def find_compilers(self): + pass diff --git a/lib/spack/spack/operating_systems/linux_distro.py b/lib/spack/spack/operating_systems/linux_distro.py new file mode 100644 index 0000000000..c5547749d8 --- /dev/null +++ b/lib/spack/spack/operating_systems/linux_distro.py @@ -0,0 +1,24 @@ +import platform as py_platform +from spack.architecture import OperatingSystem + +class LinuxDistro(OperatingSystem): + """ This class will represent the autodetected operating system + for a Linux System. Since there are many different flavors of + Linux, this class will attempt to encompass them all through + autodetection using the python module platform and the method + platform.dist() + """ + def __init__(self): + name = py_platform.dist()[0] + version = py_platform.dist()[1] + + super(LinuxDistro, self).__init__(name, version, "PATH") + + def compiler_strategy(self): + return self.compiler_strategy + + def find_compilers(self): + pass + + + diff --git a/lib/spack/spack/operating_systems/mac_osx.py b/lib/spack/spack/operating_systems/mac_osx.py new file mode 100644 index 0000000000..555c5cfb14 --- /dev/null +++ b/lib/spack/spack/operating_systems/mac_osx.py @@ -0,0 +1,40 @@ +import platform as py_platform +from spack.architecture import OperatingSystem + +class MacOsx(OperatingSystem): + """ This class represents the MAC_OSX operating system. This will be auto + detected using the python platform.mac_ver. The MAC_OSX platform + will be represented using the major version operating system name, i.e + el capitan, yosemite...etc. + """ + + def __init__(self): + """ Autodetects the mac version from a dictionary. Goes back as + far as 10.6 snowleopard. If the user has an older mac then + the version will just be a generic mac_os. + """ + mac_releases = {'10.6': "snowleopard", + "10.7": "lion", + "10.8": "mountainlion", + "10.9": "mavericks", + "10.10": "yosemite", + "10.11": "elcapitan"} + + mac_ver = py_platform.mac_ver()[0][:-2] + try: + name = mac_releases[mac_ver] + except KeyError: + name = "mac_os" + + super(MacOsx, self).__init__(name, mac_ver, "PATH") + + def compiler_strategy(self): + return self.compiler_strategy + + def find_compilers(self): + pass + + + + + From f1616099292251f208ec0cfdec3d1b092d4441b9 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 25 Feb 2016 10:36:12 -0800 Subject: [PATCH 153/481] Renamed operating_system to operating_systems --- lib/spack/spack/operating_systems/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 lib/spack/spack/operating_systems/__init__.py diff --git a/lib/spack/spack/operating_systems/__init__.py b/lib/spack/spack/operating_systems/__init__.py new file mode 100644 index 0000000000..e69de29bb2 From b43a5498a159c634ca88341d3010bc1c98197e24 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 25 Feb 2016 10:36:47 -0800 Subject: [PATCH 154/481] Changed import path to operating_systems --- lib/spack/spack/platforms/bgq.py | 1 - lib/spack/spack/platforms/cray_xc.py | 4 ++-- lib/spack/spack/platforms/darwin.py | 2 +- lib/spack/spack/platforms/linux.py | 2 +- 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/lib/spack/spack/platforms/bgq.py b/lib/spack/spack/platforms/bgq.py index 6e872d2e72..b550f1d5cb 100644 --- a/lib/spack/spack/platforms/bgq.py +++ b/lib/spack/spack/platforms/bgq.py @@ -1,5 +1,4 @@ import os - from spack.architecture import Platform, Target class Bgq(Platform): diff --git a/lib/spack/spack/platforms/cray_xc.py b/lib/spack/spack/platforms/cray_xc.py index da90726fef..ea3ae0e477 100644 --- a/lib/spack/spack/platforms/cray_xc.py +++ b/lib/spack/spack/platforms/cray_xc.py @@ -1,7 +1,7 @@ import os from spack.architecture import Platform, Target -from spack.operating_system.linux_distro import LinuxDistro -from spack.operating_system.cnl import Cnl +from spack.operating_systems.linux_distro import LinuxDistro +from spack.operating_systems.cnl import Cnl class CrayXc(Platform): priority = 20 diff --git a/lib/spack/spack/platforms/darwin.py b/lib/spack/spack/platforms/darwin.py index 4512857371..c48f7d17d2 100644 --- a/lib/spack/spack/platforms/darwin.py +++ b/lib/spack/spack/platforms/darwin.py @@ -1,6 +1,6 @@ import subprocess from spack.architecture import Platform, Target -from spack.operating_system.mac_osx import MacOsx +from spack.operating_systems.mac_osx import MacOsx class Darwin(Platform): priority = 89 diff --git a/lib/spack/spack/platforms/linux.py b/lib/spack/spack/platforms/linux.py index 220f1bbaf4..1b22b69048 100644 --- a/lib/spack/spack/platforms/linux.py +++ b/lib/spack/spack/platforms/linux.py @@ -1,6 +1,6 @@ import subprocess from spack.architecture import Platform, Target -from spack.operating_system.linux_distro import LinuxDistro +from spack.operating_systems.linux_distro import LinuxDistro class Linux(Platform): priority = 90 From 9ac2556285ae38dd6d661d9bba811a5e007cb551 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 25 Feb 2016 10:41:28 -0800 Subject: [PATCH 155/481] Deleted old operating_system (without the s) file --- lib/spack/spack/operating_system/__init__.py | 0 lib/spack/spack/operating_system/cnl.py | 20 ---------- .../spack/operating_system/linux_distro.py | 25 ------------ lib/spack/spack/operating_system/mac_osx.py | 40 ------------------- 4 files changed, 85 deletions(-) delete mode 100644 lib/spack/spack/operating_system/__init__.py delete mode 100644 lib/spack/spack/operating_system/cnl.py delete mode 100644 lib/spack/spack/operating_system/linux_distro.py delete mode 100644 lib/spack/spack/operating_system/mac_osx.py diff --git a/lib/spack/spack/operating_system/__init__.py b/lib/spack/spack/operating_system/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/lib/spack/spack/operating_system/cnl.py b/lib/spack/spack/operating_system/cnl.py deleted file mode 100644 index 07221630f2..0000000000 --- a/lib/spack/spack/operating_system/cnl.py +++ /dev/null @@ -1,20 +0,0 @@ -import spack -from spack.architecture import OperatingSystem - -class Cnl(OperatingSystem): - """ Compute Node Linux (CNL) is the operating system used for the Cray XC - series super computers. It is a very stripped down version of GNU/Linux. - Any compilers found through this operating system will be used with - modules. If updated, user must make sure that version and name are - updated to indicate that OS has been upgraded (or downgraded) - """ - def __init__(self): - name = 'CNL' - version = '10' - super(Cnl, self).__init__(name, version, "MODULES") - - def compiler_strategy(self): - return self.compiler_strategy - - def find_compilers(self): - pass diff --git a/lib/spack/spack/operating_system/linux_distro.py b/lib/spack/spack/operating_system/linux_distro.py deleted file mode 100644 index 1345b7d618..0000000000 --- a/lib/spack/spack/operating_system/linux_distro.py +++ /dev/null @@ -1,25 +0,0 @@ -import platform as py_platform -import spack -from spack.architecture import OperatingSystem - -class LinuxDistro(OperatingSystem): - """ This class will represent the autodetected operating system - for a Linux System. Since there are many different flavors of - Linux, this class will attempt to encompass them all through - autodetection using the python module platform and the method - platform.dist() - """ - def __init__(self): - name = py_platform.dist()[0] - version = py_platform.dist()[1] - - super(LinuxDistro, self).__init__(name, version, "PATH") - - def compiler_strategy(self): - return self.compiler_strategy - - def find_compilers(self): - pass - - - diff --git a/lib/spack/spack/operating_system/mac_osx.py b/lib/spack/spack/operating_system/mac_osx.py deleted file mode 100644 index 6fb6fef138..0000000000 --- a/lib/spack/spack/operating_system/mac_osx.py +++ /dev/null @@ -1,40 +0,0 @@ -""" This class represents the MAC_OSX operating system. This will be auto - detected using the python platform.mac_ver. The MAC_OSX platform - will be represented using the major version operating system name, i.e - el capitan, yosemite...etc. -""" -import platform as py_platform -import spack -from spack.architecture import OperatingSystem - -class MacOsx(OperatingSystem): - def __init__(self): - """ Autodetects the mac version from a dictionary. Goes back as - far as 10.6 snowleopard. If the user has an older mac then - the version will just be a generic mac_os. - """ - mac_releases = {'10.6': "snowleopard", - "10.7": "lion", - "10.8": "mountainlion", - "10.9": "mavericks", - "10.10": "yosemite", - "10.11": "elcapitan"} - - mac_ver = py_platform.mac_ver()[0][:-2] - try: - name = mac_releases[mac_ver] - except KeyError: - name = "mac_os" - - super(MacOsx, self).__init__(name, mac_ver, "PATH") - - def compiler_strategy(self): - return self.compiler_strategy - - def find_compilers(self): - pass - - - - - From 550df4cdd6ce1760370e18a858da624167a3139a Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 25 Feb 2016 11:47:27 -0800 Subject: [PATCH 156/481] Some cleaning up. Finally got arch_from_dict working successfully. --- lib/spack/spack/architecture.py | 81 ++++++--------------------------- 1 file changed, 15 insertions(+), 66 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 5827427003..7bf5afcf63 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -67,27 +67,6 @@ def __init__(self, name, compiler_strategy, module_name=None): def set_platform(self, platform): self.platform_name = platform.name - #def to_dict(self): - # d = {} - # d['name'] = self.name - # d['compiler_strategy'] = self.compiler_strategy - # d['module_name'] = self.module_name - # if self.platform_name: - # d['platform'] = self.platform_name - # return d - - #@staticmethod - #def from_dict(d): - # if d is None: - # return None - # target = Target.__new__(Target) - # target.name = d['name'] - # target.compiler_strategy = d['compiler_strategy'] - # target.module_name = d['module_name'] - # if 'platform' in d: - # target.platform_name = d['platform'] - # return target - def _cmp_key(self): return (self.name, self.module_name) @@ -147,24 +126,6 @@ def target(self, name): name = self.back_end return self.targets[name] - - #def _detect_linux_os(self): - # return OperatingSystem(py_platform.dist()[0], py_platform.dist()[1]) - - #def _detect_mac_os(self): - # mac_releases = {'10.6': "snowleopard", - # "10.7": "lion", - # "10.8": "mountainlion", - # "10.9": "mavericks", - # "10.10": "yosemite", - # "10.11": "elcapitan"} - # mac_ver = py_platform.mac_ver()[:-2] - # try: - # os_name = mac_releases[mac_ver] - # return OperatingSystem(os_name, mac_ver) - # except KeyError: - # os_name = "mac_os" - # return OperatingSystem(os_name, mac_ver) def add_operating_system(self, name, os_class): """ Add the operating_system class object into the @@ -242,7 +203,8 @@ class Arch(namedtuple("Arch", "platform platform_os target")): __slots__ = () def __str__(self): - return (self.platform.name +"-"+ self.platform_os.name + "-" + self.target.name) + return (self.platform.name +"-"+ + self.platform_os.name + "-" + self.target.name) def _cmp_key(self): return (self.platform.name, self.platform_os.name, self.target.name) @@ -284,52 +246,39 @@ def _platform_from_dict(platform): return platform_names[platform['name']]() -def _target_from_dict(target): +def _target_from_dict(target_dict): target = Target.__new__(Target) - target.name = d['name'] - target.compiler_strategy = d['compiler_strategy'] - target.module_name = d['module_name'] - if 'platform' in d: - target.platform_name = d['platform'] + target.name = target_dict['name'] + #target.compiler_strategy = target_dict['compiler_strategy'] + target.module_name = target_dict['module_name'] + if 'platform_name' in target_dict: + target.platform_name = target_dict['platform_name'] return target -def _operating_system_from_dict(os_dict): - #TODO: Have a way to recognize the OS subclasses +def _operating_system_from_dict(os_dict, platform_class): name = os_dict['name'] - os_list = all_platforms(True) - os_classes = {o.__name__:o for o in os_list} - + return platform_class.operating_system(name) def arch_from_dict(d): if d is None: return None - arch = Arch platform_dict = d['platform'] platform_os_dict = d['platform_os'] target_dict = d['target'] platform = _platform_from_dict(platform_dict) - platform_os = _operating_system_from_dict(platform_os_dict) + platform_os = _operating_system_from_dict(platform_os_dict, platform) target = _target_from_dict(target_dict) - arch.platform = platform - arch.platform_os = platform_os - arch.target = target + return Arch(platform, platform_os, target) - return arch - -#TODO: Haven't changed name here but all_platforms should now pull os class list @memoized -def all_platforms(operating_system=False): +def all_platforms(): modules = [] - if operating_system: - mod_path = spack.operating_system_path - mod_string = "spack.operating_systems" - else: - mod_path = spack.platform_path - mod_string = "spack.platformss" + mod_path = spack.platform_path + mod_string = "spack.platformss" for name in list_modules(mod_path): mod_name = mod_string + name From 5e4d3e7b8280e633bb9b9dd28510dc394feb2000 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 25 Feb 2016 11:48:43 -0800 Subject: [PATCH 157/481] Changed naming convention of operating_system, makes it easier to create instance from dictionary just using name without the version attached. Object will stil display as name+version --- lib/spack/spack/platforms/cray_xc.py | 10 +++++----- lib/spack/spack/platforms/linux.py | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/spack/spack/platforms/cray_xc.py b/lib/spack/spack/platforms/cray_xc.py index ea3ae0e477..b71481e46e 100644 --- a/lib/spack/spack/platforms/cray_xc.py +++ b/lib/spack/spack/platforms/cray_xc.py @@ -9,9 +9,9 @@ class CrayXc(Platform): back_end = 'ivybridge' default = 'ivybridge' - front_os = "SuSE11" - back_os = "CNL10" - default_os = "CNL10" + front_os = "SuSE" + back_os = "CNL" + default_os = "CNL" def __init__(self): ''' Since cori doesn't have ivybridge as a front end it's better @@ -37,8 +37,8 @@ def __init__(self): self.add_target('haswell', Target('haswell', 'MODULES', 'craype-haswell')) - self.add_operating_system('SuSE11', LinuxDistro()) - self.add_operating_system('CNL10', Cnl()) + self.add_operating_system('SuSE', LinuxDistro()) + self.add_operating_system('CNL', Cnl()) @classmethod def detect(self): diff --git a/lib/spack/spack/platforms/linux.py b/lib/spack/spack/platforms/linux.py index 1b22b69048..d9e33511a1 100644 --- a/lib/spack/spack/platforms/linux.py +++ b/lib/spack/spack/platforms/linux.py @@ -12,8 +12,8 @@ def __init__(self): super(Linux, self).__init__('linux') self.add_target(self.default, Target(self.default, 'PATH')) linux_dist = LinuxDistro() - self.default_os = str(linux_dist) - self.add_operating_system(str(linux_dist), linux_dist) + self.default_os = linux_dist.name + self.add_operating_system(linux_dist.name, linux_dist) @classmethod def detect(self): From 5989e3f65d83afdd497059bba9442576e2df0509 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 25 Feb 2016 11:53:02 -0800 Subject: [PATCH 158/481] Changed from_dict to arch_from_dict method --- lib/spack/spack/spec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index c8c114a8b1..abe17a8f2b 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -695,7 +695,7 @@ def from_node_dict(node): spec = Spec(name) spec.versions = VersionList.from_dict(node) # TODO: Need to fix the architecture.Target.from_dict - spec.architecture = spack.architecture.from_dict(node['architecture']) + spec.architecture = spack.architecture.arch_from_dict(node['architecture']) if node['compiler'] is None: spec.compiler = None From ac55ce989d0b5a6e7a1160dc32ac7fb12f805f76 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 25 Feb 2016 18:47:41 -0800 Subject: [PATCH 159/481] Changed compiler finding strategy to come from operating system instead of target --- lib/spack/spack/concretize.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 0e7bf53b44..3434ca66f7 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -334,9 +334,9 @@ def concretize_compiler(self, spec): # Should think whether this can be more efficient def _proper_compiler_style(cspec, architecture): compilers = spack.compilers.compilers_for_spec(cspec) - if architecture.target.compiler_strategy == 'PATH': + if architecture.platform_os.compiler_strategy == 'PATH': filter(lambda c: not c.modules, compilers) - if architecture.target.compiler_strategy == 'MODULES': + if architecture.platform_os.compiler_strategy == 'MODULES': filter(lambda c: c.modules, compilers) return compilers From f0149faf88b97b63635ba451efe008846478d115 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 25 Feb 2016 18:48:00 -0800 Subject: [PATCH 160/481] Added doc strings to functions in architecture.py --- lib/spack/spack/architecture.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 7bf5afcf63..14a8c56793 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -222,6 +222,12 @@ def _helper_to_dict(arch_field_dict, arch_field_name, *args): return d def to_dict(arch): + """ Convert the Arch tuple into a dictionary for yaml dumping. This + uses the _helper_to_dict method to create the dictionary from the + provided architecture field. Can assign the architecture + field name (either platform, platform_os or target) and any + attributes that make up that architecture field, + """ d = {} platform = arch.platform.__dict__ @@ -241,12 +247,19 @@ def to_dict(arch): return d def _platform_from_dict(platform): + """Creates all the platform class module names into a dictionary of + name : key-value pairs. From there we can construct the + platform subclass + """ platform_list = all_platforms() platform_names = {plat.__name__.lower():plat for plat in platform_list} return platform_names[platform['name']]() def _target_from_dict(target_dict): + """ Creates new instance of target and assigns all the attributes of + that target from the dictionary + """ target = Target.__new__(Target) target.name = target_dict['name'] #target.compiler_strategy = target_dict['compiler_strategy'] @@ -256,11 +269,19 @@ def _target_from_dict(target_dict): return target def _operating_system_from_dict(os_dict, platform_class): + """ uses platform's operating system method to grab the constructed + operating systems that are valid on the platform. + """ +# NOTE: Might need a better way to create operating system objects name = os_dict['name'] return platform_class.operating_system(name) def arch_from_dict(d): + """ Uses _platform_from_dict, _operating_system_from_dict, _target_from_dict + helper methods to recreate the arch tuple from the dictionary read from + a yaml file + """ if d is None: return None platform_dict = d['platform'] From eb96f3829865492c3c484de0d89e14cba578a37f Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 25 Feb 2016 18:48:25 -0800 Subject: [PATCH 161/481] Got rid of debug comment --- lib/spack/spack/spec.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index abe17a8f2b..0a55ec5d76 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -835,7 +835,6 @@ def concretize(self): changed = True force = False - # Loops forever here in my implementation while changed: changes = (self.normalize(force=force), self._expand_virtual_packages(), From ccd4a79b39b56a8408980ed27b0c5de5a1995fbc Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 25 Feb 2016 18:49:27 -0800 Subject: [PATCH 162/481] changed architecture.target to architecture.platform_os, so that compiler_for_spec uses operating system strategy --- lib/spack/spack/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index b0d84863de..5d922f6e58 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -594,7 +594,7 @@ def compiler(self): if not self.spec.concrete: raise ValueError("Can only get a compiler for a concrete package.") return spack.compilers.compiler_for_spec(self.spec.compiler, - self.spec.architecture.target) + self.spec.architecture.platform_os) def url_version(self, version): From b591d2b5015fdde5d551dd81a8906ac0b2a51260 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 25 Feb 2016 18:49:52 -0800 Subject: [PATCH 163/481] Changed operating_system path --- lib/spack/spack/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 6ad9b87ca2..48fa9edb67 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -39,7 +39,7 @@ module_path = join_path(lib_path, "spack") platform_path = join_path(module_path, 'platforms') compilers_path = join_path(module_path, "compilers") -operating_system_path = join_path(module_path, 'operating_system') +operating_system_path = join_path(module_path, 'operating_systems') test_path = join_path(module_path, "test") hooks_path = join_path(module_path, "hooks") var_path = join_path(prefix, "var", "spack") From 4601c36f572ed7cf66f539c11ebe0835976655dc Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 2 Mar 2016 09:55:44 -0800 Subject: [PATCH 164/481] cleaned up concretize architecture --- lib/spack/spack/concretize.py | 91 ++++++++++++++++++++++++----------- 1 file changed, 64 insertions(+), 27 deletions(-) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 3434ca66f7..04f932aa5d 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -218,20 +218,55 @@ def _concretize_platform(self, arch, platform): return True def _concretize_operating_system(self, arch, platform): - """ Future method for concretizing operating system """ - if isinstance(arch.platform_os, spack.architecture.OperatingSystem): - return False + if spec.architecture.platform_os is not None: + if isinstance(spec.architecture.platform_os,spack.architecture.OperatingSystem): + return False + else: + spec.add_operating_system_from_string(spec.architecture.platform_os) + return True #changed + if spec.root.architecture.platform_os: + if isinstance(spec.root.architecture.platform_os,spack.architecture.OperatingSystem): + spec.architecture.platform_os = spec.root.architecture.platform_os + else: + spec.add_operating_system_from_string(spec.root.architecture.platform_os) else: - arch.arch_os = platform.operating_system('default_os') - return True + platform = spack.architecture.sys_type() + spec.architecture.platform_os = platform.operating_system('default_os') + + return True #changed + +# """ Future method for concretizing operating system """ +# if isinstance(arch.platform_os, spack.architecture.OperatingSystem): +# return False +# else: +# arch.arch_os = platform.operating_system('default_os') +# return True def _concretize_target(self, arch, platform): - if isinstance(arch.target, spack.architecture.Target): - return False + if spec.target is not None: + if isinstance(spec.target,spack.architecture.Target): + return False + else: + spec.add_target_from_string(spec.target) + return True #changed + + if spec.root.target: + if isinstance(spec.root.target,spack.architecture.Target): + spec.target = spec.root.target + else: + spec.add_target_from_string(spec.root.target) else: - arch.target = platform.target('default') - return True + platform = spack.architecture.sys_type() + spec.target = platform.target('default') + + return True #changed + +# if isinstance(arch.target, spack.architecture.Target): +# return False +# else: +# arch.target = platform.target('default') +# return True def concretize_architecture(self, spec): """If the spec is empty provide the defaults of the platform. If the @@ -251,33 +286,35 @@ def concretize_architecture(self, spec): Arch = spack.architecture.Arch # Set the architecture to all defaults spec.architecture = Arch(platform=platform, - platform_os=platform.operating_system('default_os'), - target=platform.target('default')) + None, + None) return True #If there is a target and it is a tuple and has both filled return #False - if not isinstance(spec.architecture, basestring): - return any(( +# if isinstance(spec.architecture, basestring): +# spec.split_architecture_string(spec.architecture) + + + ret = any(( self._concretize_platform(spec.architecture, platform), self._concretize_operating_system(spec.architecture, platform), self._concretize_target(spec.architecture, platform))) - else: - spec.add_architecture_from_string(spec.architecture) + # Does not look pretty at all!!! - if spec.root.architecture and \ - not isinstance(spec.root.architecture, basestring): - bool_flag = any(( - self._concretize_platform(spec.root.architecture, platform), - self._concretize_operating_system(spec.root.architecture, - platform), - self._concretize_target(spec.root.target, platform))) - spec.architecture =spec.root.architecture - return bool_flag - else: - spec.add_architecture_from_string(spec.root.architecture) +# if spec.root.architecture and \ +# not isinstance(spec.root.architecture, basestring): +# bool_flag = any(( +# self._concretize_platform(spec.root.architecture, platform), +# self._concretize_operating_system(spec.root.architecture, +# platform), +# self._concretize_target(spec.root.target, platform))) +# spec.architecture =spec.root.architecture +# return bool_flag +# else: +# spec.add_architecture_from_string(spec.root.architecture) - return True + return ret # if there is no target specified used the defaults From 615ea969f8a998417dfbbd07935848eb37f71c47 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 2 Mar 2016 10:14:50 -0800 Subject: [PATCH 165/481] made arch instantiate as a tuple --- lib/spack/spack/spec.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 0a55ec5d76..57dd0c2181 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -461,7 +461,13 @@ def _set_architecture(self, architecture): """Called by the parser to set the architecture.""" if self.architecture: raise DuplicateArchitectureError( "Spec for '%s' cannot have two architectures." % self.name) - self.architecture = architecture # a string can be set + platform = spack.architecture.sys_type() + if '-' in architecture: + os, target = architecture.split('-') + else: + os = architecture + target = None + self.architecture = spack.architecture.Arch(platform, os, target) def _add_dependency(self, spec): From 527bb7abfeeb982d0bf22dadf4b8400502b606dd Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 2 Mar 2016 10:26:09 -0800 Subject: [PATCH 166/481] Changed _cmp_key --- lib/spack/spack/architecture.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 14a8c56793..dac3a45b79 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -207,7 +207,7 @@ def __str__(self): self.platform_os.name + "-" + self.target.name) def _cmp_key(self): - return (self.platform.name, self.platform_os.name, self.target.name) + return (self.platform, self.platform_os, self.target) def _helper_to_dict(arch_field_dict, arch_field_name, *args): From 4d7478420968182ab9535de9deffbdd671937972 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 2 Mar 2016 10:27:00 -0800 Subject: [PATCH 167/481] Changed add_architecture_from_string and split the work done into two methods add_target_from_string and add_operating_system_from_string --- lib/spack/spack/spec.py | 122 ++++++++++++++++++++++------------------ 1 file changed, 67 insertions(+), 55 deletions(-) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 57dd0c2181..06accc514b 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1256,70 +1256,82 @@ def _is_valid_os(self, os_string, platform): if os_string in platform.operating_sys: return True return False + + def add_target_from_string(self, arch): + if arch.target is None: + return arch.platform.target('default_target') + else: + return arch.platform.target(arch.target) - def add_architecture_from_string(self, arch): - """ The user is able to provide a architecture string of the form - platform-os-target. This string will be parsed by this function - and turn the architecture string into an architecture tuple of - platform, operating system and target processor classes. - The platform-os-target triplet can be delimited by a '-'. If any - portion of the architecture triplet is empty, spack will supply - the default. If the entire architecture field is blank then - defaults will be provided based off of the platform. - This happens in the concretize_architecture method in concretize.py + def add_operating_system_from_string(self, arch): + if arch.platform_os is None: + return arch.platform.operating_system('default_os') + else: + return arch.platform.operating_system(arch.platform_os) - e.g - =linux-ubuntu10-x84_64 -> (linux, ubuntu10, x86_64) + #def add_architecture_from_string(self, arch): + # """ The user is able to provide a architecture string of the form + # platform-os-target. This string will be parsed by this function + # and turn the architecture string into an architecture tuple of + # platform, operating system and target processor classes. + # The platform-os-target triplet can be delimited by a '-'. If any + # portion of the architecture triplet is empty, spack will supply + # the default. If the entire architecture field is blank then + # defaults will be provided based off of the platform. + # This happens in the concretize_architecture method in concretize.py - =cray_xc-SuSE11-haswell -> (cray_xc, SuSE11, haswell) + # e.g + # =linux-ubuntu10-x84_64 -> (linux, ubuntu10, x86_64) - =bgq -> (bgq, - default_os, - default_target) + # =cray_xc-SuSE11-haswell -> (cray_xc, SuSE11, haswell) - =elcapitan -> (darwin, elcapitan, x86_64) + # =bgq -> (bgq, + # default_os, + # default_target) - =x86_64 -> (autodetected platform, - default_os, - x86_64) - """ - if arch is None: return - - platform_list = spack.architecture.all_platforms() - platform_names = {plat.__name__.lower():plat for plat in platform_list} - Arch = spack.architecture.Arch - arch_list = arch.split("-") - - # Create instance of current platform, gets overwritten if user - # provided a platform spec. - platform = spack.architecture.sys_type() - target = None - platform_os = None + # =elcapitan -> (darwin, elcapitan, x86_64) - for entry in arch_list: - if self._is_valid_platform(entry, platform_names): - # If entry is different from platform name then create it. - # else just keep the already instantiated platform class - if entry != platform.name: - platform = platform_dict[entry]() # Create instance of platform - elif self._is_valid_target(entry, platform): - target = platform.target(entry) - # check if os is valid by checking platform operating sys dict - elif self._is_valid_os(entry, platform): - platform_os = platform.operating_system(entry) - else: - # throw error since entry is unknown - raise UnknownArchitectureSpecError(entry) + # =x86_64 -> (autodetected platform, + # default_os, + # x86_64) + # """ + # if arch is None: return + # + # platform_list = spack.architecture.all_platforms() + # platform_names = {plat.__name__.lower():plat for plat in platform_list} + # Arch = spack.architecture.Arch + # arch_list = arch.split("-") + # + # # Create instance of current platform, gets overwritten if user + # # provided a platform spec. + # platform = spack.architecture.sys_type() + # target = None + # platform_os = None - if target is None: - target = platform.target('default') - if platform_os is None: - platform_os = platform.operating_system('default_os') + # for entry in arch_list: + # if self._is_valid_platform(entry, platform_names): + # # If entry is different from platform name then create it. + # # else just keep the already instantiated platform class + # if entry != platform.name: + # platform = platform_dict[entry]() # Create instance of platform + # elif self._is_valid_target(entry, platform): + # target = platform.target(entry) + # # check if os is valid by checking platform operating sys dict + # elif self._is_valid_os(entry, platform): + # platform_os = platform.operating_system(entry) + # else: + # # throw error since entry is unknown + # raise UnknownArchitectureSpecError(entry) - self.architecture = Arch(platform=platform, - platform_os=platform_os, - target=target) - + # if target is None: + # target = platform.target('default') + # if platform_os is None: + # platform_os = platform.operating_system('default_os') + + # self.architecture = Arch(platform=platform, + # platform_os=platform_os, + # target=target) + # def satisfies(self, other, deps=True, strict=False): """determine if this spec satisfies all constraints of another. From f27f2f8e491c7e3a433e070497d074f8376233a0 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 2 Mar 2016 10:27:12 -0800 Subject: [PATCH 168/481] Fixed some indentation errors --- lib/spack/spack/concretize.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 04f932aa5d..60efba59c2 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -225,7 +225,7 @@ def _concretize_operating_system(self, arch, platform): spec.add_operating_system_from_string(spec.architecture.platform_os) return True #changed if spec.root.architecture.platform_os: - if isinstance(spec.root.architecture.platform_os,spack.architecture.OperatingSystem): + if isinstance(spec.root.architecture.platform_os,spack.architecture.OperatingSystem): spec.architecture.platform_os = spec.root.architecture.platform_os else: spec.add_operating_system_from_string(spec.root.architecture.platform_os) @@ -281,13 +281,13 @@ def concretize_architecture(self, spec): """ platform = spack.architecture.sys_type() + if spec.architecture is None: # Create an empty tuple Arch = spack.architecture.Arch # Set the architecture to all defaults - spec.architecture = Arch(platform=platform, - None, - None) + spec.architecture = Arch(platform=platform, platform_os=None, + target=None) return True #If there is a target and it is a tuple and has both filled return #False From 45887dec8e703232a4d38bd241a20e36606a00cf Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 2 Mar 2016 10:56:46 -0800 Subject: [PATCH 169/481] partial work on bug hunting --- lib/spack/spack/architecture.py | 22 +++++++++++----------- lib/spack/spack/concretize.py | 32 +++++++++++++++----------------- lib/spack/spack/spec.py | 21 +++++++++++---------- 3 files changed, 37 insertions(+), 38 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index dac3a45b79..45a9dac8ba 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -203,8 +203,8 @@ class Arch(namedtuple("Arch", "platform platform_os target")): __slots__ = () def __str__(self): - return (self.platform.name +"-"+ - self.platform_os.name + "-" + self.target.name) + return (str(self.platform) +"-"+ + str(self.platform_os) + "-" + str(self.target) ) def _cmp_key(self): return (self.platform, self.platform_os, self.target) @@ -246,14 +246,14 @@ def to_dict(arch): return d -def _platform_from_dict(platform): - """Creates all the platform class module names into a dictionary of - name : key-value pairs. From there we can construct the - platform subclass - """ - platform_list = all_platforms() - platform_names = {plat.__name__.lower():plat for plat in platform_list} - return platform_names[platform['name']]() +#def _platform_from_dict(platform): +# """Creates all the platform class module names into a dictionary of +# name : key-value pairs. From there we can construct the +# platform subclass +# """ +# platform_list = all_platforms() +# platform_names = {plat.__name__.lower():plat for plat in platform_list} +# return platform_names[platform['name']]() def _target_from_dict(target_dict): @@ -288,7 +288,7 @@ def arch_from_dict(d): platform_os_dict = d['platform_os'] target_dict = d['target'] - platform = _platform_from_dict(platform_dict) + platform = d['platform'] platform_os = _operating_system_from_dict(platform_os_dict, platform) target = _target_from_dict(target_dict) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 60efba59c2..afb12838e8 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -217,14 +217,14 @@ def _concretize_platform(self, arch, platform): arch.platform = platform return True - def _concretize_operating_system(self, arch, platform): + def _concretize_operating_system(self, spec, platform): if spec.architecture.platform_os is not None: if isinstance(spec.architecture.platform_os,spack.architecture.OperatingSystem): return False else: spec.add_operating_system_from_string(spec.architecture.platform_os) return True #changed - if spec.root.architecture.platform_os: + if spec.root.architecture and spec.root.architecture.platform_os: if isinstance(spec.root.architecture.platform_os,spack.architecture.OperatingSystem): spec.architecture.platform_os = spec.root.architecture.platform_os else: @@ -243,22 +243,22 @@ def _concretize_operating_system(self, arch, platform): # return True - def _concretize_target(self, arch, platform): - if spec.target is not None: - if isinstance(spec.target,spack.architecture.Target): + def _concretize_target(self, spec, platform): + if spec.architecture.target is not None: + if isinstance(spec.architecture.target,spack.architecture.Target): return False else: - spec.add_target_from_string(spec.target) + spec.add_target_from_string(spec.architecture.target) return True #changed - if spec.root.target: - if isinstance(spec.root.target,spack.architecture.Target): - spec.target = spec.root.target + if spec.root.architecture and spec.root.architecture.target: + if isinstance(spec.root.architecture.target,spack.architecture.Target): + spec.architecture.target = spec.root.architecture.target else: - spec.add_target_from_string(spec.root.target) + spec.add_target_from_string(spec.root.architecture.target) else: platform = spack.architecture.sys_type() - spec.target = platform.target('default') + spec.architecture.target = platform.target('default') return True #changed @@ -283,22 +283,20 @@ def concretize_architecture(self, spec): platform = spack.architecture.sys_type() if spec.architecture is None: - # Create an empty tuple - Arch = spack.architecture.Arch # Set the architecture to all defaults - spec.architecture = Arch(platform=platform, platform_os=None, + spec.architecture = spack.architecture.Arch(platform=platform, platform_os=None, target=None) return True #If there is a target and it is a tuple and has both filled return #False # if isinstance(spec.architecture, basestring): # spec.split_architecture_string(spec.architecture) - + print spec.architecture ret = any(( self._concretize_platform(spec.architecture, platform), - self._concretize_operating_system(spec.architecture, platform), - self._concretize_target(spec.architecture, platform))) + self._concretize_operating_system(spec, platform), + self._concretize_target(spec, platform))) # Does not look pretty at all!!! diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 06accc514b..5cb94907d3 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -462,11 +462,12 @@ def _set_architecture(self, architecture): if self.architecture: raise DuplicateArchitectureError( "Spec for '%s' cannot have two architectures." % self.name) platform = spack.architecture.sys_type() + print architecture if '-' in architecture: os, target = architecture.split('-') else: - os = architecture - target = None + os = None + target = architecture self.architecture = spack.architecture.Arch(platform, os, target) @@ -1257,17 +1258,17 @@ def _is_valid_os(self, os_string, platform): return True return False - def add_target_from_string(self, arch): - if arch.target is None: - return arch.platform.target('default_target') + def add_target_from_string(self, target): + if target is None: + self.architecture.target = self.architecture.platform.target('default_target') else: - return arch.platform.target(arch.target) + self.architecture.target = self.architecture.platform.target(target) - def add_operating_system_from_string(self, arch): - if arch.platform_os is None: - return arch.platform.operating_system('default_os') + def add_operating_system_from_string(self, os): + if os is None: + self.architecture.platform_os = self.architecture.platform.operating_system('default_os') else: - return arch.platform.operating_system(arch.platform_os) + self.architecture.platform_os = self.architecture.platform.operating_system(os) #def add_architecture_from_string(self, arch): # """ The user is able to provide a architecture string of the form From 81e236b2de157405a2bc65e71a9a7fc7d1a13d7d Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 2 Mar 2016 11:04:55 -0800 Subject: [PATCH 170/481] Changed Arch to class instead of namedtuple. Now platform is automatically set upon creation of class --- lib/spack/spack/architecture.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 45a9dac8ba..94bd59a9d5 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -196,11 +196,13 @@ def _cmp_key(self): #NOTE: Key error caused because Architecture has no comparison method @key_ordering -class Arch(namedtuple("Arch", "platform platform_os target")): - """ namedtuple for Architecture. Will have it's own __str__ method - to make printing out the tuple easier and also won't make directory - paths look odd """ - __slots__ = () +class Arch(object): + "Architecture is now a class to help with setting attributes" + + def __init__(self, platform_os=None, target=None): + self.platform = sys_type() + self.platform_os = platform_os + self.target = target def __str__(self): return (str(self.platform) +"-"+ From 86e90bba8741eec38eae3898576da272ed41d9c2 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 2 Mar 2016 11:05:05 -0800 Subject: [PATCH 171/481] another partial, narrowing in on proper arch --- lib/spack/spack/concretize.py | 14 +------ lib/spack/spack/spec.py | 74 +++-------------------------------- 2 files changed, 6 insertions(+), 82 deletions(-) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index afb12838e8..add3b8435d 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -210,13 +210,6 @@ def concretize_version(self, spec): return True # Things changed - def _concretize_platform(self, arch, platform): - if issubclass(arch.platform.__class__, spack.architecture.Platform): - return False - else: - arch.platform = platform - return True - def _concretize_operating_system(self, spec, platform): if spec.architecture.platform_os is not None: if isinstance(spec.architecture.platform_os,spack.architecture.OperatingSystem): @@ -279,13 +272,9 @@ def concretize_architecture(self, spec): DAG has an architecture, then use the root otherwise use the defaults on the platform. """ - - platform = spack.architecture.sys_type() - if spec.architecture is None: # Set the architecture to all defaults - spec.architecture = spack.architecture.Arch(platform=platform, platform_os=None, - target=None) + spec.architecture = spack.architecture.Arch() return True #If there is a target and it is a tuple and has both filled return #False @@ -294,7 +283,6 @@ def concretize_architecture(self, spec): print spec.architecture ret = any(( - self._concretize_platform(spec.architecture, platform), self._concretize_operating_system(spec, platform), self._concretize_target(spec, platform))) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 5cb94907d3..169c06242e 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -468,7 +468,7 @@ def _set_architecture(self, architecture): else: os = None target = architecture - self.architecture = spack.architecture.Arch(platform, os, target) + self.architecture = spack.architecture.Arch(os, target) def _add_dependency(self, spec): @@ -1250,13 +1250,10 @@ def _is_valid_platform(self, platform, platform_list): return False def _is_valid_target(self, target, platform): - if target in platform.targets: - return True - return False + return target in platform.targets + def _is_valid_os(self, os_string, platform): - if os_string in platform.operating_sys: - return True - return False + return os_string in platform.operating_sys def add_target_from_string(self, target): if target is None: @@ -1270,69 +1267,7 @@ def add_operating_system_from_string(self, os): else: self.architecture.platform_os = self.architecture.platform.operating_system(os) - #def add_architecture_from_string(self, arch): - # """ The user is able to provide a architecture string of the form - # platform-os-target. This string will be parsed by this function - # and turn the architecture string into an architecture tuple of - # platform, operating system and target processor classes. - # The platform-os-target triplet can be delimited by a '-'. If any - # portion of the architecture triplet is empty, spack will supply - # the default. If the entire architecture field is blank then - # defaults will be provided based off of the platform. - # This happens in the concretize_architecture method in concretize.py - # e.g - # =linux-ubuntu10-x84_64 -> (linux, ubuntu10, x86_64) - - # =cray_xc-SuSE11-haswell -> (cray_xc, SuSE11, haswell) - - # =bgq -> (bgq, - # default_os, - # default_target) - - # =elcapitan -> (darwin, elcapitan, x86_64) - - # =x86_64 -> (autodetected platform, - # default_os, - # x86_64) - # """ - # if arch is None: return - # - # platform_list = spack.architecture.all_platforms() - # platform_names = {plat.__name__.lower():plat for plat in platform_list} - # Arch = spack.architecture.Arch - # arch_list = arch.split("-") - # - # # Create instance of current platform, gets overwritten if user - # # provided a platform spec. - # platform = spack.architecture.sys_type() - # target = None - # platform_os = None - - # for entry in arch_list: - # if self._is_valid_platform(entry, platform_names): - # # If entry is different from platform name then create it. - # # else just keep the already instantiated platform class - # if entry != platform.name: - # platform = platform_dict[entry]() # Create instance of platform - # elif self._is_valid_target(entry, platform): - # target = platform.target(entry) - # # check if os is valid by checking platform operating sys dict - # elif self._is_valid_os(entry, platform): - # platform_os = platform.operating_system(entry) - # else: - # # throw error since entry is unknown - # raise UnknownArchitectureSpecError(entry) - - # if target is None: - # target = platform.target('default') - # if platform_os is None: - # platform_os = platform.operating_system('default_os') - - # self.architecture = Arch(platform=platform, - # platform_os=platform_os, - # target=target) - # def satisfies(self, other, deps=True, strict=False): """determine if this spec satisfies all constraints of another. @@ -1378,6 +1313,7 @@ def satisfies(self, other, deps=True, strict=False): if not self.variants.satisfies(other.variants, strict=strict): return False + # Target satisfaction is currently just class equality. # If not strict, None means unconstrained. if isinstance(self.architecture, basestring): From 25f20b19673d71fb11ada83567c9e89de5555839 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 2 Mar 2016 11:40:21 -0800 Subject: [PATCH 172/481] bug hunting --- lib/spack/spack/architecture.py | 5 ++++- lib/spack/spack/concretize.py | 15 ++++++--------- lib/spack/spack/spec.py | 1 - var/spack/mock_packages/multimethod/package.py | 7 ++++--- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 94bd59a9d5..c9e4eb132d 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -209,7 +209,10 @@ def __str__(self): str(self.platform_os) + "-" + str(self.target) ) def _cmp_key(self): - return (self.platform, self.platform_os, self.target) + platform = self.platform.name + os = self.platform_os.name if isinstance(self.platform_os, OperatingSystem) else self.platform_os + target = self.target.name if isinstance(self.target, Target) else self.target + return (platform, os, target) def _helper_to_dict(arch_field_dict, arch_field_name, *args): diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index add3b8435d..37204e32a4 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -210,7 +210,7 @@ def concretize_version(self, spec): return True # Things changed - def _concretize_operating_system(self, spec, platform): + def _concretize_operating_system(self, spec): if spec.architecture.platform_os is not None: if isinstance(spec.architecture.platform_os,spack.architecture.OperatingSystem): return False @@ -223,8 +223,7 @@ def _concretize_operating_system(self, spec, platform): else: spec.add_operating_system_from_string(spec.root.architecture.platform_os) else: - platform = spack.architecture.sys_type() - spec.architecture.platform_os = platform.operating_system('default_os') + spec.architecture.platform_os = spec.architecture.platform.operating_system('default_os') return True #changed @@ -236,7 +235,7 @@ def _concretize_operating_system(self, spec, platform): # return True - def _concretize_target(self, spec, platform): + def _concretize_target(self, spec): if spec.architecture.target is not None: if isinstance(spec.architecture.target,spack.architecture.Target): return False @@ -250,8 +249,7 @@ def _concretize_target(self, spec, platform): else: spec.add_target_from_string(spec.root.architecture.target) else: - platform = spack.architecture.sys_type() - spec.architecture.target = platform.target('default') + spec.architecture.target = spec.architecture.platform.target('default') return True #changed @@ -280,11 +278,10 @@ def concretize_architecture(self, spec): #False # if isinstance(spec.architecture, basestring): # spec.split_architecture_string(spec.architecture) - print spec.architecture ret = any(( - self._concretize_operating_system(spec, platform), - self._concretize_target(spec, platform))) + self._concretize_operating_system(spec), + self._concretize_target(spec))) # Does not look pretty at all!!! diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 169c06242e..fdc5e3bc96 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -462,7 +462,6 @@ def _set_architecture(self, architecture): if self.architecture: raise DuplicateArchitectureError( "Spec for '%s' cannot have two architectures." % self.name) platform = spack.architecture.sys_type() - print architecture if '-' in architecture: os, target = architecture.split('-') else: diff --git a/var/spack/mock_packages/multimethod/package.py b/var/spack/mock_packages/multimethod/package.py index b926d42d7b..03da9ef69d 100644 --- a/var/spack/mock_packages/multimethod/package.py +++ b/var/spack/mock_packages/multimethod/package.py @@ -120,9 +120,10 @@ def has_a_default(self): for target in targets: @when('='+target.name) def different_by_target(self): - return self.spec.architecture.target.name - - + if isinstance(self.spec.architecture.target,basestring): + return self.spec.architecture.target + else: + return self.spec.architecture.target.name # # Make sure we can switch methods on different dependencies # From 47c8e1366f9659850704b03fd39dbfa3dd54fbbb Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 2 Mar 2016 12:08:17 -0800 Subject: [PATCH 173/481] Changed to_dictionary and from_dictionary methods --- lib/spack/spack/architecture.py | 112 +++++++++++++++++++------------- 1 file changed, 67 insertions(+), 45 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index c9e4eb132d..e432374781 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -58,14 +58,11 @@ class Target(object): Targets will have compiler finding strategies """ - def __init__(self, name, compiler_strategy, module_name=None): + def __init__(self, name, module_name=None): self.name = name # case of cray "ivybridge" but if it's x86_64 - self.compiler_strategy = compiler_strategy self.module_name = module_name # craype-ivybridge # Sets only the platform name to avoid recursiveness - def set_platform(self, platform): - self.platform_name = platform.name def _cmp_key(self): return (self.name, self.module_name) @@ -74,10 +71,14 @@ def __repr__(self): return self.__str__() def __str__(self): - if self.platform_name: - return self.platform_name + '-' + self.name return self.name + def to_dict(self): + d = {} + d['name'] = self.name + d['module_name'] = self.module_name + + return d @key_ordering class Platform(object): @@ -109,7 +110,6 @@ def add_target(self, name, target): raise ValueError( "%s is a spack reserved alias " "and cannot be the name of a target" % name) - target.set_platform(self) self.targets[name] = target def target(self, name): @@ -193,6 +193,14 @@ def compiler_strategy(self): def _cmp_key(self): return (self.name, self.version, self.compiler_strategy) + + def to_dict(self): + d = {} + d['name'] = self.name + d['version'] = self.version + d['compiler_strategy'] = self.compiler_strategy + + return d #NOTE: Key error caused because Architecture has no comparison method @key_ordering @@ -213,43 +221,55 @@ def _cmp_key(self): os = self.platform_os.name if isinstance(self.platform_os, OperatingSystem) else self.platform_os target = self.target.name if isinstance(self.target, Target) else self.target return (platform, os, target) - - -def _helper_to_dict(arch_field_dict, arch_field_name, *args): - """ General method to turn each class in architecture into a - dictionary. Takes as argument the class dictionary, the field name - (platform, platform_os, target) and then any attribute args - """ - d = {} - d[arch_field_name] = {} - for items in args: - d[arch_field_name][items] = arch_field_dict[items] - return d - -def to_dict(arch): - """ Convert the Arch tuple into a dictionary for yaml dumping. This - uses the _helper_to_dict method to create the dictionary from the - provided architecture field. Can assign the architecture - field name (either platform, platform_os or target) and any - attributes that make up that architecture field, - """ - d = {} - platform = arch.platform.__dict__ - platform_os = arch.platform_os.__dict__ - target = arch.target.__dict__ + def to_dict(self): + d = {} + platform = self.platform + platform_os = self.platform_os + target = self.target - platform_dict = _helper_to_dict(platform,'platform','name') - os_dict = _helper_to_dict(platform_os, 'platform_os', 'name','version', - 'compiler_strategy') - target_dict = _helper_to_dict(target,'target', 'name', - 'module_name','platform_name') + d['platform'] = self.platform.name + d['platform_os'] = self.platform_os.to_dict() + d['target'] = self.target.to_dict() + + return d - d.update(platform_dict) - d.update(os_dict) - d.update(target_dict) +#def _helper_to_dict(arch_field_dict, arch_field_name, *args): +# """ General method to turn each class in architecture into a +# dictionary. Takes as argument the class dictionary, the field name +# (platform, platform_os, target) and then any attribute args +# """ +# d = {} +# d[arch_field_name] = {} +# for items in args: +# d[arch_field_name][items] = arch_field_dict[items] +# return d +# - return d +#def to_dict(arch): +# """ Convert the Arch tuple into a dictionary for yaml dumping. This +# uses the _helper_to_dict method to create the dictionary from the +# provided architecture field. Can assign the architecture +# field name (either platform, platform_os or target) and any +# attributes that make up that architecture field, +# """ +# d = {} +# +# platform = arch.platform.__dict__ +# platform_os = arch.platform_os.__dict__ +# target = arch.target.__dict__ +# +# platform_dict = _helper_to_dict(platform,'platform','name') +# os_dict = _helper_to_dict(platform_os, 'platform_os', 'name','version', +# 'compiler_strategy') +# target_dict = _helper_to_dict(target,'target', 'name', +# 'module_name','platform_name') +# +# d.update(platform_dict) +# d.update(os_dict) +# d.update(target_dict) +# +# return d #def _platform_from_dict(platform): # """Creates all the platform class module names into a dictionary of @@ -287,17 +307,19 @@ def arch_from_dict(d): helper methods to recreate the arch tuple from the dictionary read from a yaml file """ + arch = Arch() + if d is None: return None - platform_dict = d['platform'] - platform_os_dict = d['platform_os'] + os_dict = d['platform_os'] target_dict = d['target'] - platform = d['platform'] - platform_os = _operating_system_from_dict(platform_os_dict, platform) target = _target_from_dict(target_dict) + platform_os = _operating_system_from_dict(os_dict, arch.platform) + arch.target =target + arch.platform_os = platform_os - return Arch(platform, platform_os, target) + return arch @memoized def all_platforms(): From 90e90f61c161facfb87dd2dc6389f80d56eba2d7 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 2 Mar 2016 12:08:43 -0800 Subject: [PATCH 174/481] Got rid of the compiler strategy from target --- lib/spack/spack/platforms/bgq.py | 4 ++-- lib/spack/spack/platforms/cray_xc.py | 6 +++--- lib/spack/spack/platforms/darwin.py | 2 +- lib/spack/spack/platforms/linux.py | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/spack/spack/platforms/bgq.py b/lib/spack/spack/platforms/bgq.py index b550f1d5cb..e0eb76f336 100644 --- a/lib/spack/spack/platforms/bgq.py +++ b/lib/spack/spack/platforms/bgq.py @@ -9,8 +9,8 @@ class Bgq(Platform): def __init__(self): super(Bgq, self).__init__('bgq') - self.add_target(self.front_end, Target(self.front_end, 'PATH')) - self.add_target(self.back_end, Target(self.back_end, 'PATH')) + self.add_target(self.front_end, Target(self.front_end)) + self.add_target(self.back_end, Target(self.back_end,)) @classmethod def detect(self): diff --git a/lib/spack/spack/platforms/cray_xc.py b/lib/spack/spack/platforms/cray_xc.py index b71481e46e..4843a47c62 100644 --- a/lib/spack/spack/platforms/cray_xc.py +++ b/lib/spack/spack/platforms/cray_xc.py @@ -31,11 +31,11 @@ def __init__(self): # Could switch to use modules and fe targets for front end # Currently using compilers by path for front end. - self.add_target('sandybridge', Target('sandybridge', 'PATH')) + self.add_target('sandybridge', Target('sandybridge')) self.add_target('ivybridge', - Target('ivybridge', 'MODULES', 'craype-ivybridge')) + Target('ivybridge', 'craype-ivybridge')) self.add_target('haswell', - Target('haswell', 'MODULES', 'craype-haswell')) + Target('haswell','craype-haswell')) self.add_operating_system('SuSE', LinuxDistro()) self.add_operating_system('CNL', Cnl()) diff --git a/lib/spack/spack/platforms/darwin.py b/lib/spack/spack/platforms/darwin.py index c48f7d17d2..4c3d38851f 100644 --- a/lib/spack/spack/platforms/darwin.py +++ b/lib/spack/spack/platforms/darwin.py @@ -10,7 +10,7 @@ class Darwin(Platform): def __init__(self): super(Darwin, self).__init__('darwin') - self.add_target(self.default, Target(self.default, 'PATH')) + self.add_target(self.default, Target(self.default)) mac_os = MacOsx() self.default_os = mac_os.name self.add_operating_system(mac_os.name, mac_os) diff --git a/lib/spack/spack/platforms/linux.py b/lib/spack/spack/platforms/linux.py index d9e33511a1..3243a1dcdf 100644 --- a/lib/spack/spack/platforms/linux.py +++ b/lib/spack/spack/platforms/linux.py @@ -10,7 +10,7 @@ class Linux(Platform): def __init__(self): super(Linux, self).__init__('linux') - self.add_target(self.default, Target(self.default, 'PATH')) + self.add_target(self.default, Target(self.default)) linux_dist = LinuxDistro() self.default_os = linux_dist.name self.add_operating_system(linux_dist.name, linux_dist) From e46bac19fa9baafe1e4b32ff940a883218f8b019 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 2 Mar 2016 12:09:02 -0800 Subject: [PATCH 175/481] Changed architecture to dict --- lib/spack/spack/spec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index fdc5e3bc96..6cca61ca4e 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -671,7 +671,7 @@ def to_node_dict(self): if self.architecture: # TODO: Fix the target.to_dict to account for the tuple # Want it to be a dict of dicts - d['architecture'] = spack.architecture.to_dict(self.architecture) + d['architecture'] = self.architecture.to_dict() else: d['architecture'] = None From 975cba295bc3141a9888292623702f1ee45f3d3a Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 2 Mar 2016 14:16:57 -0800 Subject: [PATCH 176/481] Added 'better' tests --- lib/spack/spack/test/architecture.py | 58 +++++++++++----------------- 1 file changed, 23 insertions(+), 35 deletions(-) diff --git a/lib/spack/spack/test/architecture.py b/lib/spack/spack/test/architecture.py index 5bc78d1b65..a390a3fbaa 100644 --- a/lib/spack/spack/test/architecture.py +++ b/lib/spack/spack/test/architecture.py @@ -14,43 +14,31 @@ class ArchitectureTest(unittest.TestCase): - def setUp(self): - zlib = spack.spec.Spec("zlib") - zlib.concretize() - self.architecture = zlib.architecture - self.platform = sys_type() - self.platform_os = self.platform.operating_system('default_os') - self.target = self.platform.target('default') - - #def test_to_dict_function_with_target(self): - # d = spack.architecture.to_dict(self.architecture) - # print d['target'] - # self.assertEquals(d['target'], {'name': self.target.name, - # 'module_name' : self.target.module_name, - # 'platform_name' : self.target.platform_name, - # 'compiler_strategy': 'MODULES' - # }) - def test_to_dict_function_with_architecture(self): - d = spack.architecture.to_dict(self.architecture) - self.assertEquals(d, {'architecture': - {'platform' : {'name': 'crayxc'}, - 'platform_os': { - 'compiler_strategy': 'MODULES', - 'name':'CNL', - 'version':'10'}, - 'target' : {'platform_name' :'crayxc', - 'module_name': 'craype-haswell', - 'name':'haswell'}}}) + arch = Arch() + arch.platform_os = arch.platform.operating_system('default_os') + arch.target = arch.platform.target('default') - #def test_to_dict_function_with_operating_system(self): - # d = spack.architecture.to_dict(self.architecture) - # self.assertEquals(d['platform_os'], {'name': self.platform_os.name, - # 'version': self.platform_os.version, - # 'compiler_strategy': self.platform_os.compiler_strategy}) - - def test_architecture_from_dict(self): - pass + d = arch.to_dict() + self.assertEqual(d, {'platform' : 'crayxc', + 'platform_os' : {'name': 'CNL', + 'compiler_strategy' : 'MODULES', + 'version':'10'}, + 'target' : {'name': 'haswell', + 'module_name': 'craype-haswell'}}) + + def test_from_dict_function_with_architecture(self): + d = {'platform':'crayxc', + 'platform_os' : {'name' : 'CNL', 'compiler_strategy': 'MODULES', + 'version': '10'}, + 'target' : {'name':'haswell', 'module_name': 'craype-haswell'}} + + arch = spack.architecture.arch_from_dict(d) + self.assertIsInstance(arch, Arch) + self.assertIsInstance(arch.platform, Platform) + self.assertIsInstance(arch.platform_os, OperatingSystem) + self.assertIsInstance(arch.target, Target) + def test_platform_class_and_compiler_strategies(self): a = CrayXc() From 676591ffc0d62a4d3b20653a2faa788c06d96d77 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 2 Mar 2016 14:17:46 -0800 Subject: [PATCH 177/481] fixed a bug and a test bug --- lib/spack/spack/spec.py | 2 +- .../site_spackconfig/compilers.yaml | 23 +++++++++++++++---- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 6cca61ca4e..6d2138d4c8 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1307,7 +1307,7 @@ def satisfies(self, other, deps=True, strict=False): if not self.compiler.satisfies(other.compiler, strict=strict): return False elif strict and (other.compiler and not self.compiler): - return True + return False if not self.variants.satisfies(other.variants, strict=strict): return False diff --git a/var/spack/mock_configs/site_spackconfig/compilers.yaml b/var/spack/mock_configs/site_spackconfig/compilers.yaml index fcbf7a53f1..72fa252881 100644 --- a/var/spack/mock_configs/site_spackconfig/compilers.yaml +++ b/var/spack/mock_configs/site_spackconfig/compilers.yaml @@ -13,10 +13,23 @@ compilers: fc: /path/to/gfortran modules: None gcc@5.2.0: - cc: cc - cxx: CC - f77: ftn - fc: ftn - modules: + cc: cc + cxx: CC + f77: ftn + fc: ftn + modules: - PrgEnv-gnu - gcc/5.2.0 + intel@15.0.1: + cc: cc + ccx: CC + f77: ftn + fc: ftn + modules: + - PrgEnv-intel + - intel/15.0.1 + intel@15.1.2: + cc: /path/to/icc + cxx: /path/to/ic++ + f77: /path/to/ifort + fc: /path/to/ifort From 6e7b00a0f6229a2c1df7a5b2f7244bdf5a793d65 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 2 Mar 2016 15:54:23 -0800 Subject: [PATCH 178/481] Made module handling much saner and safer --- lib/spack/spack/build_environment.py | 2 +- lib/spack/spack/compiler.py | 26 ++++++++++--------- lib/spack/spack/compilers/__init__.py | 21 ++++++++++----- lib/spack/spack/concretize.py | 9 ++++--- lib/spack/spack/test/architecture.py | 8 +++--- .../site_spackconfig/compilers.yaml | 5 ++++ 6 files changed, 43 insertions(+), 28 deletions(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 6e03394777..0cb4d77499 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -171,7 +171,7 @@ def set_compiler_environment_variables(pkg): os.environ['SPACK_COMPILER_SPEC'] = str(pkg.spec.compiler) - if compiler.modules: + if compiler.strategy == 'MODULES': for mod in compiler.modules: load_module(mod) diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index 46f2dfaec7..5d851f5a3d 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -106,26 +106,28 @@ class Compiler(object): PrgEnv_compiler = None - def __init__(self, cspec, paths, modules=None): + def __init__(self, cspec, strategy, paths, modules=None): def check(exe): if exe is None: return None _verify_executables(exe) return exe + self.strategy = strategy + self.cc = check(paths[0]) self.cxx = check(paths[1]) - self.f77 = check(paths[2]) - if len(paths) == 3: - self.fc = self.f77 - else: - self.fc = check(paths[3]) + if len(paths) > 2: + self.f77 = check(paths[2]) + if len(paths) == 3: + self.fc = self.f77 + else: + self.fc = check(paths[3]) self.spec = cspec self.modules = modules - @property def version(self): return self.spec.version @@ -267,7 +269,7 @@ def find_in_path(cls, *path): if newcount <= prevcount: continue - compilers[ver] = cls(spec, paths) + compilers[ver] = cls(spec, 'PATH', paths) return list(compilers.values()) @@ -287,7 +289,7 @@ def find_in_modules(cls): for name, version in matches: v = version - comp = cls(spack.spec.CompilerSpec(name + '@' + v), + comp = cls(spack.spec.CompilerSpec(name + '@' + v), 'MODULES', ['cc', 'CC', 'ftn'], [cls.PrgEnv, name +'/' + v]) compilers.append(comp) @@ -302,12 +304,12 @@ def __repr__(self): def __str__(self): """Return a string represntation of the compiler toolchain.""" - if self.modules: + if self.strategy is 'MODULES': return "%s(%s)" % ( - self.name, '\n '.join((str(s) for s in (self.cc, self.cxx, self.f77, self.fc, self.modules)))) + self.name, '\n '.join((str(s) for s in (self.strategy, self.cc, self.cxx, self.f77, self.fc, self.modules)))) else: return "%s(%s)" % ( - self.name, '\n '.join((str(s) for s in (self.cc, self.cxx, self.f77, self.fc)))) + self.name, '\n '.join((str(s) for s in (self.strategy, self.cc, self.cxx, self.f77, self.fc)))) class CompilerAccessError(spack.error.SpackError): diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index e38ef949d7..32885062fd 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -129,6 +129,11 @@ def add_compilers_to_config(scope, *compilers): for compiler in compilers: compiler_entry = {} + val = getattr(compiler, 'strategy') + if not val: + val = 'None' + compiler_entry[c] = val + for c in _required_instance_vars: val = getattr(compiler, c) if not val: @@ -190,6 +195,11 @@ def get_compiler(cspec): raise InvalidCompilerConfigurationError(cspec) cls = class_for_compiler_name(cspec.name) + + strategy = items['strategy'] + if not strategy: + raise InvalidCompilerConfigurationError(cspec) + compiler_paths = [] for c in _required_instance_vars: compiler_path = items[c] @@ -203,22 +213,19 @@ def get_compiler(cspec): items[m] = None mods = items[m] - return cls(cspec, compiler_paths, mods) + return cls(cspec, strategy, compiler_paths, mods) matches = find(compiler_spec) return [get_compiler(cspec) for cspec in matches] @_auto_compiler_spec -def compiler_for_spec(compiler_spec, target): +def compiler_for_spec(compiler_spec, operating_system): """Get the compiler that satisfies compiler_spec. compiler_spec must be concrete.""" assert(compiler_spec.concrete) - compilers = compilers_for_spec(compiler_spec) - if target.compiler_strategy == "PATH": - compilers = [c for c in compilers if c.modules is None] - elif target.compiler_strategy == "MODULES": - compilers = [c for c in compilers if c.modules is not None] + compilers = [c for c in compilers_for_spec(compiler_spec) + if c.strategy == operating_system.strategy] if len(compilers) < 1: raise NoCompilerForSpecError(compiler_spec, target) if len(compilers) > 1: diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 37204e32a4..7a87a824d5 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -354,10 +354,11 @@ def concretize_compiler(self, spec): # Should think whether this can be more efficient def _proper_compiler_style(cspec, architecture): compilers = spack.compilers.compilers_for_spec(cspec) - if architecture.platform_os.compiler_strategy == 'PATH': - filter(lambda c: not c.modules, compilers) - if architecture.platform_os.compiler_strategy == 'MODULES': - filter(lambda c: c.modules, compilers) + filter(lambda c: c.strategy == architecture.platform_os.compiler_strategy, compilers) +#if architecture.platform_os.compiler_strategy == 'PATH': + # filter(lambda c: not c.modules, compilers) + #if architecture.platform_os.compiler_strategy == 'MODULES': + # filter(lambda c: c.modules, compilers) return compilers diff --git a/lib/spack/spack/test/architecture.py b/lib/spack/spack/test/architecture.py index a390a3fbaa..75a67bf02f 100644 --- a/lib/spack/spack/test/architecture.py +++ b/lib/spack/spack/test/architecture.py @@ -34,10 +34,10 @@ def test_from_dict_function_with_architecture(self): 'target' : {'name':'haswell', 'module_name': 'craype-haswell'}} arch = spack.architecture.arch_from_dict(d) - self.assertIsInstance(arch, Arch) - self.assertIsInstance(arch.platform, Platform) - self.assertIsInstance(arch.platform_os, OperatingSystem) - self.assertIsInstance(arch.target, Target) + self.assertTrue( isinstance(arch, Arch) ) + self.assertTrue( isinstance(arch.platform, Platform) ) + self.assertTrue( isinstance(arch.platform_os, OperatingSystem) ) + self.assertTrue( isinstance(arch.target, Target) ) def test_platform_class_and_compiler_strategies(self): diff --git a/var/spack/mock_configs/site_spackconfig/compilers.yaml b/var/spack/mock_configs/site_spackconfig/compilers.yaml index 72fa252881..5f8b38007b 100644 --- a/var/spack/mock_configs/site_spackconfig/compilers.yaml +++ b/var/spack/mock_configs/site_spackconfig/compilers.yaml @@ -6,12 +6,14 @@ compilers: f77: None fc: None modules: None + strategy: PATH gcc@4.5.0: cc: /path/to/gcc cxx: /path/to/g++ f77: /path/to/gfortran fc: /path/to/gfortran modules: None + strategy: PATH gcc@5.2.0: cc: cc cxx: CC @@ -20,6 +22,7 @@ compilers: modules: - PrgEnv-gnu - gcc/5.2.0 + strategy: MODULES intel@15.0.1: cc: cc ccx: CC @@ -28,8 +31,10 @@ compilers: modules: - PrgEnv-intel - intel/15.0.1 + strategy: MODULES intel@15.1.2: cc: /path/to/icc cxx: /path/to/ic++ f77: /path/to/ifort fc: /path/to/ifort + strategy: PATH \ No newline at end of file From ffb9574312e8b6368c7696c2eadd17e46ec9a400 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 2 Mar 2016 15:59:23 -0800 Subject: [PATCH 179/481] fixed type/bug --- lib/spack/spack/compilers/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 32885062fd..b50ad02b07 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -225,7 +225,7 @@ def compiler_for_spec(compiler_spec, operating_system): be concrete.""" assert(compiler_spec.concrete) compilers = [c for c in compilers_for_spec(compiler_spec) - if c.strategy == operating_system.strategy] + if c.strategy == operating_system.compiler_strategy] if len(compilers) < 1: raise NoCompilerForSpecError(compiler_spec, target) if len(compilers) > 1: From 65d4169f003ce73c59f12e426e1e7fa4bf2062f3 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 2 Mar 2016 16:03:11 -0800 Subject: [PATCH 180/481] fixed type/bug --- lib/spack/spack/compilers/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index b50ad02b07..518eea55c4 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -132,7 +132,7 @@ def add_compilers_to_config(scope, *compilers): val = getattr(compiler, 'strategy') if not val: val = 'None' - compiler_entry[c] = val + compiler_entry['strategy'] = val for c in _required_instance_vars: val = getattr(compiler, c) From 16d8c25b231e795f8c4f3d11f937a4658596b238 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 2 Mar 2016 16:12:32 -0800 Subject: [PATCH 181/481] fixed floating 'target' --- lib/spack/spack/compilers/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 518eea55c4..d271b60261 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -227,7 +227,7 @@ def compiler_for_spec(compiler_spec, operating_system): compilers = [c for c in compilers_for_spec(compiler_spec) if c.strategy == operating_system.compiler_strategy] if len(compilers) < 1: - raise NoCompilerForSpecError(compiler_spec, target) + raise NoCompilerForSpecError(compiler_spec, operating_system) if len(compilers) > 1: raise CompilerSpecInsufficientlySpecificError(compiler_spec) return compilers[0] From 58efe1550d913e0bf31f2d5b6e279066cf5363d5 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 2 Mar 2016 16:14:42 -0800 Subject: [PATCH 182/481] Default PATH arg --- lib/spack/spack/architecture.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index e432374781..85ced6e942 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -170,7 +170,7 @@ class OperatingSystem(object): find compilers we call find_compilers method for each operating system """ - def __init__(self, name, version, compiler_strategy): + def __init__(self, name, version, compiler_strategy="PATH"): self.name = name self.version = version self.compiler_strategy = compiler_strategy @@ -180,19 +180,9 @@ def __str__(self): def __repr__(self): return self.__str__() - - - def compiler_strategy(self): - """ Operating Systems will now be in charge of the compiler finding - strategy. They will each have their own find_compilers method - which depending on their compiler strategy will find the compilers - using a specific method (i.e PATH vs MODULES). - """ - raise NotImplementedError() def _cmp_key(self): return (self.name, self.version, self.compiler_strategy) - def to_dict(self): d = {} From 2b487287419528142b16d9e4201a44a034c11d20 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 2 Mar 2016 16:15:34 -0800 Subject: [PATCH 183/481] Changed os subclasses CNL to change attribute to MODULES --- lib/spack/spack/operating_systems/cnl.py | 6 ------ lib/spack/spack/operating_systems/linux_distro.py | 11 +---------- lib/spack/spack/operating_systems/mac_osx.py | 13 +------------ 3 files changed, 2 insertions(+), 28 deletions(-) diff --git a/lib/spack/spack/operating_systems/cnl.py b/lib/spack/spack/operating_systems/cnl.py index 20cb4c215b..9a0bf6c194 100644 --- a/lib/spack/spack/operating_systems/cnl.py +++ b/lib/spack/spack/operating_systems/cnl.py @@ -11,9 +11,3 @@ def __init__(self): name = 'CNL' version = '10' super(Cnl, self).__init__(name, version, "MODULES") - - def compiler_strategy(self): - return self.compiler_strategy - - def find_compilers(self): - pass diff --git a/lib/spack/spack/operating_systems/linux_distro.py b/lib/spack/spack/operating_systems/linux_distro.py index c5547749d8..81c7a86430 100644 --- a/lib/spack/spack/operating_systems/linux_distro.py +++ b/lib/spack/spack/operating_systems/linux_distro.py @@ -12,13 +12,4 @@ def __init__(self): name = py_platform.dist()[0] version = py_platform.dist()[1] - super(LinuxDistro, self).__init__(name, version, "PATH") - - def compiler_strategy(self): - return self.compiler_strategy - - def find_compilers(self): - pass - - - + super(LinuxDistro, self).__init__(name, version) diff --git a/lib/spack/spack/operating_systems/mac_osx.py b/lib/spack/spack/operating_systems/mac_osx.py index 555c5cfb14..a9de03d2cc 100644 --- a/lib/spack/spack/operating_systems/mac_osx.py +++ b/lib/spack/spack/operating_systems/mac_osx.py @@ -26,15 +26,4 @@ def __init__(self): except KeyError: name = "mac_os" - super(MacOsx, self).__init__(name, mac_ver, "PATH") - - def compiler_strategy(self): - return self.compiler_strategy - - def find_compilers(self): - pass - - - - - + super(MacOsx, self).__init__(name, mac_ver) From 487b2495a1dbda730a6525f17f175f3eff1674c0 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 2 Mar 2016 16:33:10 -0800 Subject: [PATCH 184/481] made spack more conservative in compiler strategies attempted --- lib/spack/spack/compiler.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index 5d851f5a3d..999984740f 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -34,6 +34,7 @@ import spack.error import spack.spec +import spack.architecture from spack.util.multiproc import parmap from spack.util.executable import * from spack.util.environment import get_path @@ -221,7 +222,15 @@ def check(key): @classmethod def find(cls, *path): - return cls.find_in_path(*path) + cls.find_in_modules() + compilers = [] + platform = spack.architecture.sys_type() + strategies = [o.compiler_strategy for o in platform.operating_systems.values()] + if 'PATH' in strategies: + compilers.extend(cls.find_in_path(*path)) + if 'MODULES' in strategies: + compilers.extend(cls.find_in_modules()) + return compilers + @classmethod def find_in_path(cls, *path): From 4f9a309de850f339ed43cd788be45d208bebf6c6 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 2 Mar 2016 16:35:02 -0800 Subject: [PATCH 185/481] made spack more conservative in compiler strategies attempted --- lib/spack/spack/compiler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index 999984740f..c8a025f83c 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -224,7 +224,7 @@ def check(key): def find(cls, *path): compilers = [] platform = spack.architecture.sys_type() - strategies = [o.compiler_strategy for o in platform.operating_systems.values()] + strategies = [o.compiler_strategy for o in platform.operating_sys.values()] if 'PATH' in strategies: compilers.extend(cls.find_in_path(*path)) if 'MODULES' in strategies: From 688eca23ff4488a4cd90125b69d46cb69271a5cd Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 23 Mar 2016 09:54:57 -0700 Subject: [PATCH 186/481] cleanup --- lib/spack/spack/spec.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 4d2c61c0c4..d800a1667d 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -110,7 +110,7 @@ import spack.architecture import spack.parse import spack.error -import spack.compilers as compilers +#import spack.compilers as compilers from spack.version import * from spack.util.naming import mod_to_class @@ -1174,9 +1174,9 @@ def validate_names(self): spack.repo.get(spec.fullname) # validate compiler in addition to the package name. - if spec.compiler: - if not compilers.supported(spec.compiler): - raise UnsupportedCompilerError(spec.compiler.name) +# if spec.compiler: +# if not compilers.supported(spec.compiler): +# raise UnsupportedCompilerError(spec.compiler.name) # Ensure that variants all exist. for vname, variant in spec.variants.items(): From 315623d3612d64fd058324d4dbbe5264def759c0 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Thu, 24 Mar 2016 16:55:46 -0700 Subject: [PATCH 187/481] Fixed things from merge. --- lib/spack/spack/architecture.py | 7 ++++++ lib/spack/spack/build_environment.py | 10 ++++---- lib/spack/spack/cmd/compiler.py | 2 +- lib/spack/spack/compilers/__init__.py | 34 ++++++++++++++------------- lib/spack/spack/concretize.py | 6 ++--- lib/spack/spack/config.py | 8 +++++++ lib/spack/spack/spec.py | 1 + lib/spack/spack/test/multimethod.py | 4 ++-- 8 files changed, 45 insertions(+), 27 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index aef38b8a55..8d9686cf99 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -202,6 +202,13 @@ def __init__(self, platform_os=None, target=None): self.platform_os = platform_os self.target = target + @property + def concrete(self): + return all( (self.platform is not None, isinstance(self.platform, Platform), + self.platform_os is not None, isinstance(self.platform_os, OperatingSystem), + self.target is not None, isinstance(self.target, Target) ) ) + + def __str__(self): return (str(self.platform) +"-"+ str(self.platform_os) + "-" + str(self.target) ) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index a64ce8aeb4..8ed3c4e1a1 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -101,12 +101,12 @@ def load_module(mod): # We do this without checking that they are already installed # for ease of programming because unloading a module that is not # loaded does nothing. - text = modulecmd('show', mod, return_oe=True).split() + text = modulecmd('show', mod, output=str, error=str).split() for i, word in enumerate(text): if word == 'conflict': - exec(compile(modulecmd('unload', text[i+1], return_oe=True), '', 'exec')) + exec(compile(modulecmd('unload', text[i+1], output=str, error=str), '', 'exec')) # Load the module now that there are no conflicts - load = modulecmd('load', mod, return_oe=True) + load = modulecmd('load', mod, output=str, error=str) exec(compile(load, '', 'exec')) @@ -119,7 +119,7 @@ def get_path_from_module(mod): modulecmd.add_default_arg('python') # Read the module - text = modulecmd('show', mod, return_oe=True).split('\n') + text = modulecmd('show', mod, output=str, error=str).split('\n') # If it lists its package directory, return that for line in text: @@ -348,8 +348,8 @@ def parent_class_modules(cls): def setup_package(pkg): """Execute all environment setup routines.""" - set_compiler_environment_variables(pkg) set_build_environment_variables(pkg) + set_compiler_environment_variables(pkg) # If a user makes their own package repo, e.g. # spack.repos.mystuff.libelf.Libelf, and they inherit from diff --git a/lib/spack/spack/cmd/compiler.py b/lib/spack/spack/cmd/compiler.py index bed5db3d47..a7de61e1e5 100644 --- a/lib/spack/spack/cmd/compiler.py +++ b/lib/spack/spack/cmd/compiler.py @@ -23,7 +23,7 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import sys -import argparse +from external import argparse import llnl.util.tty as tty from llnl.util.tty.color import colorize diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 93c5172dde..55e2db700f 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -46,7 +46,7 @@ _imported_compilers_module = 'spack.compilers' _required_instance_vars = ['cc', 'cxx', 'f77', 'fc'] -_optional_instance_vars = ['modules'] +_optional_instance_vars = ['modules', 'strategy'] _default_order = [] # TODO: customize order in config file @@ -69,7 +69,7 @@ def _to_dict(compiler): return { str(compiler.spec) : dict( (attr, getattr(compiler, attr, None)) - for attr in _required_instance_vars) + for attr in _required_instance_vars + _optional_instance_vars) } @@ -77,20 +77,24 @@ def get_compiler_config(arch=None, scope=None): """Return the compiler configuration for the specified architecture. """ # Check whether we're on a front-end (native) architecture. - my_arch = spack.architecture.sys_type() + + my_arch = spack.architecture.Arch() + if isinstance(arch, basestring): + if arch == 'all': + my_arch.platform.name = 'all' + arch = my_arch if arch is None: arch = my_arch def init_compiler_config(): """Compiler search used when Spack has no compilers.""" - config[arch] = {} + config[arch.platform.name] = {} compilers = find_compilers(*get_path('PATH')) for compiler in compilers: - config[arch].update(_to_dict(compiler)) + config[arch.platform.name].update(_to_dict(compiler)) spack.config.update_config('compilers', config, scope=scope) config = spack.config.get_config('compilers', scope=scope) - # Update the configuration if there are currently no compilers # configured. Avoid updating automatically if there ARE site # compilers configured but no user ones. @@ -105,7 +109,7 @@ def init_compiler_config(): if not site_config: init_compiler_config() - return config[arch] if arch in config else {} + return config[arch.platform.name] if arch.platform.name in config else {} def add_compilers_to_config(compilers, arch=None, scope=None): @@ -117,15 +121,15 @@ def add_compilers_to_config(compilers, arch=None, scope=None): - scope: configuration scope to modify. """ if arch is None: - arch = spack.architecture.sys_type() + arch = spack.architecture.Arch() compiler_config = get_compiler_config(arch, scope) for compiler in compilers: compiler_config[str(compiler.spec)] = dict( (c, getattr(compiler, c, "None")) - for c in _required_instance_vars + ['strategy'] + _optional_instance_vars) + for c in _required_instance_vars + _optional_instance_vars) - update = { arch : compiler_config } + update = { arch.platform.name : compiler_config } spack.config.update_config('compilers', update, scope) @@ -139,7 +143,7 @@ def remove_compiler_from_config(compiler_spec, arch=None, scope=None): - scope: configuration scope to modify. """ if arch is None: - arch = spack.architecture.sys_type() + arch = spack.architecture.Arch() compiler_config = get_compiler_config(arch, scope) del compiler_config[str(compiler_spec)] @@ -154,7 +158,6 @@ def all_compilers_config(arch=None, scope=None): """ # Get compilers for this architecture. arch_config = get_compiler_config(arch, scope) - # Merge 'all' compilers with arch-specific ones. # Arch-specific compilers have higher precedence. merged_config = get_compiler_config('all', scope=scope) @@ -262,10 +265,9 @@ def get_compiler(cspec): else: compiler_paths.append(None) - for m in _optional_instance_vars: - if m not in items: - items[m] = None - mods = items[m] + if 'modules' not in items: + items['modules'] = None + mods = items['modules'] return cls(cspec, strategy, compiler_paths, mods) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 330ecdf509..c5dad93aef 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -79,7 +79,7 @@ def _valid_virtuals_and_externals(self, spec): externals = spec_externals(pkg) buildable = not is_spec_nobuild(pkg) if buildable: - result.append((pkg, None)) + result.append((pkg, None, None)) for ext in externals: if ext[0].satisfies(spec): result.append(ext) @@ -354,7 +354,7 @@ def concretize_compiler(self, spec): link to this one, to maximize compatibility. """ # Pass on concretizing the compiler if the target is not yet determined - if not spec.architecture.target: + if not spec.architecture.platform_os: #Although this usually means changed, this means awaiting other changes return True @@ -371,7 +371,7 @@ def _proper_compiler_style(cspec, architecture): return compilers - all_compilers = spack.compilers.all_compilers() + all_compilers = spack.compilers.all_compilers(spec.architecture) if (spec.compiler and spec.compiler.concrete and diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 182c174baa..c9a770a53f 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -168,6 +168,10 @@ {'type' : 'null' }]}, 'fc': { 'anyOf': [ {'type' : 'string' }, {'type' : 'null' }]}, + 'strategy': { 'anyOf': [ {'type' : 'string' }, + {'type' : 'null' }]}, + 'modules': { 'anyOf': [ {'type' : 'string' }, + {'type' : 'null' }]} },},},},},},},}, 'mirrors': { @@ -224,6 +228,10 @@ 'type': 'boolean', 'default': False, }, + 'module': { + 'anyOf' : [{'type': 'string'}, + {'type': 'null'}] + }, 'providers': { 'type': 'object', 'default': {}, diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index d800a1667d..fa2e9fd4c8 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -542,6 +542,7 @@ def concrete(self): and self.versions.concrete and self.variants.concrete and self.architecture + and self.architecture.concrete and self.compiler and self.compiler.concrete and self.dependencies.concrete) diff --git a/lib/spack/spack/test/multimethod.py b/lib/spack/spack/test/multimethod.py index 020d25015d..6a723d123e 100644 --- a/lib/spack/spack/test/multimethod.py +++ b/lib/spack/spack/test/multimethod.py @@ -92,10 +92,10 @@ def test_target_match(self): platform = spack.architecture.sys_type() targets = platform.targets.values() for target in targets[:-1]: - pkg = spack.db.get('multimethod='+target.name) + pkg = spack.repo.get('multimethod='+target.name) self.assertEqual(pkg.different_by_target(), target.name) - pkg = spack.db.get('multimethod='+targets[-1].name) + pkg = spack.repo.get('multimethod='+targets[-1].name) if len(targets) == 1: self.assertEqual(pkg.different_by_target(), targets[-1].name) else: From 618a9b2d7b1fbe297a2165181016fc45153cecf1 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 25 Mar 2016 15:41:22 -0700 Subject: [PATCH 188/481] Fixed bug in platform names that I introduced yesterday --- lib/spack/spack/compilers/__init__.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 55e2db700f..9601ae3e3a 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -28,6 +28,7 @@ import imp import os import platform +import copy from llnl.util.lang import memoized, list_modules from llnl.util.filesystem import join_path @@ -77,28 +78,27 @@ def get_compiler_config(arch=None, scope=None): """Return the compiler configuration for the specified architecture. """ # Check whether we're on a front-end (native) architecture. - my_arch = spack.architecture.Arch() - if isinstance(arch, basestring): - if arch == 'all': - my_arch.platform.name = 'all' - arch = my_arch if arch is None: arch = my_arch + if isinstance(arch, basestring) and arch == 'all': + name = 'all' + else: + name = arch.platform.name def init_compiler_config(): """Compiler search used when Spack has no compilers.""" - config[arch.platform.name] = {} + config[name] = {} compilers = find_compilers(*get_path('PATH')) for compiler in compilers: - config[arch.platform.name].update(_to_dict(compiler)) + config[name].update(_to_dict(compiler)) spack.config.update_config('compilers', config, scope=scope) config = spack.config.get_config('compilers', scope=scope) # Update the configuration if there are currently no compilers # configured. Avoid updating automatically if there ARE site # compilers configured but no user ones. - if arch == my_arch and arch not in config: + if (isinstance(arch, basestring) or arch == my_arch) and arch not in config: if scope is None: # We know no compilers were configured in any scope. init_compiler_config() @@ -109,7 +109,7 @@ def init_compiler_config(): if not site_config: init_compiler_config() - return config[arch.platform.name] if arch.platform.name in config else {} + return config[name] if name in config else {} def add_compilers_to_config(compilers, arch=None, scope=None): From 57ced372291bfa537e37d325d8e4333623eb306d Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 25 Mar 2016 17:03:49 -0700 Subject: [PATCH 189/481] Fixed a bug in the testing --- lib/spack/spack/test/mock_packages_test.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index 079cbcc136..8f04076f19 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -42,11 +42,15 @@ cxx: /path/to/clang++ f77: None fc: None + strategy: PATH + modules: None gcc@4.5.0: cc: /path/to/gcc cxx: /path/to/g++ f77: /path/to/gfortran fc: /path/to/gfortran + strategy: PATH + modules: None """ mock_packages_config = """\ From 7e9baf9e2533eb040aa142bd3bf6ffbd142516ac Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Tue, 29 Mar 2016 11:23:36 -0700 Subject: [PATCH 190/481] Made architecture test robust for more architectures --- lib/spack/spack/test/architecture.py | 28 +++++++++++----------------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/lib/spack/spack/test/architecture.py b/lib/spack/spack/test/architecture.py index 75a67bf02f..2274a9901d 100644 --- a/lib/spack/spack/test/architecture.py +++ b/lib/spack/spack/test/architecture.py @@ -13,32 +13,26 @@ from spack.platforms.darwin import Darwin class ArchitectureTest(unittest.TestCase): - - def test_to_dict_function_with_architecture(self): + + def test_dict_functions_for_architecture(self): arch = Arch() arch.platform_os = arch.platform.operating_system('default_os') arch.target = arch.platform.target('default') d = arch.to_dict() - self.assertEqual(d, {'platform' : 'crayxc', - 'platform_os' : {'name': 'CNL', - 'compiler_strategy' : 'MODULES', - 'version':'10'}, - 'target' : {'name': 'haswell', - 'module_name': 'craype-haswell'}}) - - def test_from_dict_function_with_architecture(self): - d = {'platform':'crayxc', - 'platform_os' : {'name' : 'CNL', 'compiler_strategy': 'MODULES', - 'version': '10'}, - 'target' : {'name':'haswell', 'module_name': 'craype-haswell'}} - - arch = spack.architecture.arch_from_dict(d) + + new_arch = spack.architecture.arch_from_dict(d) + self.assertEqual(arch, new_arch) + self.assertTrue( isinstance(arch, Arch) ) self.assertTrue( isinstance(arch.platform, Platform) ) self.assertTrue( isinstance(arch.platform_os, OperatingSystem) ) self.assertTrue( isinstance(arch.target, Target) ) - + self.assertTrue( isinstance(new_arch, Arch) ) + self.assertTrue( isinstance(new_arch.platform, Platform) ) + self.assertTrue( isinstance(new_arch.platform_os, OperatingSystem) ) + self.assertTrue( isinstance(new_arch.target, Target) ) + def test_platform_class_and_compiler_strategies(self): a = CrayXc() From 9c071ea40d6bacef2056184f7b256c2f1fb7cbd0 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 4 Apr 2016 13:44:24 -0700 Subject: [PATCH 191/481] Changed modulecmd parameters. return_oe is not one of the parameters in the mainline spack repo. Changed the args to the appropriate ones in the new spack --- lib/spack/spack/compiler.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index e3ea918aca..c285c8a29d 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -285,16 +285,15 @@ def find_in_path(cls, *path): @classmethod def find_in_modules(cls): compilers = [] - if cls.PrgEnv: if not cls.PrgEnv_compiler: tty.die('Must supply PrgEnv_compiler with PrgEnv') modulecmd = which('modulecmd') modulecmd.add_default_arg('python') - output = modulecmd('avail', cls.PrgEnv_compiler, return_oe=True) + + output = modulecmd('avail', cls.PrgEnv_compiler, output=str, error=str) matches = re.findall(r'(%s)/([\d\.]+[\d])' % cls.PrgEnv_compiler, output) - for name, version in matches: v = version comp = cls(spack.spec.CompilerSpec(name + '@' + v), 'MODULES', From 6d73c862096ecd6426d42dfe666c0908c2069a21 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 4 Apr 2016 13:45:15 -0700 Subject: [PATCH 192/481] Gives error on cori that [PrgEnv-xx, gcc/x.x.x] is not part of the schema. Changed it to oneOf : type : array to get rid of the error. --- lib/spack/spack/config.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index c9a770a53f..f978ed12b0 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -159,7 +159,9 @@ 'type': 'object', 'additionalProperties': False, 'required': ['cc', 'cxx', 'f77', 'fc'], - 'properties': { + 'properties': { + 'strategy': { 'anyOf': [ {'type' : 'string' }, + {'type' : 'null' }]}, 'cc': { 'anyOf': [ {'type' : 'string' }, {'type' : 'null' }]}, 'cxx': { 'anyOf': [ {'type' : 'string' }, @@ -168,10 +170,10 @@ {'type' : 'null' }]}, 'fc': { 'anyOf': [ {'type' : 'string' }, {'type' : 'null' }]}, - 'strategy': { 'anyOf': [ {'type' : 'string' }, - {'type' : 'null' }]}, - 'modules': { 'anyOf': [ {'type' : 'string' }, - {'type' : 'null' }]} + 'modules': { 'anyOf': [ {'type' : 'string'}, + {'type' : 'null' }, + {'type': 'array'}, + ]} },},},},},},},}, 'mirrors': { @@ -221,7 +223,7 @@ 'items' : { 'anyOf' : [ { 'type' : 'string' }, { 'type' : 'number'}]}}, #version strings 'compiler': { - 'type' : 'array', + 'typee' : 'array', 'default' : [], 'items' : { 'type' : 'string' } }, #compiler specs 'nobuild': { From 9bfc83ac5c3ae09cbcb281296d210d17984cd1db Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 4 Apr 2016 15:36:10 -0700 Subject: [PATCH 193/481] Changed so that py_platform.dist() grabs the major version number. --- lib/spack/spack/operating_systems/linux_distro.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/operating_systems/linux_distro.py b/lib/spack/spack/operating_systems/linux_distro.py index 81c7a86430..d0f24d842f 100644 --- a/lib/spack/spack/operating_systems/linux_distro.py +++ b/lib/spack/spack/operating_systems/linux_distro.py @@ -10,6 +10,6 @@ class LinuxDistro(OperatingSystem): """ def __init__(self): name = py_platform.dist()[0] - version = py_platform.dist()[1] + version = py_platform.dist()[1].split(".")[0] # Grabs major version from tuple super(LinuxDistro, self).__init__(name, version) From 3bf75fab3f37bc2edb70bd49060c3831777c21dc Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 6 Apr 2016 12:03:26 -0700 Subject: [PATCH 194/481] Cleaned up architecture. Changed operating_system from dictionary method --- lib/spack/spack/architecture.py | 68 +++++++-------------------------- 1 file changed, 14 insertions(+), 54 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 8d9686cf99..555fb14408 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -210,8 +210,13 @@ def concrete(self): def __str__(self): + if self.platform.name == 'darwin': + os_name = self.platform_os.name + else: + os_name = str(self.platform_os) + return (str(self.platform) +"-"+ - str(self.platform_os) + "-" + str(self.target) ) + os_name + "-" + str(self.target)) def _cmp_key(self): platform = self.platform.name @@ -231,52 +236,6 @@ def to_dict(self): return d -#def _helper_to_dict(arch_field_dict, arch_field_name, *args): -# """ General method to turn each class in architecture into a -# dictionary. Takes as argument the class dictionary, the field name -# (platform, platform_os, target) and then any attribute args -# """ -# d = {} -# d[arch_field_name] = {} -# for items in args: -# d[arch_field_name][items] = arch_field_dict[items] -# return d -# - -#def to_dict(arch): -# """ Convert the Arch tuple into a dictionary for yaml dumping. This -# uses the _helper_to_dict method to create the dictionary from the -# provided architecture field. Can assign the architecture -# field name (either platform, platform_os or target) and any -# attributes that make up that architecture field, -# """ -# d = {} -# -# platform = arch.platform.__dict__ -# platform_os = arch.platform_os.__dict__ -# target = arch.target.__dict__ -# -# platform_dict = _helper_to_dict(platform,'platform','name') -# os_dict = _helper_to_dict(platform_os, 'platform_os', 'name','version', -# 'compiler_strategy') -# target_dict = _helper_to_dict(target,'target', 'name', -# 'module_name','platform_name') -# -# d.update(platform_dict) -# d.update(os_dict) -# d.update(target_dict) -# -# return d - -#def _platform_from_dict(platform): -# """Creates all the platform class module names into a dictionary of -# name : key-value pairs. From there we can construct the -# platform subclass -# """ -# platform_list = all_platforms() -# platform_names = {plat.__name__.lower():plat for plat in platform_list} -# return platform_names[platform['name']]() - def _target_from_dict(target_dict): """ Creates new instance of target and assigns all the attributes of @@ -284,20 +243,21 @@ def _target_from_dict(target_dict): """ target = Target.__new__(Target) target.name = target_dict['name'] - #target.compiler_strategy = target_dict['compiler_strategy'] target.module_name = target_dict['module_name'] if 'platform_name' in target_dict: target.platform_name = target_dict['platform_name'] return target -def _operating_system_from_dict(os_dict, platform_class): +def _operating_system_from_dict(os_dict): """ uses platform's operating system method to grab the constructed operating systems that are valid on the platform. """ # NOTE: Might need a better way to create operating system objects - name = os_dict['name'] - return platform_class.operating_system(name) - + operating_system = OperatingSystem.__new__(OperatingSystem) + operating_system.name = os_dict['name'] + operating_system.version = os_dict['version'] + operating_system.compiler_strategy = os_dict['compiler_strategy'] + return operating_system def arch_from_dict(d): """ Uses _platform_from_dict, _operating_system_from_dict, _target_from_dict @@ -312,8 +272,8 @@ def arch_from_dict(d): target_dict = d['target'] target = _target_from_dict(target_dict) - platform_os = _operating_system_from_dict(os_dict, arch.platform) - arch.target =target + platform_os = _operating_system_from_dict(os_dict) + arch.target = target arch.platform_os = platform_os return arch From 749508af010ee90009a61bb42a8f8214a6fabd21 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 6 Apr 2016 12:05:29 -0700 Subject: [PATCH 195/481] Found loops calling all_compilers that were always reading in the config file which lead to a slowdown, the fix is to cache the config file so we don't waste time re-reading the config file in again. --- lib/spack/spack/compilers/__init__.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 9601ae3e3a..caf438e232 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -151,20 +151,25 @@ def remove_compiler_from_config(compiler_spec, arch=None, scope=None): spack.config.update_config('compilers', update, scope) +_cache_config_file = {} def all_compilers_config(arch=None, scope=None): """Return a set of specs for all the compiler versions currently available to build with. These are instances of CompilerSpec. """ # Get compilers for this architecture. - arch_config = get_compiler_config(arch, scope) - # Merge 'all' compilers with arch-specific ones. - # Arch-specific compilers have higher precedence. - merged_config = get_compiler_config('all', scope=scope) - merged_config = spack.config._merge_yaml(merged_config, arch_config) + global _cache_config_file #Create a cache of the config file so we don't load all the time. - return merged_config + if not _cache_config_file: + arch_config = get_compiler_config(arch, scope) + # Merge 'all' compilers with arch-specific ones. + # Arch-specific compilers have higher precedence. + _cache_config_file = get_compiler_config('all', scope=scope) + _cache_config_file = spack.config._merge_yaml(_cache_config_file, arch_config) + return _cache_config_file + else: + return _cache_config_file def all_compilers(arch=None, scope=None): # Return compiler specs from the merged config. From 918ad28a936abf6112ba91ca7eb2e121f84e60cf Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 6 Apr 2016 13:58:29 -0700 Subject: [PATCH 196/481] Changed all target values to architecture. Also added a sort_key method so that older architecture specs can be sorted with the newer ones --- lib/spack/spack/build_environment.py | 90 +--------------------------- lib/spack/spack/cmd/find.py | 18 ++++-- 2 files changed, 16 insertions(+), 92 deletions(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 8ed3c4e1a1..87fc310b5a 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -32,8 +32,6 @@ import shutil import multiprocessing import platform -import re - from llnl.util.filesystem import * import spack @@ -59,9 +57,6 @@ SPACK_SHORT_SPEC = 'SPACK_SHORT_SPEC' SPACK_DEBUG_LOG_DIR = 'SPACK_DEBUG_LOG_DIR' -SPACK_CRAYPE = 'SPACK_CRAYPE' -SPACK_COMP_MODULE = 'SPACK_COMP_MODULE' - class MakeExecutable(Executable): """Special callable executable object for make so the user can @@ -88,68 +83,6 @@ def __call__(self, *args, **kwargs): return super(MakeExecutable, self).__call__(*args, **kwargs) -def load_module(mod): - """Takes a module name and removes modules until it is possible to - load that module. It then loads the provided module. Depends on the - modulecmd implementation of modules used in cray and lmod. - """ - #Create an executable of the module command that will output python code - modulecmd = which('modulecmd') - modulecmd.add_default_arg('python') - - # Read the module and remove any conflicting modules - # We do this without checking that they are already installed - # for ease of programming because unloading a module that is not - # loaded does nothing. - text = modulecmd('show', mod, output=str, error=str).split() - for i, word in enumerate(text): - if word == 'conflict': - exec(compile(modulecmd('unload', text[i+1], output=str, error=str), '', 'exec')) - # Load the module now that there are no conflicts - load = modulecmd('load', mod, output=str, error=str) - exec(compile(load, '', 'exec')) - - -def get_path_from_module(mod): - """Inspects a TCL module for entries that indicate the absolute path - at which the library supported by said module can be found. - """ - # Create a modulecmd executable - modulecmd = which('modulecmd') - modulecmd.add_default_arg('python') - - # Read the module - text = modulecmd('show', mod, output=str, error=str).split('\n') - - # If it lists its package directory, return that - for line in text: - if line.find(mod.upper()+'_DIR') >= 0: - words = line.split() - return words[2] - - # If it lists a -rpath instruction, use that - for line in text: - rpath = line.find('-rpath/') - if rpath >= 0: - return line[rpath+6:line.find('/lib')] - - # If it lists a -L instruction, use that - for line in text: - L = line.find('-L/') - if L >= 0: - return line[L+2:line.find('/lib')] - - # If it sets the LD_LIBRARY_PATH or CRAY_LD_LIBRARY_PATH, use that - for line in text: - if line.find('LD_LIBRARY_PATH') >= 0: - words = line.split() - path = words[2] - return path[:path.find('/lib')] - - # Unable to find module path - return None - - def set_compiler_environment_variables(pkg): assert(pkg.spec.concrete) compiler = pkg.compiler @@ -176,10 +109,6 @@ def set_compiler_environment_variables(pkg): os.environ['SPACK_COMPILER_SPEC'] = str(pkg.spec.compiler) - if compiler.strategy == 'MODULES': - for mod in compiler.modules: - load_module(mod) - def set_build_environment_variables(pkg): """This ensures a clean install environment when we build packages. @@ -243,8 +172,6 @@ def add_env_path(path): pkg_config_dirs.append(pcdir) path_set("PKG_CONFIG_PATH", pkg_config_dirs) - if pkg.spec.architecture.target.module_name: - load_module(pkg.spec.architecture.target.module_name) def set_module_variables_for_package(pkg, m): """Populate the module scope of install() with some useful functions. @@ -314,21 +241,10 @@ def set_module_variables_for_package(pkg, m): def get_rpaths(pkg): """Get a list of all the rpaths for a package.""" - - # First load all modules for external packages and update the external - # packages' paths to reflect what is found in the modules so that we can - # rpath through the modules when possible, but if not possible they are - # already loaded. - for spec in pkg.spec.traverse(root=False): - if spec.external_module: - load_module(spec.external_module) - spec.external = get_path_from_module(spec.external_module) - - # Construct rpaths from the paths of each dep rpaths = [pkg.prefix.lib, pkg.prefix.lib64] - rpaths.extend(d.prefix.lib for d in pkg.spec.traverse(root=False) + rpaths.extend(d.prefix.lib for d in pkg.spec.dependencies.values() if os.path.isdir(d.prefix.lib)) - rpaths.extend(d.prefix.lib64 for d in pkg.spec.traverse(root=False) + rpaths.extend(d.prefix.lib64 for d in pkg.spec.dependencies.values() if os.path.isdir(d.prefix.lib64)) return rpaths @@ -348,8 +264,8 @@ def parent_class_modules(cls): def setup_package(pkg): """Execute all environment setup routines.""" - set_build_environment_variables(pkg) set_compiler_environment_variables(pkg) + set_build_environment_variables(pkg) # If a user makes their own package repo, e.g. # spack.repos.mystuff.libelf.Libelf, and they inherit from diff --git a/lib/spack/spack/cmd/find.py b/lib/spack/spack/cmd/find.py index 4223c53b06..95f035c7bf 100644 --- a/lib/spack/spack/cmd/find.py +++ b/lib/spack/spack/cmd/find.py @@ -90,18 +90,26 @@ def display_specs(specs, **kwargs): hlen = None # Make a dict with specs keyed by target and compiler. - index = index_by(specs, ('target', 'compiler')) + index = index_by(specs, ('architecture', 'compiler')) - # Traverse the index and print out each package - for i, (target, compiler) in enumerate(sorted(index)): + # Create sort key based off of whether read architecture is string + def sort_key(index_dict): + if isinstance(index_dict[0],basestring): + return(str, index_dict[1]._cmp_key()) + elif isinstance(index_dict[0], spack.architecture.Arch): + return (index_dict[0]._cmp_key(), index_dict[1]._cmp_key()) + + sorted_index = sorted(index, key=sort_key) + + for i, (architecture, compiler) in enumerate(sorted_index): if i > 0: print header = "%s{%s} / %s{%s}" % ( - spack.spec.target_color, target, + spack.spec.architecture_color, architecture, spack.spec.compiler_color, compiler) tty.hline(colorize(header), char='-') - specs = index[(target,compiler)] + specs = index[(architecture, compiler)] specs.sort() nfmt = '.' if namespace else '_' From 99117cbc9067df814c99f8427f658ad9a811f69b Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 6 Apr 2016 14:07:22 -0700 Subject: [PATCH 197/481] pulled from current repo the build_environment.py. Added module loading methods from our cray port --- lib/spack/spack/build_environment.py | 71 ++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 87fc310b5a..12a529d070 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -83,6 +83,66 @@ def __call__(self, *args, **kwargs): return super(MakeExecutable, self).__call__(*args, **kwargs) +def load_module(mod): + """Takes a module name and removes modules until it is possible to + load that module. It then loads the provided module. Depends on the + modulecmd implementation of modules used in cray and lmod. + """ + #Create an executable of the module command that will output python code + modulecmd = which('modulecmd') + modulecmd.add_default_arg('python') + + # Read the module and remove any conflicting modules + # We do this without checking that they are already installed + # for ease of programming because unloading a module that is not + # loaded does nothing. + text = modulecmd('show', mod, output=str, error=str).split() + for i, word in enumerate(text): + if word == 'conflict': + exec(compile(modulecmd('unload', text[i+1], output=str, error=str), '', 'exec')) + # Load the module now that there are no conflicts + load = modulecmd('load', mod, output=str, error=str) + exec(compile(load, '', 'exec')) + +def get_path_from_module(mod): + """Inspects a TCL module for entries that indicate the absolute path + at which the library supported by said module can be found. + """ + # Create a modulecmd executable + modulecmd = which('modulecmd') + modulecmd.add_default_arg('python') + + # Read the module + text = modulecmd('show', mod, output=str, error=str).split('\n') + + # If it lists its package directory, return that + for line in text: + if line.find(mod.upper()+'_DIR') >= 0: + words = line.split() + return words[2] + + # If it lists a -rpath instruction, use that + for line in text: + rpath = line.find('-rpath/') + if rpath >= 0: + return line[rpath+6:line.find('/lib')] + + # If it lists a -L instruction, use that + for line in text: + L = line.find('-L/') + if L >= 0: + return line[L+2:line.find('/lib')] + + # If it sets the LD_LIBRARY_PATH or CRAY_LD_LIBRARY_PATH, use that + for line in text: + if line.find('LD_LIBRARY_PATH') >= 0: + words = line.split() + path = words[2] + return path[:path.find('/lib')] + + # Unable to find module path + return None + def set_compiler_environment_variables(pkg): assert(pkg.spec.concrete) compiler = pkg.compiler @@ -109,6 +169,9 @@ def set_compiler_environment_variables(pkg): os.environ['SPACK_COMPILER_SPEC'] = str(pkg.spec.compiler) + if compiler.strategy == 'MODULES': + for mod in compiler.modules: + load_module(mod) def set_build_environment_variables(pkg): """This ensures a clean install environment when we build packages. @@ -172,6 +235,9 @@ def add_env_path(path): pkg_config_dirs.append(pcdir) path_set("PKG_CONFIG_PATH", pkg_config_dirs) + if pkg.spec.architecture.target.module_name: + load_module(pkg.spec.architecture.target.module_name) + def set_module_variables_for_package(pkg, m): """Populate the module scope of install() with some useful functions. @@ -241,6 +307,11 @@ def set_module_variables_for_package(pkg, m): def get_rpaths(pkg): """Get a list of all the rpaths for a package.""" + for spec in pkg.spec.traverse(root=False): + if spec.external_module: + load_module(spec.external_module) + spec.external = get_path_from_module(spec.external_module) + rpaths = [pkg.prefix.lib, pkg.prefix.lib64] rpaths.extend(d.prefix.lib for d in pkg.spec.dependencies.values() if os.path.isdir(d.prefix.lib)) From 143a4d7f7688b506811a564768ed49a2a12d1992 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 7 Apr 2016 14:36:26 -0700 Subject: [PATCH 198/481] Changed default to default_target, also added frontend and backend to operating system --- lib/spack/spack/architecture.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 555fb14408..ca306b0484 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -118,14 +118,14 @@ def target(self, name): front-end, and back-end. This can be overwritten by a subclass for which we want to provide further aliasing options. """ - if name == 'default': + if name == 'default_target': name = self.default - elif name == 'front_end' or name == 'fe': + elif name == 'frontend' or name == 'fe': name = self.front_end - elif name == 'back_end' or name == 'be': + elif name == 'backend' or name == 'be': name = self.back_end - return self.targets[name] + return self.targets.get(name, None) def add_operating_system(self, name, os_class): """ Add the operating_system class object into the @@ -136,12 +136,12 @@ def add_operating_system(self, name, os_class): def operating_system(self, name): if name == 'default_os': name = self.default_os - if name == 'front_os': + if name == 'frontend' or name == "fe": name = self.front_os - if name == 'back_os': + if name == 'backend' or name == 'be': name = self.back_os - return self.operating_sys[name] + return self.operating_sys.get(name, None) @classmethod def detect(self): From 07df40320331c118306c40c36deae8d57ca6073c Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 7 Apr 2016 14:37:00 -0700 Subject: [PATCH 199/481] Cleaned up the concretize_architecture method by removing commented out code --- lib/spack/spack/concretize.py | 62 +++-------------------------------- 1 file changed, 5 insertions(+), 57 deletions(-) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index c5dad93aef..14edcf1f3d 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -236,14 +236,6 @@ def _concretize_operating_system(self, spec): return True #changed -# """ Future method for concretizing operating system """ -# if isinstance(arch.platform_os, spack.architecture.OperatingSystem): -# return False -# else: -# arch.arch_os = platform.operating_system('default_os') -# return True - - def _concretize_target(self, spec): if spec.architecture.target is not None: if isinstance(spec.architecture.target,spack.architecture.Target): @@ -258,16 +250,10 @@ def _concretize_target(self, spec): else: spec.add_target_from_string(spec.root.architecture.target) else: - spec.architecture.target = spec.architecture.platform.target('default') + spec.architecture.target = spec.architecture.platform.target('default_target') return True #changed -# if isinstance(arch.target, spack.architecture.Target): -# return False -# else: -# arch.target = platform.target('default') -# return True - def concretize_architecture(self, spec): """If the spec is empty provide the defaults of the platform. If the architecture is not a basestring, then check if either the platform, @@ -283,50 +269,12 @@ def concretize_architecture(self, spec): # Set the architecture to all defaults spec.architecture = spack.architecture.Arch() return True - #If there is a target and it is a tuple and has both filled return - #False -# if isinstance(spec.architecture, basestring): -# spec.split_architecture_string(spec.architecture) - - ret = any(( - self._concretize_operating_system(spec), - self._concretize_target(spec))) - - - # Does not look pretty at all!!! -# if spec.root.architecture and \ -# not isinstance(spec.root.architecture, basestring): -# bool_flag = any(( -# self._concretize_platform(spec.root.architecture, platform), -# self._concretize_operating_system(spec.root.architecture, -# platform), -# self._concretize_target(spec.root.target, platform))) -# spec.architecture =spec.root.architecture -# return bool_flag -# else: -# spec.add_architecture_from_string(spec.root.architecture) - + + # Concretize the operating_system and target based of the spec + ret = any((self._concretize_operating_system(spec), + self._concretize_target(spec))) return ret - # if there is no target specified used the defaults - - #if spec.target is not None: - # if isinstance(spec.target,spack.architecture.Target): - # return False - # else: - # spec.add_target_from_string(spec.target) - # return True #changed - - #if spec.root.target: - # if isinstance(spec.root.target,spack.architecture.Target): - # spec.target = spec.root.target - # else: - # spec.add_target_from_string(spec.root.target) - #else: - # platform = spack.architecture.sys_type() - # spec.target = platform.target('default') - - #return True #changed def concretize_variants(self, spec): From 0ad317213c0f32c125c7043b5e57aa678ba63f8f Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 7 Apr 2016 14:38:04 -0700 Subject: [PATCH 200/481] Changed architecture parser so that if user just enters in frontend or backend, then both the os and target will take those names. In the concretize method the frontend target/os and backend target/os will be picked to match each other --- lib/spack/spack/spec.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index fa2e9fd4c8..94d881da28 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -462,11 +462,10 @@ def _set_architecture(self, architecture): """Called by the parser to set the architecture.""" if self.architecture: raise DuplicateArchitectureError( "Spec for '%s' cannot have two architectures." % self.name) - platform = spack.architecture.sys_type() if '-' in architecture: os, target = architecture.split('-') - else: - os = None + elif architecture == 'frontend' or architecture == 'backend': + os = architecture target = architecture self.architecture = spack.architecture.Arch(os, target) From 7b777568ed27f2dc0f15283d0209541b65b47b24 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 7 Apr 2016 14:38:43 -0700 Subject: [PATCH 201/481] Added new concretization of architecture tests, test each combination of user input to make sure the correct os and target are chosen --- lib/spack/spack/test/architecture.py | 45 ++++++++++++++++++++++++++-- 1 file changed, 43 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/test/architecture.py b/lib/spack/spack/test/architecture.py index 2274a9901d..1df1e3ffa2 100644 --- a/lib/spack/spack/test/architecture.py +++ b/lib/spack/spack/test/architecture.py @@ -6,7 +6,7 @@ import platform import spack from spack.architecture import * -import spack.spec +from spack.spec import * from spack.platforms.cray_xc import CrayXc from spack.platforms.linux import Linux from spack.platforms.bgq import Bgq @@ -17,7 +17,7 @@ class ArchitectureTest(unittest.TestCase): def test_dict_functions_for_architecture(self): arch = Arch() arch.platform_os = arch.platform.operating_system('default_os') - arch.target = arch.platform.target('default') + arch.target = arch.platform.target('default_target') d = arch.to_dict() @@ -55,3 +55,44 @@ def test_sys_type(self): my_platform_class = Darwin() self.assertEqual(str(output_platform_class), str(my_platform_class)) + + def setUp(self): + self.platform = sys_type() + + def test_user_front_end_input(self): + """Test when user inputs just frontend that both the frontend target + and frontend operating system match + """ + frontend_os = self.platform.operating_system("frontend") + frontend_target = self.platform.target("frontend") + print frontend_os + print frontend_target + frontend_spec = Spec("zlib=frontend") + frontend_spec.concretize() + self.assertEqual(frontend_os, frontend_spec.architecture.platform_os) + self.assertEqual(frontend_target, frontend_spec.architecture.target) + + def test_user_back_end_input(self): + """Test when user inputs backend that both the backend target and + backend operating system match + """ + backend_os = self.platform.operating_system("backend") + backend_target = self.platform.target("backend") + print backend_os + print backend_target + backend_spec = Spec("zlib=backend") + backend_spec.concretize() + self.assertEqual(backend_os, backend_spec.architecture.platform_os) + self.assertEqual(backend_target, backend_spec.architecture.target) + + def test_user_defaults(self): + default_os = self.platform.operating_system("default_os") + default_target = self.platform.target("default_target") + + default_spec = Spec("zlib") # default is no args + default_spec.concretize() + self.assertEqual(default_os, default_spec.architecture.platform_os) + self.assertEqual(default_target, default_spec.architecture.target) + + def test_user_combination(self): + From 6ff6c805af2a4626e0611f3798eedd3a6f1b3e06 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 8 Apr 2016 10:28:30 -0700 Subject: [PATCH 202/481] Assigned self.front_os, and back_os to self.default. Helps with testing. If we're on a darwin or linux machine, chances are there is no back end or front end operating system, but rather than have those as None, I just assign them the default which is detected via python platform. --- lib/spack/spack/platforms/darwin.py | 4 ++++ lib/spack/spack/platforms/linux.py | 2 ++ 2 files changed, 6 insertions(+) diff --git a/lib/spack/spack/platforms/darwin.py b/lib/spack/spack/platforms/darwin.py index 4c3d38851f..7901f82d06 100644 --- a/lib/spack/spack/platforms/darwin.py +++ b/lib/spack/spack/platforms/darwin.py @@ -12,7 +12,11 @@ def __init__(self): super(Darwin, self).__init__('darwin') self.add_target(self.default, Target(self.default)) mac_os = MacOsx() + self.default_os = mac_os.name + self.front_os = mac_os.name + self.back_os = mac_os.name + self.add_operating_system(mac_os.name, mac_os) @classmethod diff --git a/lib/spack/spack/platforms/linux.py b/lib/spack/spack/platforms/linux.py index 3243a1dcdf..18050ac79e 100644 --- a/lib/spack/spack/platforms/linux.py +++ b/lib/spack/spack/platforms/linux.py @@ -13,6 +13,8 @@ def __init__(self): self.add_target(self.default, Target(self.default)) linux_dist = LinuxDistro() self.default_os = linux_dist.name + self.front_os = linux_dist.name + self.back_os = linux_dist.name self.add_operating_system(linux_dist.name, linux_dist) @classmethod From 0d1a1b7526ce1c129bed352b175f33f2875b8b41 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 8 Apr 2016 10:29:37 -0700 Subject: [PATCH 203/481] Changed _set_architecture so if user inputs only the target, then os is None. This prevents the os being referenced before assignment error --- lib/spack/spack/spec.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 94d881da28..28cd6a0274 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -461,12 +461,17 @@ def _set_compiler(self, compiler): def _set_architecture(self, architecture): """Called by the parser to set the architecture.""" if self.architecture: raise DuplicateArchitectureError( - "Spec for '%s' cannot have two architectures." % self.name) + "Spec for '%s' cannot have two architectures." % self.name) if '-' in architecture: os, target = architecture.split('-') + elif architecture == 'frontend' or architecture == 'backend': os = architecture target = architecture + else: + os = None + target = architecture + self.architecture = spack.architecture.Arch(os, target) From 277efc1dfbb00117db65918ff22928d42e1194e2 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 8 Apr 2016 11:17:56 -0700 Subject: [PATCH 204/481] Added test that works on every type of combination the user could possibly enter for arch spec --- lib/spack/spack/test/architecture.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/lib/spack/spack/test/architecture.py b/lib/spack/spack/test/architecture.py index 1df1e3ffa2..1ca3e58831 100644 --- a/lib/spack/spack/test/architecture.py +++ b/lib/spack/spack/test/architecture.py @@ -65,8 +65,6 @@ def test_user_front_end_input(self): """ frontend_os = self.platform.operating_system("frontend") frontend_target = self.platform.target("frontend") - print frontend_os - print frontend_target frontend_spec = Spec("zlib=frontend") frontend_spec.concretize() self.assertEqual(frontend_os, frontend_spec.architecture.platform_os) @@ -78,8 +76,6 @@ def test_user_back_end_input(self): """ backend_os = self.platform.operating_system("backend") backend_target = self.platform.target("backend") - print backend_os - print backend_target backend_spec = Spec("zlib=backend") backend_spec.concretize() self.assertEqual(backend_os, backend_spec.architecture.platform_os) @@ -94,5 +90,24 @@ def test_user_defaults(self): self.assertEqual(default_os, default_spec.architecture.platform_os) self.assertEqual(default_target, default_spec.architecture.target) - def test_user_combination(self): + def test_user_input_combination(self): + os_list = self.platform.operating_sys.keys() + target_list = self.platform.targets.keys() + additional = ["fe", "be", "frontend", "backend"] + + os_list.extend(additional) + target_list.extend(additional) + combinations = itertools.product(os_list, target_list) + results = [] + for arch in combinations: + o,t = arch + arch_spec = "-".join(arch) + spec = Spec("zlib=%s" % arch_spec) + spec.concretize() + results.append(spec.architecture.platform_os == self.platform.operating_system(o)) + results.append(spec.architecture.target == self.platform.target(t)) + res = all(results) + print res + self.assertTrue(res) + From 618b3f5f2d149fe3c20f90c0890748c8f3589afb Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 8 Apr 2016 11:18:30 -0700 Subject: [PATCH 205/481] Changed default to default_target so the tests can pass --- lib/spack/spack/test/spec_semantics.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/spack/spack/test/spec_semantics.py b/lib/spack/spack/test/spec_semantics.py index 0b310488a9..84d8c0edd8 100644 --- a/lib/spack/spack/test/spec_semantics.py +++ b/lib/spack/spack/test/spec_semantics.py @@ -308,7 +308,7 @@ def test_constrain_variants(self): def test_constrain_target(self): platform = spack.architecture.sys_type() - target = platform.target('default').name + target = platform.target('default_target').name self.check_constrain('libelf='+target, 'libelf='+target, 'libelf='+target) self.check_constrain('libelf='+target, 'libelf', 'libelf='+target) @@ -339,7 +339,7 @@ def test_constrain_changed(self): self.check_constrain_changed('libelf', '+debug') self.check_constrain_changed('libelf', '~debug') platform = spack.architecture.sys_type() - self.check_constrain_changed('libelf', '='+platform.target('default').name) + self.check_constrain_changed('libelf', '='+platform.target('default_target').name) def test_constrain_not_changed(self): @@ -351,8 +351,8 @@ def test_constrain_not_changed(self): self.check_constrain_not_changed('libelf+debug', '+debug') self.check_constrain_not_changed('libelf~debug', '~debug') platform = spack.architecture.sys_type() - default = platform.target('default').name - self.check_constrain_not_changed('libelf='+default, '='+default) + default_target = platform.target('default_target').name + self.check_constrain_not_changed('libelf='+default_target, '='+default_target) self.check_constrain_not_changed('libelf^foo', 'libelf^foo') self.check_constrain_not_changed('libelf^foo^bar', 'libelf^foo^bar') @@ -365,8 +365,8 @@ def test_constrain_dependency_changed(self): self.check_constrain_changed('libelf^foo', 'libelf^foo+debug') self.check_constrain_changed('libelf^foo', 'libelf^foo~debug') platform = spack.architecture.sys_type() - default = platform.target('default').name - self.check_constrain_changed('libelf^foo', 'libelf^foo='+default) + default_target = platform.target('default_target').name + self.check_constrain_changed('libelf^foo', 'libelf^foo='+default_target) def test_constrain_dependency_not_changed(self): @@ -377,6 +377,6 @@ def test_constrain_dependency_not_changed(self): self.check_constrain_not_changed('libelf^foo+debug', 'libelf^foo+debug') self.check_constrain_not_changed('libelf^foo~debug', 'libelf^foo~debug') platform = spack.architecture.sys_type() - default = platform.target('default').name - self.check_constrain_not_changed('libelf^foo='+default, 'libelf^foo='+default) + default_target = platform.target('default_target').name + self.check_constrain_not_changed('libelf^foo='+default_target, 'libelf^foo='+default_target) From 5bcd1e7ccd9ff6568c06a13998f2d552d1b37b60 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 8 Apr 2016 12:43:08 -0700 Subject: [PATCH 206/481] If we already have the name and the platform, then we should probably go ahead and concretize it in the constructor. Else leave it as None and concretize it later. --- lib/spack/spack/architecture.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index ca306b0484..e2c189980a 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -199,8 +199,13 @@ class Arch(object): def __init__(self, platform_os=None, target=None): self.platform = sys_type() + if platform_os: + platform_os = self.platform.operating_system(platform_os) self.platform_os = platform_os - self.target = target + if target: + target = self.platform.target(target) + self.target = target + @property def concrete(self): From 01a36ab33341e2af91c6e6f5ca58efbeecfb6ec4 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 8 Apr 2016 12:44:27 -0700 Subject: [PATCH 207/481] Changed the logic of concretize. We don't have to worry about whether the part of the spec is a string since any string parsed is made into an object via the Arch constructor. Any dependencies will take the root arch spec as well --- lib/spack/spack/concretize.py | 32 +++++++++++--------------------- 1 file changed, 11 insertions(+), 21 deletions(-) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 14edcf1f3d..3422e86ae8 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -220,39 +220,29 @@ def concretize_version(self, spec): return True # Things changed def _concretize_operating_system(self, spec): - if spec.architecture.platform_os is not None: - if isinstance(spec.architecture.platform_os,spack.architecture.OperatingSystem): - return False - else: - spec.add_operating_system_from_string(spec.architecture.platform_os) - return True #changed + platform = spec.architecture.platform + if spec.architecture.platform_os is not None and isinstance( + spec.architecture.platform_os,spack.architecture.OperatingSystem): + return False + if spec.root.architecture and spec.root.architecture.platform_os: if isinstance(spec.root.architecture.platform_os,spack.architecture.OperatingSystem): spec.architecture.platform_os = spec.root.architecture.platform_os - else: - spec.add_operating_system_from_string(spec.root.architecture.platform_os) else: spec.architecture.platform_os = spec.architecture.platform.operating_system('default_os') - - return True #changed + return True #changed def _concretize_target(self, spec): - if spec.architecture.target is not None: - if isinstance(spec.architecture.target,spack.architecture.Target): - return False - else: - spec.add_target_from_string(spec.architecture.target) - return True #changed - + platform = spec.architecture.platform + if spec.architecture.target is not None and isinstance( + spec.architecture.target, spack.architecture.Target): + return False if spec.root.architecture and spec.root.architecture.target: if isinstance(spec.root.architecture.target,spack.architecture.Target): spec.architecture.target = spec.root.architecture.target - else: - spec.add_target_from_string(spec.root.architecture.target) else: spec.architecture.target = spec.architecture.platform.target('default_target') - - return True #changed + return True #changed def concretize_architecture(self, spec): """If the spec is empty provide the defaults of the platform. If the From 3a281b239998133c7ed3fe03a54e44d643d0d0bd Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 8 Apr 2016 12:46:44 -0700 Subject: [PATCH 208/481] Commented out the add_x_from_string methods since we don't need to worry about it. Also, changed the way architecture is set. Todd wanted to have users be able to input just frontend and/or backend in the architecture spec. This will be able to parse those inputs. --- lib/spack/spack/spec.py | 33 +++++++++++---------------------- 1 file changed, 11 insertions(+), 22 deletions(-) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 28cd6a0274..eee9318550 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -465,7 +465,7 @@ def _set_architecture(self, architecture): if '-' in architecture: os, target = architecture.split('-') - elif architecture == 'frontend' or architecture == 'backend': + elif architecture in ['frontend','backend','fe','be']: os = architecture target = architecture else: @@ -1302,28 +1302,17 @@ def _autospec(self, spec_like): except SpecError: return parse_anonymous_spec(spec_like, self.name) - def _is_valid_platform(self, platform, platform_list): - if platform in platform_list: - return True - return False + #def add_target_from_string(self, target): + # if target is None: + # self.architecture.target = self.architecture.platform.target('default_target') + # else: + # self.architecture.target = self.architecture.platform.target(target) - def _is_valid_target(self, target, platform): - return target in platform.targets - - def _is_valid_os(self, os_string, platform): - return os_string in platform.operating_sys - - def add_target_from_string(self, target): - if target is None: - self.architecture.target = self.architecture.platform.target('default_target') - else: - self.architecture.target = self.architecture.platform.target(target) - - def add_operating_system_from_string(self, os): - if os is None: - self.architecture.platform_os = self.architecture.platform.operating_system('default_os') - else: - self.architecture.platform_os = self.architecture.platform.operating_system(os) + #def add_operating_system_from_string(self, os): + # if os is None: + # self.architecture.platform_os = self.architecture.platform.operating_system('default_os') + # else: + # self.architecture.platform_os = self.architecture.platform.operating_system(os) def satisfies(self, other, deps=True, strict=False): From 49956faab97e2dea52371755d0fcb9b013567ac7 Mon Sep 17 00:00:00 2001 From: Brett Viren Date: Wed, 24 Feb 2016 12:19:06 +0100 Subject: [PATCH 209/481] First try to make file system views of specs. --- lib/spack/spack/cmd/view.py | 108 ++++++++++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 lib/spack/spack/cmd/view.py diff --git a/lib/spack/spack/cmd/view.py b/lib/spack/spack/cmd/view.py new file mode 100644 index 0000000000..c7f1c21f67 --- /dev/null +++ b/lib/spack/spack/cmd/view.py @@ -0,0 +1,108 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import os +import argparse + +import spack +import spack.cmd +import llnl.util.tty as tty + +description = "Produce a single-rooted directory view of a spec." + +def setup_parser(subparser): + setup_parser.parser = subparser + + subparser.add_argument('prefix', nargs=1, + help="Path to top-level directory for the view.") + subparser.add_argument('specs', nargs=argparse.REMAINDER, + help="specs of packages to expose in the view.") + +def assuredir(path): + 'Assure path exists as a directory' + if not os.path.exists(path): + os.makedirs(path) + +def relative_to(prefix, path): + assert 0 == path.find(prefix) + reldir = path[len(prefix):] + if reldir.startswith('/'): + reldir = reldir[1:] + return reldir + +def view_one(prefix, spec): + print spec.name,spec.prefix + + dotspack = os.path.join(prefix, '.spack', spec.name) + + if os.path.exists(os.path.join(dotspack)): + tty.warn("Skipping previously added package %s"%spec.name) + return + + + for dirpath,dirnames,filenames in os.walk(spec.prefix): + if not filenames: + continue # avoid explicitly making empty dirs + + reldir = relative_to(spec.prefix, dirpath) + + # fixme: assumes .spack has no subdirs + if dirpath.endswith('.spack'): + targdir = dotspack + else: + targdir = os.path.join(prefix, reldir) + + assuredir(targdir) + + print '\t%s...'%reldir, + nlinks = 0 + for fname in filenames: + src = os.path.join(dirpath, fname) + dst = os.path.join(targdir, fname) + if os.path.exists(dst): + tty.warn("Skipping existing file for view: %s" % dst) + continue + os.symlink(src,dst) + nlinks += 1 + print nlinks + + +def view(parser, args): + # if not args.specs: + # tty.die("view creation requires at least one spec argument") + + specs = spack.cmd.parse_specs(args.specs, normalize=True, concretize=True) + if not specs: + setup_parser.parser.print_help() + return 1 + + prefix = args.prefix[0] + assuredir(prefix) + + flat = set() + for spec in specs: + flat.update(spec.normalized().traverse()) + + for spec in flat: + view_one(prefix, spec) From 8cd13d4b3552e6737ed9fc768bbe544024e05b10 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 28 Apr 2016 12:49:21 -0700 Subject: [PATCH 210/481] Make arch command print out the platform. --- lib/spack/spack/cmd/arch.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/spack/spack/cmd/arch.py b/lib/spack/spack/cmd/arch.py index db27544ffd..cd0ce2bb2e 100644 --- a/lib/spack/spack/cmd/arch.py +++ b/lib/spack/spack/cmd/arch.py @@ -28,8 +28,4 @@ description = "Print the architecture for this machine" def arch(parser, args): - configured_sys_type = architecture.get_sys_type_from_spack_globals() - if not configured_sys_type: - configured_sys_type = "autodetect" - print "Configured sys_type: %s" % configured_sys_type - print "Autodetected default sys_type: %s" % architecture.sys_type() + print architecture.sys_type() From a2b9a000dcb18868ea9f1c110632a251fbea92f3 Mon Sep 17 00:00:00 2001 From: Brett Viren Date: Fri, 29 Apr 2016 15:08:54 -0400 Subject: [PATCH 211/481] Add add removal and status actions in addition to link and add various ways to filter what is done. --- lib/spack/spack/cmd/view.py | 130 +++++++++++++++++++++++++++++------- 1 file changed, 107 insertions(+), 23 deletions(-) diff --git a/lib/spack/spack/cmd/view.py b/lib/spack/spack/cmd/view.py index c7f1c21f67..dd698350b2 100644 --- a/lib/spack/spack/cmd/view.py +++ b/lib/spack/spack/cmd/view.py @@ -1,3 +1,8 @@ +''' +Produce a file-system "view" of a Spack DAG. + +Concept from Nix, implemented by brett.viren@gmail.com ca 2016. +''' ############################################################################## # Copyright (c) 2013, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory. @@ -22,7 +27,9 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## + import os +import re import argparse import spack @@ -34,8 +41,15 @@ def setup_parser(subparser): setup_parser.parser = subparser - subparser.add_argument('prefix', nargs=1, - help="Path to top-level directory for the view.") + subparser.add_argument('-e','--exclude', action='append', default=[], + help="exclude any packages which the given re pattern") + subparser.add_argument('-a','--action', default='link', + choices=['link','remove','status'], # names match action_*() functions below + help="what action to perform on the view") + subparser.add_argument('--no-dependencies', action='store_true', default=False, + help="just operate on named packages and do not follow dependencies") + subparser.add_argument('-p','--prefix', required=True, + help="Path to a top-level directory to receive the view.") subparser.add_argument('specs', nargs=argparse.REMAINDER, help="specs of packages to expose in the view.") @@ -45,64 +59,134 @@ def assuredir(path): os.makedirs(path) def relative_to(prefix, path): + 'Return end of `path` relative to `prefix`' assert 0 == path.find(prefix) reldir = path[len(prefix):] if reldir.startswith('/'): reldir = reldir[1:] return reldir -def view_one(prefix, spec): - print spec.name,spec.prefix + +def transform_path(spec, path, prefix=None): + 'Return the a relative path corresponding to given path spec.prefix' + if os.path.isabs(path): + path = relative_to(spec.prefix, path) + subdirs = path.split(os.path.sep) + if subdirs[0] == '.spack': + lst = ['.spack',spec.name]+subdirs[1:] + path = os.path.join(*lst) + if prefix: + path = os.path.join(prefix, path) + return path +def action_status(spec, prefix): + 'Check status of view in prefix against spec' dotspack = os.path.join(prefix, '.spack', spec.name) - if os.path.exists(os.path.join(dotspack)): - tty.warn("Skipping previously added package %s"%spec.name) + tty.info("Package added: %s"%spec.name) + return + tty.info("Package missing: %s"%spec.name) + return + +def action_remove(spec, prefix): + 'Remove any files found in spec from prefix and purge empty directories.' + + if not os.path.exists(prefix): return + dotspack = transform_path(spec, '.spack', prefix) + if not os.path.exists(dotspack): + tty.info("Skipping nonexistent package %s"%spec.name) + return + + for dirpath,dirnames,filenames in os.walk(spec.prefix): + if not filenames: + continue + + targdir = transform_path(spec, dirpath, prefix) + for fname in filenames: + src = os.path.join(dirpath, fname) + dst = os.path.join(targdir, fname) + if not os.path.exists(dst): + #tty.warn("Skipping nonexistent file for view: %s" % dst) + continue + os.unlink(dst) + + +def action_link(spec, prefix): + 'Symlink all files in `spec` into directory `prefix`.' + + dotspack = transform_path(spec, '.spack', prefix) + if os.path.exists(dotspack): + tty.warn("Skipping previously added package %s"%spec.name) + return for dirpath,dirnames,filenames in os.walk(spec.prefix): if not filenames: continue # avoid explicitly making empty dirs - reldir = relative_to(spec.prefix, dirpath) - - # fixme: assumes .spack has no subdirs - if dirpath.endswith('.spack'): - targdir = dotspack - else: - targdir = os.path.join(prefix, reldir) - + targdir = transform_path(spec, dirpath, prefix) assuredir(targdir) - print '\t%s...'%reldir, - nlinks = 0 for fname in filenames: src = os.path.join(dirpath, fname) dst = os.path.join(targdir, fname) if os.path.exists(dst): + if '.spack' in dst.split(os.path.sep): + continue # silence these tty.warn("Skipping existing file for view: %s" % dst) continue os.symlink(src,dst) - nlinks += 1 - print nlinks - + + + +def purge_empty_directories(path): + 'Ascend up from the leaves accessible from `path` and remove empty directories.' + for dirpath, subdirs, files in os.walk(path, topdown=False): + for sd in subdirs: + sdp = os.path.join(dirpath,sd) + try: + os.rmdir(sdp) + except OSError: + tty.warn("Not removing directory with contents: %s" % sdp) + def view(parser, args): - # if not args.specs: - # tty.die("view creation requires at least one spec argument") + 'The view command.' + to_exclude = [re.compile(e) for e in args.exclude] + def exclude(spec): + for e in to_exclude: + if e.match(spec.name): + return True + return False specs = spack.cmd.parse_specs(args.specs, normalize=True, concretize=True) if not specs: setup_parser.parser.print_help() return 1 - prefix = args.prefix[0] + prefix = args.prefix assuredir(prefix) flat = set() for spec in specs: + if args.no_dependencies: + flat.add(spec) + continue flat.update(spec.normalized().traverse()) + action = args.action.lower() + method = eval('action_' + action) + for spec in flat: - view_one(prefix, spec) + if exclude(spec): + tty.info('Skipping excluded package: "%s"' % spec.name) + continue + if not os.path.exists(spec.prefix): + tty.warn('Skipping unknown package: %s in %s' % (spec.name, spec.prefix)) + continue + tty.info("%s %s" % (action, spec.name)) + method(spec, prefix) + + if action in ['remove']: + purge_empty_directories(prefix) From 8c140f4725608510d0d7ecc6195736ca513b02ab Mon Sep 17 00:00:00 2001 From: Brett Viren Date: Sat, 30 Apr 2016 08:46:47 -0400 Subject: [PATCH 212/481] Change the action from parameter to subcommand to fit Spack convention. --- lib/spack/spack/cmd/view.py | 58 ++++++++++++++++++++++++++----------- 1 file changed, 41 insertions(+), 17 deletions(-) diff --git a/lib/spack/spack/cmd/view.py b/lib/spack/spack/cmd/view.py index dd698350b2..a82c034398 100644 --- a/lib/spack/spack/cmd/view.py +++ b/lib/spack/spack/cmd/view.py @@ -41,17 +41,30 @@ def setup_parser(subparser): setup_parser.parser = subparser - subparser.add_argument('-e','--exclude', action='append', default=[], - help="exclude any packages which the given re pattern") - subparser.add_argument('-a','--action', default='link', - choices=['link','remove','status'], # names match action_*() functions below - help="what action to perform on the view") - subparser.add_argument('--no-dependencies', action='store_true', default=False, - help="just operate on named packages and do not follow dependencies") - subparser.add_argument('-p','--prefix', required=True, - help="Path to a top-level directory to receive the view.") - subparser.add_argument('specs', nargs=argparse.REMAINDER, - help="specs of packages to expose in the view.") + sp = subparser.add_subparsers(metavar='ACTION', dest='action') + + # The action parameterizes the command but in keeping with Spack + # patterns we make it a subcommand. + sps = [ + sp.add_parser('link', aliases=['add'], + help='Add packages to the view, create view if needed.'), + sp.add_parser('remove', aliases=['rm'], + help='Remove packages from the view, and view if empty.'), + sp.add_parser('status', aliases=['check'], + help='Check status of packages in the view.') + ] + + # All these options and arguments are common to every action. + for p in sps: + p.add_argument('-e','--exclude', action='append', default=[], + help="exclude any packages which the given re pattern") + p.add_argument('--no-dependencies', action='store_true', default=False, + help="just operate on named packages and do not follow dependencies") + p.add_argument('prefix', nargs=1, + help="Path to a top-level directory to receive the view.") + p.add_argument('specs', nargs=argparse.REMAINDER, + help="specs of packages to expose in the view.") + def assuredir(path): 'Assure path exists as a directory' @@ -151,7 +164,9 @@ def purge_empty_directories(path): tty.warn("Not removing directory with contents: %s" % sdp) -def view(parser, args): + + +def view_action(action, parser, args): 'The view command.' to_exclude = [re.compile(e) for e in args.exclude] def exclude(spec): @@ -162,7 +177,7 @@ def exclude(spec): specs = spack.cmd.parse_specs(args.specs, normalize=True, concretize=True) if not specs: - setup_parser.parser.print_help() + parser.print_help() return 1 prefix = args.prefix @@ -175,9 +190,6 @@ def exclude(spec): continue flat.update(spec.normalized().traverse()) - action = args.action.lower() - method = eval('action_' + action) - for spec in flat: if exclude(spec): tty.info('Skipping excluded package: "%s"' % spec.name) @@ -186,7 +198,19 @@ def exclude(spec): tty.warn('Skipping unknown package: %s in %s' % (spec.name, spec.prefix)) continue tty.info("%s %s" % (action, spec.name)) - method(spec, prefix) + action(spec, prefix) if action in ['remove']: purge_empty_directories(prefix) + + +def view(parser, args): + action = { + 'add': action_link, + 'link': action_link, + 'remove': action_remove, + 'rm': action_remove, + 'status': action_status, + 'check': action_status + }[args.action] + view_action(action, parser, args) From 346f1022343d608ffabd565636ec8358c90464c8 Mon Sep 17 00:00:00 2001 From: Brett Viren Date: Sun, 1 May 2016 13:54:07 -0400 Subject: [PATCH 213/481] Bug fixes, tty tweaks. --- lib/spack/spack/cmd/view.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/lib/spack/spack/cmd/view.py b/lib/spack/spack/cmd/view.py index a82c034398..1bdaaecadc 100644 --- a/lib/spack/spack/cmd/view.py +++ b/lib/spack/spack/cmd/view.py @@ -46,7 +46,7 @@ def setup_parser(subparser): # The action parameterizes the command but in keeping with Spack # patterns we make it a subcommand. sps = [ - sp.add_parser('link', aliases=['add'], + sp.add_parser('add', aliases=['link'], help='Add packages to the view, create view if needed.'), sp.add_parser('remove', aliases=['rm'], help='Remove packages from the view, and view if empty.'), @@ -112,6 +112,7 @@ def action_remove(spec, prefix): tty.info("Skipping nonexistent package %s"%spec.name) return + tty.info("remove %s"%spec.name) for dirpath,dirnames,filenames in os.walk(spec.prefix): if not filenames: continue @@ -125,7 +126,6 @@ def action_remove(spec, prefix): continue os.unlink(dst) - def action_link(spec, prefix): 'Symlink all files in `spec` into directory `prefix`.' @@ -134,6 +134,7 @@ def action_link(spec, prefix): tty.warn("Skipping previously added package %s"%spec.name) return + tty.info("link %s" % spec.name) for dirpath,dirnames,filenames in os.walk(spec.prefix): if not filenames: continue # avoid explicitly making empty dirs @@ -151,8 +152,6 @@ def action_link(spec, prefix): continue os.symlink(src,dst) - - def purge_empty_directories(path): 'Ascend up from the leaves accessible from `path` and remove empty directories.' for dirpath, subdirs, files in os.walk(path, topdown=False): @@ -161,13 +160,14 @@ def purge_empty_directories(path): try: os.rmdir(sdp) except OSError: - tty.warn("Not removing directory with contents: %s" % sdp) + #tty.warn("Not removing directory with contents: %s" % sdp) + pass def view_action(action, parser, args): - 'The view command.' + 'The view command parameterized by the action.' to_exclude = [re.compile(e) for e in args.exclude] def exclude(spec): for e in to_exclude: @@ -180,7 +180,7 @@ def exclude(spec): parser.print_help() return 1 - prefix = args.prefix + prefix = args.prefix[0] assuredir(prefix) flat = set() @@ -197,14 +197,15 @@ def exclude(spec): if not os.path.exists(spec.prefix): tty.warn('Skipping unknown package: %s in %s' % (spec.name, spec.prefix)) continue - tty.info("%s %s" % (action, spec.name)) action(spec, prefix) - if action in ['remove']: + if args.action in ['remove','rm']: purge_empty_directories(prefix) + def view(parser, args): + 'The view command.' action = { 'add': action_link, 'link': action_link, From 26c5bc9d979b95df560bdb7d7a6bd747ef93f83d Mon Sep 17 00:00:00 2001 From: Brett Viren Date: Sun, 1 May 2016 17:10:32 -0400 Subject: [PATCH 214/481] A few minimal test packages, fodder for testing 'spack view' command. --- .../builtin.mock/packages/test-a/package.py | 29 ++++++++++++++++ .../packages/test-a/test-a-0.0.tar.gz | Bin 0 -> 112 bytes .../builtin.mock/packages/test-b/package.py | 31 +++++++++++++++++ .../packages/test-b/test-b-0.0.tar.gz | Bin 0 -> 112 bytes .../builtin.mock/packages/test-c/package.py | 31 +++++++++++++++++ .../packages/test-c/test-c-0.0.tar.gz | Bin 0 -> 112 bytes .../builtin.mock/packages/test-d/package.py | 32 ++++++++++++++++++ .../packages/test-d/test-d-0.0.tar.gz | Bin 0 -> 112 bytes 8 files changed, 123 insertions(+) create mode 100644 var/spack/repos/builtin.mock/packages/test-a/package.py create mode 100644 var/spack/repos/builtin.mock/packages/test-a/test-a-0.0.tar.gz create mode 100644 var/spack/repos/builtin.mock/packages/test-b/package.py create mode 100644 var/spack/repos/builtin.mock/packages/test-b/test-b-0.0.tar.gz create mode 100644 var/spack/repos/builtin.mock/packages/test-c/package.py create mode 100644 var/spack/repos/builtin.mock/packages/test-c/test-c-0.0.tar.gz create mode 100644 var/spack/repos/builtin.mock/packages/test-d/package.py create mode 100644 var/spack/repos/builtin.mock/packages/test-d/test-d-0.0.tar.gz diff --git a/var/spack/repos/builtin.mock/packages/test-a/package.py b/var/spack/repos/builtin.mock/packages/test-a/package.py new file mode 100644 index 0000000000..2f72370580 --- /dev/null +++ b/var/spack/repos/builtin.mock/packages/test-a/package.py @@ -0,0 +1,29 @@ +from spack import * + +import os + +mydir = os.path.dirname(__file__) +source = os.path.join(mydir,'test-a-0.0.tar.gz') + +class TestA(Package): + """The test-a package""" + + url = 'file://'+source + + version('0.0', '4e823d0af4154fcf52b75dad41b7fd63') + + variant('nom', default=True, description='Nominal variant') + variant('var', default=False, description='Variant variant') + + def install(self, spec, prefix): + bindir = os.path.join(prefix,'bin') + os.makedirs(bindir) + script = os.path.join(bindir, 'test-a') + with open(script,'w') as fp: + fp.write("""#!/bin/bash +echo 'name: %s' +echo 'prefix: %s' +echo 'spec: %s' + """ % (spec.name, prefix, spec)) + + os.chmod(script, 0555) diff --git a/var/spack/repos/builtin.mock/packages/test-a/test-a-0.0.tar.gz b/var/spack/repos/builtin.mock/packages/test-a/test-a-0.0.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..3588e05765cd799dfe19e1f77e057bf7508ec47e GIT binary patch literal 112 zcmb2|=3uzVq!!M={Pvt8-ysJPmJ7D0<~95|Kkb{1f}-PrX$3W18BI)~SE|#F^vw8N zv-i!f&C_|0CdssyKPx$LK4$72|C!Dyr^A#M>z4NF`mcRfZGSIIclq(}ch}1?z`(y3 OtXZo*ykgK`U;qF>c`_gX literal 0 HcmV?d00001 diff --git a/var/spack/repos/builtin.mock/packages/test-b/package.py b/var/spack/repos/builtin.mock/packages/test-b/package.py new file mode 100644 index 0000000000..db64b0a556 --- /dev/null +++ b/var/spack/repos/builtin.mock/packages/test-b/package.py @@ -0,0 +1,31 @@ +from spack import * + +import os + +mydir = os.path.dirname(__file__) +source = os.path.join(mydir,'test-b-0.0.tar.gz') + +class TestB(Package): + """The test-b package""" + + url = 'file://'+source + + version('0.0', '4e823d0af4154fcf52b75dad41b7fd63') + + variant('nom', default=True, description='Nominal variant') + variant('var', default=False, description='Variant variant') + + depends_on('test-a') + + def install(self, spec, prefix): + bindir = os.path.join(prefix,'bin') + os.makedirs(bindir) + script = os.path.join(bindir, 'test-b') + with open(script,'w') as fp: + fp.write("""#!/bin/bash +echo 'name: %s' +echo 'prefix: %s' +echo 'spec: %s' + """ % (spec.name, prefix, spec)) + + os.chmod(script, 0555) diff --git a/var/spack/repos/builtin.mock/packages/test-b/test-b-0.0.tar.gz b/var/spack/repos/builtin.mock/packages/test-b/test-b-0.0.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..3588e05765cd799dfe19e1f77e057bf7508ec47e GIT binary patch literal 112 zcmb2|=3uzVq!!M={Pvt8-ysJPmJ7D0<~95|Kkb{1f}-PrX$3W18BI)~SE|#F^vw8N zv-i!f&C_|0CdssyKPx$LK4$72|C!Dyr^A#M>z4NF`mcRfZGSIIclq(}ch}1?z`(y3 OtXZo*ykgK`U;qF>c`_gX literal 0 HcmV?d00001 diff --git a/var/spack/repos/builtin.mock/packages/test-c/package.py b/var/spack/repos/builtin.mock/packages/test-c/package.py new file mode 100644 index 0000000000..0b03037466 --- /dev/null +++ b/var/spack/repos/builtin.mock/packages/test-c/package.py @@ -0,0 +1,31 @@ +from spack import * + +import os + +mydir = os.path.dirname(__file__) +source = os.path.join(mydir,'test-c-0.0.tar.gz') + +class TestC(Package): + """The test-c package""" + + url = 'file://'+source + + version('0.0', '4e823d0af4154fcf52b75dad41b7fd63') + + variant('nom', default=True, description='Nominal variant') + variant('var', default=False, description='Variant variant') + + depends_on('test-a+var',when='+var') + + def install(self, spec, prefix): + bindir = os.path.join(prefix,'bin') + os.makedirs(bindir) + script = os.path.join(bindir, 'test-c') + with open(script,'w') as fp: + fp.write("""#!/bin/bash +echo 'name: %s' +echo 'prefix: %s' +echo 'spec: %s' + """ % (spec.name, prefix, spec)) + + os.chmod(script, 0555) diff --git a/var/spack/repos/builtin.mock/packages/test-c/test-c-0.0.tar.gz b/var/spack/repos/builtin.mock/packages/test-c/test-c-0.0.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..3588e05765cd799dfe19e1f77e057bf7508ec47e GIT binary patch literal 112 zcmb2|=3uzVq!!M={Pvt8-ysJPmJ7D0<~95|Kkb{1f}-PrX$3W18BI)~SE|#F^vw8N zv-i!f&C_|0CdssyKPx$LK4$72|C!Dyr^A#M>z4NF`mcRfZGSIIclq(}ch}1?z`(y3 OtXZo*ykgK`U;qF>c`_gX literal 0 HcmV?d00001 diff --git a/var/spack/repos/builtin.mock/packages/test-d/package.py b/var/spack/repos/builtin.mock/packages/test-d/package.py new file mode 100644 index 0000000000..5cb7dcb2cb --- /dev/null +++ b/var/spack/repos/builtin.mock/packages/test-d/package.py @@ -0,0 +1,32 @@ +from spack import * + +import os + +mydir = os.path.dirname(__file__) +source = os.path.join(mydir,'test-d-0.0.tar.gz') + +class TestD(Package): + """The test-d package""" + + url = 'file://'+source + + version('0.0', '4e823d0af4154fcf52b75dad41b7fd63') + + variant('nom', default=True, description='Nominal variant') + variant('var', default=False, description='Variant variant') + + depends_on('test-b') + depends_on('test-c') + + def install(self, spec, prefix): + bindir = os.path.join(prefix,'bin') + os.makedirs(bindir) + script = os.path.join(bindir, 'test-d') + with open(script,'w') as fp: + fp.write("""#!/bin/bash +echo 'name: %s' +echo 'prefix: %s' +echo 'spec: %s' + """ % (spec.name, prefix, spec)) + + os.chmod(script, 0555) diff --git a/var/spack/repos/builtin.mock/packages/test-d/test-d-0.0.tar.gz b/var/spack/repos/builtin.mock/packages/test-d/test-d-0.0.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..3588e05765cd799dfe19e1f77e057bf7508ec47e GIT binary patch literal 112 zcmb2|=3uzVq!!M={Pvt8-ysJPmJ7D0<~95|Kkb{1f}-PrX$3W18BI)~SE|#F^vw8N zv-i!f&C_|0CdssyKPx$LK4$72|C!Dyr^A#M>z4NF`mcRfZGSIIclq(}ch}1?z`(y3 OtXZo*ykgK`U;qF>c`_gX literal 0 HcmV?d00001 From 90b7b7ba5c483054a3b6687f84306601d0e62d15 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 4 May 2016 21:42:59 -0700 Subject: [PATCH 215/481] Reworked compiler finding/storing/format to allow for multiple compilers with the same spec for different operating systems. TODO: fix config tests. All others up to date --- lib/spack/spack/architecture.py | 96 ++++++++- lib/spack/spack/build_environment.py | 5 +- lib/spack/spack/cmd/compiler.py | 4 +- lib/spack/spack/compiler.py | 166 ++++----------- lib/spack/spack/compilers/__init__.py | 224 ++++++++++----------- lib/spack/spack/concretize.py | 8 +- lib/spack/spack/config.py | 35 ++-- lib/spack/spack/operating_systems/cnl.py | 51 ++++- lib/spack/spack/test/architecture.py | 39 ++-- lib/spack/spack/test/config.py | 95 ++++++--- lib/spack/spack/test/mock_packages_test.py | 43 +++- lib/spack/spack/test/spec_dag.py | 2 +- 12 files changed, 451 insertions(+), 317 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index e2c189980a..301495f728 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -22,6 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +import os from collections import namedtuple import imp import platform as py_platform @@ -32,7 +33,10 @@ import llnl.util.tty as tty import spack +import spack.compilers from spack.util.naming import mod_to_class +from spack.util.environment import get_path +from spack.util.multiproc import parmap import spack.error as serr @@ -143,6 +147,7 @@ def operating_system(self, name): return self.operating_sys.get(name, None) + @classmethod def detect(self): """ Subclass is responsible for implementing this method. @@ -170,10 +175,9 @@ class OperatingSystem(object): find compilers we call find_compilers method for each operating system """ - def __init__(self, name, version, compiler_strategy="PATH"): + def __init__(self, name, version): self.name = name self.version = version - self.compiler_strategy = compiler_strategy def __str__(self): return self.name + self.version @@ -182,13 +186,96 @@ def __repr__(self): return self.__str__() def _cmp_key(self): - return (self.name, self.version, self.compiler_strategy) + return (self.name, self.version) + + + def find_compilers(self, *paths): + """ + Return a list of compilers found in the suppied paths. + This invokes the find() method for each Compiler class, + and appends the compilers detected to a list. + """ + if not paths: + paths = get_path('PATH') + # Make sure path elements exist, and include /bin directories + # under prefixes. + filtered_path = [] + for p in paths: + # Eliminate symlinks and just take the real directories. + p = os.path.realpath(p) + if not os.path.isdir(p): + continue + filtered_path.append(p) + + # Check for a bin directory, add it if it exists + bin = join_path(p, 'bin') + if os.path.isdir(bin): + filtered_path.append(os.path.realpath(bin)) + + # Once the paths are cleaned up, do a search for each type of + # compiler. We can spawn a bunch of parallel searches to reduce + # the overhead of spelunking all these directories. + types = spack.compilers.all_compiler_types() + compiler_lists = parmap(lambda cmp_cls: self.find_compiler(cmp_cls, *filtered_path), types) + + # ensure all the version calls we made are cached in the parent + # process, as well. This speeds up Spack a lot. + clist = reduce(lambda x,y: x+y, compiler_lists) + return clist + + def find_compiler(self, cmp_cls, *path): + """Try to find the given type of compiler in the user's + environment. For each set of compilers found, this returns + compiler objects with the cc, cxx, f77, fc paths and the + version filled in. + + This will search for compilers with the names in cc_names, + cxx_names, etc. and it will group them if they have common + prefixes, suffixes, and versions. e.g., gcc-mp-4.7 would + be grouped with g++-mp-4.7 and gfortran-mp-4.7. + """ + dicts = parmap( + lambda t: cmp_cls._find_matches_in_path(*t), + [(cmp_cls.cc_names, cmp_cls.cc_version) + tuple(path), + (cmp_cls.cxx_names, cmp_cls.cxx_version) + tuple(path), + (cmp_cls.f77_names, cmp_cls.f77_version) + tuple(path), + (cmp_cls.fc_names, cmp_cls.fc_version) + tuple(path)]) + + all_keys = set() + for d in dicts: + all_keys.update(d) + + compilers = {} + for k in all_keys: + ver, pre, suf = k + + # Skip compilers with unknown version. + if ver == 'unknown': + continue + + paths = tuple(pn[k] if k in pn else None for pn in dicts) + spec = spack.spec.CompilerSpec(cmp_cls.name, ver) + + if ver in compilers: + prev = compilers[ver] + + # prefer the one with more compilers. + prev_paths = [prev.cc, prev.cxx, prev.f77, prev.fc] + newcount = len([p for p in paths if p is not None]) + prevcount = len([p for p in prev_paths if p is not None]) + + # Don't add if it's not an improvement over prev compiler. + if newcount <= prevcount: + continue + + compilers[ver] = cmp_cls(spec, self, paths) + + return list(compilers.values()) def to_dict(self): d = {} d['name'] = self.name d['version'] = self.version - d['compiler_strategy'] = self.compiler_strategy return d @@ -261,7 +348,6 @@ def _operating_system_from_dict(os_dict): operating_system = OperatingSystem.__new__(OperatingSystem) operating_system.name = os_dict['name'] operating_system.version = os_dict['version'] - operating_system.compiler_strategy = os_dict['compiler_strategy'] return operating_system def arch_from_dict(d): diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 12a529d070..82d0792e26 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -169,9 +169,8 @@ def set_compiler_environment_variables(pkg): os.environ['SPACK_COMPILER_SPEC'] = str(pkg.spec.compiler) - if compiler.strategy == 'MODULES': - for mod in compiler.modules: - load_module(mod) + for mod in compiler.modules: + load_module(mod) def set_build_environment_variables(pkg): """This ensures a clean install environment when we build packages. diff --git a/lib/spack/spack/cmd/compiler.py b/lib/spack/spack/cmd/compiler.py index a7de61e1e5..e3d311b6b8 100644 --- a/lib/spack/spack/cmd/compiler.py +++ b/lib/spack/spack/cmd/compiler.py @@ -80,7 +80,7 @@ def compiler_add(args): if not paths: paths = get_path('PATH') - compilers = [c for c in spack.compilers.find_compilers(*args.add_paths) + compilers = [c for c in spack.compilers.find_compilers(*paths) if c.spec not in spack.compilers.all_compilers(scope=args.scope)] if compilers: @@ -125,6 +125,8 @@ def compiler_info(args): print "\tcxx = %s" % c.cxx print "\tf77 = %s" % c.f77 print "\tfc = %s" % c.fc + print "\tmodules = %s" % c.modules + print "\toperating system = %s" % c.operating_system def compiler_list(args): diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index c285c8a29d..8b973d8190 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -106,14 +106,14 @@ class Compiler(object): PrgEnv_compiler = None - def __init__(self, cspec, strategy, paths, modules=None): + def __init__(self, cspec, operating_system, paths, modules=[], alias=None): def check(exe): if exe is None: return None _verify_executables(exe) return exe - self.strategy = strategy + self.operating_system = operating_system self.cc = check(paths[0]) self.cxx = check(paths[1]) @@ -126,62 +126,33 @@ def check(exe): self.spec = cspec self.modules = modules - + self.alias = alias @property def version(self): return self.spec.version - # - # Compiler classes have methods for querying the version of - # specific compiler executables. This is used when discovering compilers. - # - # Compiler *instances* are just data objects, and can only be - # constructed from an actual set of executables. - # - - @classmethod - def default_version(cls, cc): - """Override just this to override all compiler version functions.""" - return dumpversion(cc) - - @classmethod - def cc_version(cls, cc): - return cls.default_version(cc) - - @classmethod - def cxx_version(cls, cxx): - return cls.default_version(cxx) - - @classmethod - def f77_version(cls, f77): - return cls.default_version(f77) - - @classmethod - def fc_version(cls, fc): - return cls.default_version(fc) - @classmethod def _find_matches_in_path(cls, compiler_names, detect_version, *path): """Finds compilers in the paths supplied. - - Looks for all combinations of ``compiler_names`` with the - ``prefixes`` and ``suffixes`` defined for this compiler - class. If any compilers match the compiler_names, - prefixes, or suffixes, uses ``detect_version`` to figure - out what version the compiler is. - - This returns a dict with compilers grouped by (prefix, - suffix, version) tuples. This can be further organized by - find(). - """ + + Looks for all combinations of ``compiler_names`` with the + ``prefixes`` and ``suffixes`` defined for this compiler + class. If any compilers match the compiler_names, + prefixes, or suffixes, uses ``detect_version`` to figure + out what version the compiler is. + + This returns a dict with compilers grouped by (prefix, + suffix, version) tuples. This can be further organized by + find(). + """ if not path: path = get_path('PATH') - + prefixes = [''] + cls.prefixes suffixes = [''] + cls.suffixes - + checks = [] for directory in path: if not (os.path.isdir(directory) and @@ -219,89 +190,34 @@ def check(key): successful = [key for key in parmap(check, checks) if key is not None] return dict(((v, p, s), path) for v, p, s, path in successful) - @classmethod - def find(cls, *path): - compilers = [] - platform = spack.architecture.sys_type() - strategies = [o.compiler_strategy for o in platform.operating_sys.values()] - if 'PATH' in strategies: - compilers.extend(cls.find_in_path(*path)) - if 'MODULES' in strategies: - compilers.extend(cls.find_in_modules()) - return compilers - + # + # Compiler classes have methods for querying the version of + # specific compiler executables. This is used when discovering compilers. + # + # Compiler *instances* are just data objects, and can only be + # constructed from an actual set of executables. + # @classmethod - def find_in_path(cls, *path): - """Try to find this type of compiler in the user's - environment. For each set of compilers found, this returns - compiler objects with the cc, cxx, f77, fc paths and the - version filled in. - - This will search for compilers with the names in cc_names, - cxx_names, etc. and it will group them if they have common - prefixes, suffixes, and versions. e.g., gcc-mp-4.7 would - be grouped with g++-mp-4.7 and gfortran-mp-4.7. - """ - dicts = parmap( - lambda t: cls._find_matches_in_path(*t), - [(cls.cc_names, cls.cc_version) + tuple(path), - (cls.cxx_names, cls.cxx_version) + tuple(path), - (cls.f77_names, cls.f77_version) + tuple(path), - (cls.fc_names, cls.fc_version) + tuple(path)]) - - all_keys = set() - for d in dicts: - all_keys.update(d) - - compilers = {} - for k in all_keys: - ver, pre, suf = k - - # Skip compilers with unknown version. - if ver == 'unknown': - continue - - paths = tuple(pn[k] if k in pn else None for pn in dicts) - spec = spack.spec.CompilerSpec(cls.name, ver) - - if ver in compilers: - prev = compilers[ver] - - # prefer the one with more compilers. - prev_paths = [prev.cc, prev.cxx, prev.f77, prev.fc] - newcount = len([p for p in paths if p is not None]) - prevcount = len([p for p in prev_paths if p is not None]) - - # Don't add if it's not an improvement over prev compiler. - if newcount <= prevcount: - continue - - compilers[ver] = cls(spec, 'PATH', paths) - - return list(compilers.values()) - + def default_version(cls, cc): + """Override just this to override all compiler version functions.""" + return dumpversion(cc) @classmethod - def find_in_modules(cls): - compilers = [] - if cls.PrgEnv: - if not cls.PrgEnv_compiler: - tty.die('Must supply PrgEnv_compiler with PrgEnv') + def cc_version(cls, cc): + return cls.default_version(cc) - modulecmd = which('modulecmd') - modulecmd.add_default_arg('python') - - output = modulecmd('avail', cls.PrgEnv_compiler, output=str, error=str) - matches = re.findall(r'(%s)/([\d\.]+[\d])' % cls.PrgEnv_compiler, output) - for name, version in matches: - v = version - comp = cls(spack.spec.CompilerSpec(name + '@' + v), 'MODULES', - ['cc', 'CC', 'ftn'], [cls.PrgEnv, name +'/' + v]) + @classmethod + def cxx_version(cls, cxx): + return cls.default_version(cxx) - compilers.append(comp) + @classmethod + def f77_version(cls, f77): + return cls.default_version(f77) - return compilers + @classmethod + def fc_version(cls, fc): + return cls.default_version(fc) def __repr__(self): @@ -311,12 +227,8 @@ def __repr__(self): def __str__(self): """Return a string represntation of the compiler toolchain.""" - if self.strategy is 'MODULES': - return "%s(%s)" % ( - self.name, '\n '.join((str(s) for s in (self.strategy, self.cc, self.cxx, self.f77, self.fc, self.modules)))) - else: - return "%s(%s)" % ( - self.name, '\n '.join((str(s) for s in (self.strategy, self.cc, self.cxx, self.f77, self.fc)))) + return "%s(%s)" % ( + self.name, '\n '.join((str(s) for s in (self.cc, self.cxx, self.f77, self.fc, self.modules, str(self.operating_system))))) class CompilerAccessError(spack.error.SpackError): diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index caf438e232..8ef6e976ba 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -29,6 +29,10 @@ import os import platform import copy +import hashlib +import base64 +import yaml +import sys from llnl.util.lang import memoized, list_modules from llnl.util.filesystem import join_path @@ -46,8 +50,8 @@ from spack.util.environment import get_path _imported_compilers_module = 'spack.compilers' -_required_instance_vars = ['cc', 'cxx', 'f77', 'fc'] -_optional_instance_vars = ['modules', 'strategy'] +_path_instance_vars = ['cc', 'cxx', 'f77', 'fc'] +_other_instance_vars = ['modules', 'operating_system'] _default_order = [] # TODO: customize order in config file @@ -67,93 +71,85 @@ def converter(cspec_like, *args, **kwargs): def _to_dict(compiler): """Return a dict version of compiler suitable to insert in YAML.""" + d = {} + d['spec'] = str(compiler.spec) + d['paths'] = dict( (attr, getattr(compiler, attr, None)) for attr in _path_instance_vars ) + d['operating_system'] = compiler.operating_system.to_dict() + d['modules'] = compiler.modules + + if not compiler.alias: + yaml_text = yaml.dump( + d, default_flow_style=True, width=sys.maxint) + sha = hashlib.sha1(yaml_text) + compiler.alias = base64.b32encode(sha.digest()).lower()[:8] return { - str(compiler.spec) : dict( - (attr, getattr(compiler, attr, None)) - for attr in _required_instance_vars + _optional_instance_vars) + compiler.alias: d } -def get_compiler_config(arch=None, scope=None): +def get_compiler_config(scope=None): """Return the compiler configuration for the specified architecture. """ - # Check whether we're on a front-end (native) architecture. - my_arch = spack.architecture.Arch() - if arch is None: - arch = my_arch - if isinstance(arch, basestring) and arch == 'all': - name = 'all' - else: - name = arch.platform.name - def init_compiler_config(): """Compiler search used when Spack has no compilers.""" - config[name] = {} - compilers = find_compilers(*get_path('PATH')) + config = {} + compilers = find_compilers() for compiler in compilers: - config[name].update(_to_dict(compiler)) + config.update(_to_dict(compiler)) spack.config.update_config('compilers', config, scope=scope) config = spack.config.get_config('compilers', scope=scope) # Update the configuration if there are currently no compilers # configured. Avoid updating automatically if there ARE site # compilers configured but no user ones. - if (isinstance(arch, basestring) or arch == my_arch) and arch not in config: - if scope is None: - # We know no compilers were configured in any scope. +# if (isinstance(arch, basestring) or arch == my_arch) and arch not in config: + if scope is None: + # We know no compilers were configured in any scope. + init_compiler_config() + elif scope == 'user': + # Check the site config and update the user config if + # nothing is configured at the site level. + site_config = spack.config.get_config('compilers', scope='site') + if not site_config: init_compiler_config() - elif scope == 'user': - # Check the site config and update the user config if - # nothing is configured at the site level. - site_config = spack.config.get_config('compilers', scope='site') - if not site_config: - init_compiler_config() - return config[name] if name in config else {} + return config -def add_compilers_to_config(compilers, arch=None, scope=None): +def add_compilers_to_config(compilers, scope=None): """Add compilers to the config for the specified architecture. Arguments: - compilers: a list of Compiler objects. - - arch: arch to add compilers for. - scope: configuration scope to modify. """ - if arch is None: - arch = spack.architecture.Arch() - - compiler_config = get_compiler_config(arch, scope) + compiler_config = get_compiler_config(scope) for compiler in compilers: - compiler_config[str(compiler.spec)] = dict( - (c, getattr(compiler, c, "None")) - for c in _required_instance_vars + _optional_instance_vars) + compiler_config = _to_dict(compiler) - update = { arch.platform.name : compiler_config } - spack.config.update_config('compilers', update, scope) + spack.config.update_config('compilers', compiler_config, scope) @_auto_compiler_spec -def remove_compiler_from_config(compiler_spec, arch=None, scope=None): +def remove_compiler_from_config(compiler_spec, scope=None): """Remove compilers from the config, by spec. Arguments: - compiler_specs: a list of CompilerSpec objects. - - arch: arch to add compilers for. - scope: configuration scope to modify. """ - if arch is None: - arch = spack.architecture.Arch() + compiler_config = get_compiler_config(scope) + matches = [(a,c) for (a,c) in compiler_config.items() if c['spec'] == compiler_spec] + if len(matches) == 1: + del compiler_config[matches[0][0]] + else: + CompilerSpecInsufficientlySpecificError(compiler_spec) - compiler_config = get_compiler_config(arch, scope) - del compiler_config[str(compiler_spec)] - update = { arch : compiler_config } - - spack.config.update_config('compilers', update, scope) + spack.config.update_config('compilers', compiler_config, scope) _cache_config_file = {} -def all_compilers_config(arch=None, scope=None): +def all_compilers_config(scope=None): """Return a set of specs for all the compiler versions currently available to build with. These are instances of CompilerSpec. """ @@ -161,20 +157,16 @@ def all_compilers_config(arch=None, scope=None): global _cache_config_file #Create a cache of the config file so we don't load all the time. if not _cache_config_file: - arch_config = get_compiler_config(arch, scope) - # Merge 'all' compilers with arch-specific ones. - # Arch-specific compilers have higher precedence. - _cache_config_file = get_compiler_config('all', scope=scope) - _cache_config_file = spack.config._merge_yaml(_cache_config_file, arch_config) + _cache_config_file = get_compiler_config(scope) return _cache_config_file else: return _cache_config_file -def all_compilers(arch=None, scope=None): +def all_compilers(scope=None): # Return compiler specs from the merged config. - return [spack.spec.CompilerSpec(s) - for s in all_compilers_config(arch, scope)] + return [spack.spec.CompilerSpec(s['spec']) + for s in all_compilers_config(scope).values()] def default_compiler(): @@ -189,37 +181,19 @@ def default_compiler(): return sorted(versions)[-1] -def find_compilers(*path): +def find_compilers(): """Return a list of compilers found in the suppied paths. - This invokes the find() method for each Compiler class, - and appends the compilers detected to a list. + This invokes the find_compilers() method for each operating + system associated with the host platform, and appends + the compilers detected to a list. """ - # Make sure path elements exist, and include /bin directories - # under prefixes. - filtered_path = [] - for p in path: - # Eliminate symlinks and just take the real directories. - p = os.path.realpath(p) - if not os.path.isdir(p): - continue - filtered_path.append(p) - - # Check for a bin directory, add it if it exists - bin = join_path(p, 'bin') - if os.path.isdir(bin): - filtered_path.append(os.path.realpath(bin)) - - # Once the paths are cleaned up, do a search for each type of - # compiler. We can spawn a bunch of parallel searches to reduce - # the overhead of spelunking all these directories. - types = all_compiler_types() - compiler_lists = parmap(lambda cls: cls.find(*filtered_path), types) - - # ensure all the version calls we made are cached in the parent - # process, as well. This speeds up Spack a lot. - clist = reduce(lambda x,y: x+y, compiler_lists) - return clist + # Find compilers for each operating system class + oss = all_os_classes() + compiler_lists = [] + for os in oss: + compiler_lists.extend(os.find_compilers()) + return compiler_lists def supported_compilers(): """Return a set of names of compilers supported by Spack. @@ -237,47 +211,60 @@ def supported(compiler_spec): @_auto_compiler_spec -def find(compiler_spec, arch=None, scope=None): +def find(compiler_spec, scope=None): """Return specs of available compilers that match the supplied compiler spec. Return an list if nothing found.""" - return [c for c in all_compilers(arch, scope) if c.satisfies(compiler_spec)] + return [c for c in all_compilers(scope) if c.satisfies(compiler_spec)] @_auto_compiler_spec -def compilers_for_spec(compiler_spec, arch=None, scope=None): +def compilers_for_spec(compiler_spec, scope=None): """This gets all compilers that satisfy the supplied CompilerSpec. Returns an empty list if none are found. """ - config = all_compilers_config(arch, scope) + config = all_compilers_config(scope) - def get_compiler(cspec): - items = config[str(cspec)] + def get_compilers(cspec): + compilers = [] - if not all(n in items for n in _required_instance_vars): - raise InvalidCompilerConfigurationError(cspec) + for aka, cmp in config.items(): + if cmp['spec'] != str(cspec): + continue + items = cmp + alias = aka + + if not ('paths' in items and all(n in items['paths'] for n in _path_instance_vars)): + raise InvalidCompilerConfigurationError(cspec) - cls = class_for_compiler_name(cspec.name) + cls = class_for_compiler_name(cspec.name) - strategy = items['strategy'] - if not strategy: - raise InvalidCompilerConfigurationError(cspec) + compiler_paths = [] + for c in _path_instance_vars: + compiler_path = items['paths'][c] + if compiler_path != "None": + compiler_paths.append(compiler_path) + else: + compiler_paths.append(None) - compiler_paths = [] - for c in _required_instance_vars: - compiler_path = items[c] - if compiler_path != "None": - compiler_paths.append(compiler_path) + mods = items.get('modules') + if mods == 'None': + mods = [] + + if 'operating_system' in items: + operating_system = spack.architecture._operating_system_from_dict( items['operating_system'] ) else: - compiler_paths.append(None) + operating_system = None - if 'modules' not in items: - items['modules'] = None - mods = items['modules'] + compilers.append(cls(cspec, operating_system, compiler_paths, mods, alias)) - return cls(cspec, strategy, compiler_paths, mods) + return compilers - matches = find(compiler_spec, arch, scope) - return [get_compiler(cspec) for cspec in matches] + matches = set(find(compiler_spec, scope)) + compilers = [] + for cspec in matches: + compilers.extend(get_compilers(cspec)) + return compilers +# return [get_compilers(cspec) for cspec in matches] @_auto_compiler_spec @@ -285,8 +272,9 @@ def compiler_for_spec(compiler_spec, operating_system): """Get the compiler that satisfies compiler_spec. compiler_spec must be concrete.""" assert(compiler_spec.concrete) + compilers = [c for c in compilers_for_spec(compiler_spec) - if c.strategy == operating_system.compiler_strategy] + if c.operating_system == operating_system] if len(compilers) < 1: raise NoCompilerForSpecError(compiler_spec, operating_system) if len(compilers) > 1: @@ -308,8 +296,20 @@ def class_for_compiler_name(compiler_name): return cls +def all_os_classes(): + """ + Return the list of classes for all operating systems available on + this platform + """ + classes = [] + + platform = spack.architecture.sys_type() + for os_class in platform.operating_sys.values(): + classes.append(os_class) + + return classes + def all_compiler_types(): -# return [class_for_compiler_name(c) for c in ['gcc']] return [class_for_compiler_name(c) for c in supported_compilers()] @@ -318,7 +318,7 @@ def __init__(self, compiler_spec): super(InvalidCompilerConfigurationError, self).__init__( "Invalid configuration for [compiler \"%s\"]: " % compiler_spec, "Compiler configuration must contain entries for all compilers: %s" - % _required_instance_vars) + % _path_instance_vars) class NoCompilersError(spack.error.SpackError): diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 3422e86ae8..58ea83ac4a 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -301,15 +301,11 @@ def concretize_compiler(self, spec): # Should think whether this can be more efficient def _proper_compiler_style(cspec, architecture): compilers = spack.compilers.compilers_for_spec(cspec) - filter(lambda c: c.strategy == architecture.platform_os.compiler_strategy, compilers) -#if architecture.platform_os.compiler_strategy == 'PATH': - # filter(lambda c: not c.modules, compilers) - #if architecture.platform_os.compiler_strategy == 'MODULES': - # filter(lambda c: c.modules, compilers) + filter(lambda c: c.operating_system == architecture.platform_os, compilers) return compilers - all_compilers = spack.compilers.all_compilers(spec.architecture) + all_compilers = spack.compilers.all_compilers() if (spec.compiler and spec.compiler.concrete and diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index f978ed12b0..49f2b90bfc 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -151,17 +151,16 @@ 'default': {}, 'additionalProperties': False, 'patternProperties': { - r'\w[\w-]*': { # architecture + r'\w[\w-]*': { # alias 'type': 'object', 'additionalProperties': False, - 'patternProperties': { - r'\w[\w-]*@\w[\w-]*': { # compiler spec + 'required': ['paths', 'spec', 'modules', 'operating_system'], + 'properties': { + 'paths': { 'type': 'object', - 'additionalProperties': False, 'required': ['cc', 'cxx', 'f77', 'fc'], - 'properties': { - 'strategy': { 'anyOf': [ {'type' : 'string' }, - {'type' : 'null' }]}, + 'additionalProperties': False, + 'properties': { 'cc': { 'anyOf': [ {'type' : 'string' }, {'type' : 'null' }]}, 'cxx': { 'anyOf': [ {'type' : 'string' }, @@ -169,13 +168,21 @@ 'f77': { 'anyOf': [ {'type' : 'string' }, {'type' : 'null' }]}, 'fc': { 'anyOf': [ {'type' : 'string' }, - {'type' : 'null' }]}, - 'modules': { 'anyOf': [ {'type' : 'string'}, - {'type' : 'null' }, - {'type': 'array'}, - ]} - },},},},},},},}, - + {'type' : 'null' }]}}}, + 'spec': { 'type': 'string'},#r'\w[\w-]*@\w[\w-]*' + 'operating_system': { + 'type': 'object', + 'required': ['name', 'version'], + 'additionalProperties': False, + 'properties': { + 'name': {'type': 'string'}, + 'version': {'type': 'string'} + }}, + 'modules': { 'anyOf': [ {'type' : 'string'}, + {'type' : 'null' }, + {'type': 'array'}, + ]} + },},},},},}, 'mirrors': { '$schema': 'http://json-schema.org/schema#', 'title': 'Spack mirror configuration file schema', diff --git a/lib/spack/spack/operating_systems/cnl.py b/lib/spack/spack/operating_systems/cnl.py index 9a0bf6c194..c160a60be8 100644 --- a/lib/spack/spack/operating_systems/cnl.py +++ b/lib/spack/spack/operating_systems/cnl.py @@ -1,4 +1,11 @@ +import re +import os + from spack.architecture import OperatingSystem +from spack.util.executable import * +import spack.spec +from spack.util.multiproc import parmap +import spack.compilers class Cnl(OperatingSystem): """ Compute Node Linux (CNL) is the operating system used for the Cray XC @@ -10,4 +17,46 @@ class Cnl(OperatingSystem): def __init__(self): name = 'CNL' version = '10' - super(Cnl, self).__init__(name, version, "MODULES") + super(Cnl, self).__init__(name, version) + + + def find_compilers(self, *paths): + types = spack.compilers.all_compiler_types() + compiler_lists = parmap(lambda cmp_cls: self.find_compiler(cmp_cls, *paths), types) + + # ensure all the version calls we made are cached in the parent + # process, as well. This speeds up Spack a lot. + clist = reduce(lambda x,y: x+y, compiler_lists) + return clist + + + def find_compiler(self, cmp_cls, *paths): + compilers = [] + if cmp_cls.PrgEnv: + if not cmp_cls.PrgEnv_compiler: + tty.die('Must supply PrgEnv_compiler with PrgEnv') + + modulecmd = which('modulecmd') + modulecmd.add_default_arg('python') + + # Save the environment variable to restore later + old_modulepath = os.environ['MODULEPATH'] + # if given any explicit paths, search them for module files too + if paths: + module_paths = ':' + ':'.join(p for p in paths) + os.environ['MODULEPATH'] = module_paths + + output = modulecmd('avail', cmp_cls.PrgEnv_compiler, output=str, error=str) + matches = re.findall(r'(%s)/([\d\.]+[\d])' % cmp_cls.PrgEnv_compiler, output) + for name, version in matches: + v = version + comp = cmp_cls(spack.spec.CompilerSpec(name + '@' + v), self, + ['cc', 'CC', 'ftn'], [cmp_cls.PrgEnv, name +'/' + v]) + + compilers.append(comp) + + # Restore modulepath environment variable + if paths: + os.environ['MODULEPATH'] = old_modulepath + + return compilers diff --git a/lib/spack/spack/test/architecture.py b/lib/spack/spack/test/architecture.py index 1ca3e58831..4cba2da9a5 100644 --- a/lib/spack/spack/test/architecture.py +++ b/lib/spack/spack/test/architecture.py @@ -12,8 +12,18 @@ from spack.platforms.bgq import Bgq from spack.platforms.darwin import Darwin -class ArchitectureTest(unittest.TestCase): +from spack.test.mock_packages_test import * +#class ArchitectureTest(unittest.TestCase): +class ArchitectureTest(MockPackagesTest): + + def setUp(self): + super(ArchitectureTest, self).setUp() + self.platform = sys_type() + + def tearDown(self): + super(ArchitectureTest, self).tearDown() + def test_dict_functions_for_architecture(self): arch = Arch() arch.platform_os = arch.platform.operating_system('default_os') @@ -34,13 +44,13 @@ def test_dict_functions_for_architecture(self): self.assertTrue( isinstance(new_arch.target, Target) ) - def test_platform_class_and_compiler_strategies(self): - a = CrayXc() - t = a.operating_system('default_os') - self.assertEquals(t.compiler_strategy, 'MODULES') - b = Linux() - s = b.operating_system('default_os') - self.assertEquals(s.compiler_strategy, 'PATH') +# def test_platform_class_and_compiler_strategies(self): +# a = CrayXc() +# t = a.operating_system('default_os') +# self.assertEquals(t.compiler_strategy, 'MODULES') +# b = Linux() +# s = b.operating_system('default_os') +# self.assertEquals(s.compiler_strategy, 'PATH') def test_sys_type(self): output_platform_class = sys_type() @@ -56,16 +66,13 @@ def test_sys_type(self): self.assertEqual(str(output_platform_class), str(my_platform_class)) - def setUp(self): - self.platform = sys_type() - def test_user_front_end_input(self): """Test when user inputs just frontend that both the frontend target and frontend operating system match """ frontend_os = self.platform.operating_system("frontend") frontend_target = self.platform.target("frontend") - frontend_spec = Spec("zlib=frontend") + frontend_spec = Spec("libelf=frontend") frontend_spec.concretize() self.assertEqual(frontend_os, frontend_spec.architecture.platform_os) self.assertEqual(frontend_target, frontend_spec.architecture.target) @@ -76,7 +83,7 @@ def test_user_back_end_input(self): """ backend_os = self.platform.operating_system("backend") backend_target = self.platform.target("backend") - backend_spec = Spec("zlib=backend") + backend_spec = Spec("libelf=backend") backend_spec.concretize() self.assertEqual(backend_os, backend_spec.architecture.platform_os) self.assertEqual(backend_target, backend_spec.architecture.target) @@ -85,7 +92,7 @@ def test_user_defaults(self): default_os = self.platform.operating_system("default_os") default_target = self.platform.target("default_target") - default_spec = Spec("zlib") # default is no args + default_spec = Spec("libelf") # default is no args default_spec.concretize() self.assertEqual(default_os, default_spec.architecture.platform_os) self.assertEqual(default_target, default_spec.architecture.target) @@ -103,11 +110,11 @@ def test_user_input_combination(self): for arch in combinations: o,t = arch arch_spec = "-".join(arch) - spec = Spec("zlib=%s" % arch_spec) + spec = Spec("libelf=%s" % arch_spec) spec.concretize() results.append(spec.architecture.platform_os == self.platform.operating_system(o)) results.append(spec.architecture.target == self.platform.target(t)) res = all(results) - print res + self.assertTrue(res) diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index 0562d2d620..2a8d642584 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -33,43 +33,91 @@ # Some sample compiler config data a_comps = { - "all": { - "gcc@4.7.3" : { + 'gcc473': { + 'paths': { "cc" : "/gcc473", "cxx": "/g++473", "f77": None, - "fc" : None }, - "gcc@4.5.0" : { + "fc" : None + }, + 'modules': None, + 'spec': 'gcc@4.7.3', + 'operating_system': { + 'name': 'CNL', + 'version': '10' + } + }, + 'gcc450': { + 'paths': { "cc" : "/gcc450", "cxx": "/g++450", - "f77": "/gfortran", - "fc" : "/gfortran" }, - "clang@3.3" : { + "f77": 'gfortran', + "fc" : 'gfortran' + }, + 'modules': None, + 'spec': 'gcc@4.5.0', + 'operating_system': { + 'name': 'CNL', + 'version': '10' + } + }, + 'clang33': { + 'paths': { "cc" : "", "cxx": "", - "f77": "", - "fc" : "" } - } + "f77": '', + "fc" : '' }, + 'modules': None, + 'spec': 'clang@3.3', + 'operating_system': { + 'name': 'CNL', + 'version': '10' + } + } } b_comps = { - "all": { - "icc@10.0" : { + 'icc100': { + 'paths': { "cc" : "/icc100", - "cxx": "/icc100", + "cxx": "/icp100", "f77": None, - "fc" : None }, - "icc@11.1" : { + "fc" : None + }, + 'modules': None, + 'spec': 'icc@10.0', + 'operating_system': { + 'name': 'CNL', + 'version': '10' + } + }, + 'icc111': { + 'paths': { "cc" : "/icc111", "cxx": "/icp111", - "f77": "/ifort", - "fc" : "/ifort" }, - "clang@3.3" : { - "cc" : "/clang", - "cxx": "/clang++", - "f77": None, - "fc" : None} - } + "f77": 'ifort', + "fc" : 'ifort' + }, + 'modules': None, + 'spec': 'icc@11.1', + 'operating_system': { + 'name': 'CNL', + 'version': '10' + } + }, + 'clang33': { + 'paths': { + "cc" : "", + "cxx": "", + "f77": '', + "fc" : '' }, + 'modules': None, + 'spec': 'clang@3.3', + 'operating_system': { + 'name': 'CNL', + 'version': '10' + } + } } class ConfigTest(MockPackagesTest): @@ -96,7 +144,6 @@ def check_config(self, comps, *compiler_names): actual = config['all'][key][c] self.assertEqual(expected, actual) - def test_write_key_in_memory(self): # Write b_comps "on top of" a_comps. spack.config.update_config('compilers', a_comps, 'test_low_priority') diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index 8f04076f19..612008ac58 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -36,21 +36,50 @@ mock_compiler_config = """\ compilers: - all: - clang@3.3: + clang3.3CNL: + spec: clang@3.3 + operating_system: + name: CNL + version: '10' + paths: cc: /path/to/clang cxx: /path/to/clang++ f77: None fc: None - strategy: PATH - modules: None - gcc@4.5.0: + modules: 'None' + clang3.3RHL: + spec: clang@3.3 + operating_system: + name: redhat + version: '6.7' + paths: + cc: /path/to/clang + cxx: /path/to/clang++ + f77: None + fc: None + modules: 'None' + gcc4.5.0CNL: + paths: cc: /path/to/gcc cxx: /path/to/g++ f77: /path/to/gfortran fc: /path/to/gfortran - strategy: PATH - modules: None + operating_system: + name: CNL + version: '10' + spec: gcc@4.5.0 + modules: 'None' + gcc4.5.0RHL: + paths: + cc: /path/to/gcc + cxx: /path/to/g++ + f77: /path/to/gfortran + fc: /path/to/gfortran + operating_system: + name: RHL + version: '6.7' + spec: gcc@4.5.0 + modules: 'None' """ mock_packages_config = """\ diff --git a/lib/spack/spack/test/spec_dag.py b/lib/spack/spack/test/spec_dag.py index eee65c5c09..4ed7de9c7d 100644 --- a/lib/spack/spack/test/spec_dag.py +++ b/lib/spack/spack/test/spec_dag.py @@ -243,7 +243,7 @@ def test_unsatisfiable_target(self): if len(platform.targets) > 1: first = platform.targets.values()[0].name second = platform.targets.values()[1].name - set_pkg_dep('mpileaks', 'mpich='+first) + self.set_pkg_dep('mpileaks', 'mpich='+first) spec = Spec('mpileaks ^mpich='+ second +' ^callpath ^dyninst ^libelf ^libdwarf') self.assertRaises(spack.spec.UnsatisfiableTargetSpecError, spec.normalize) From 2d1430da130e33fdd8d8345ce9b4cbe3dbdc2871 Mon Sep 17 00:00:00 2001 From: Brett Viren Date: Sat, 7 May 2016 18:24:24 -0400 Subject: [PATCH 216/481] Address all coments in @trws's latest comment in PR #869. I addressed them by factoring the code better to follow the visitor pattern. This will allow actions to be easily added in the future. These may not even be file sytsem views. One could add actions to generate shell init scripts, JSON DAG-dumpers, GraphViz DOT file generators, etc (yes, some of these are alread in there - just to give the idea). Also added is a top-level test $ source share/spack/setup-env.sh $ ./share/spack/examples/test_view.sh Read the top of that script first. --- lib/spack/spack/cmd/view.py | 318 +++++++++++------- share/spack/examples/test_view.sh | 31 ++ .../builtin.mock/packages/test-a/package.py | 1 + .../builtin.mock/packages/test-b/package.py | 1 + .../builtin.mock/packages/test-c/package.py | 1 + .../builtin.mock/packages/test-d/package.py | 1 + 6 files changed, 231 insertions(+), 122 deletions(-) create mode 100755 share/spack/examples/test_view.sh diff --git a/lib/spack/spack/cmd/view.py b/lib/spack/spack/cmd/view.py index 1bdaaecadc..31937b39a2 100644 --- a/lib/spack/spack/cmd/view.py +++ b/lib/spack/spack/cmd/view.py @@ -1,8 +1,3 @@ -''' -Produce a file-system "view" of a Spack DAG. - -Concept from Nix, implemented by brett.viren@gmail.com ca 2016. -''' ############################################################################## # Copyright (c) 2013, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory. @@ -27,6 +22,45 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +'''Produce a "view" of a Spack DAG. + +A "view" is the product of applying a function on a set of package specs. + +This set consists of: + +- specs resolved from the package names given by the user (the seeds) +- all depenencies of the seeds unless user specifies `--no-depenencies` +- less any specs with names matching the regular expressions given by `--exclude` + +The `view` command provides a number of functions (the "actions"): + +- symlink :: a file system view which is a directory hierarchy that is + the union of the hierarchies of the installed packages in the DAG + where installed files are referenced via symlinks. + +- hardlink :: like the symlink view but hardlinks are used + +- statlink :: a view producing a status report of a symlink or + hardlink view. + + +The file system view concept is imspired by Nix, implemented by +brett.viren@gmail.com ca 2016. + +''' +# Implementation notes: +# +# This is implemented as a visitor pattern on the set of package specs. +# +# The command line ACTION maps to a visitor_*() function which takes +# the set of package specs and any args which may be specific to the +# ACTION. +# +# To add a new view: +# 1. add a new cmd line args sub parser ACTION +# 2. add any action-specific options/arguments, most likely a list of specs. +# 3. add a visitor_MYACTION() function +# 4. add any visitor_MYALIAS assignments to match any command line aliases import os import re @@ -38,33 +72,48 @@ description = "Produce a single-rooted directory view of a spec." -def setup_parser(subparser): - setup_parser.parser = subparser +def setup_parser(sp): + setup_parser.parser = sp - sp = subparser.add_subparsers(metavar='ACTION', dest='action') + sp.add_argument('-v','--verbose', action='store_true', default=False, + help="Display verbose output.") + sp.add_argument('-e','--exclude', action='append', default=[], + help="Exclude packages with names matching the given regex pattern.") + sp.add_argument('-d', '--dependencies', choices=['true','false','yes','no'], + default='true', + help="Follow dependencies.") + + + ssp = sp.add_subparsers(metavar='ACTION', dest='action') # The action parameterizes the command but in keeping with Spack # patterns we make it a subcommand. - sps = [ - sp.add_parser('add', aliases=['link'], - help='Add packages to the view, create view if needed.'), - sp.add_parser('remove', aliases=['rm'], - help='Remove packages from the view, and view if empty.'), - sp.add_parser('status', aliases=['check'], - help='Check status of packages in the view.') + file_system_view_actions = [ + ssp.add_parser('symlink', aliases=['add','soft'], + help='Add package files to a filesystem view via symbolic links.'), + ssp.add_parser('hardlink', aliases=['hard'], + help='Add packages files to a filesystem via via hard links.'), + ssp.add_parser('remove', aliases=['rm'], + help='Remove packages from a filesystem view.'), + ssp.add_parser('statlink', aliases=['status','check'], + help='Check status of packages in a filesystem view.') ] - # All these options and arguments are common to every action. - for p in sps: - p.add_argument('-e','--exclude', action='append', default=[], - help="exclude any packages which the given re pattern") - p.add_argument('--no-dependencies', action='store_true', default=False, - help="just operate on named packages and do not follow dependencies") - p.add_argument('prefix', nargs=1, - help="Path to a top-level directory to receive the view.") - p.add_argument('specs', nargs=argparse.REMAINDER, - help="specs of packages to expose in the view.") + for act in file_system_view_actions: + act.add_argument('path', nargs=1, + help="Path to file system view directory.") + act.add_argument('specs', metavar='spec', nargs='+', + help="Seed specs of the packages to view.") + ## Other VIEW ACTIONS might be added here. + ## Some ideas are the following (and some are redundant with existing cmds) + ## A JSON view that dumps a DAG to a JSON file + ## A DOT view that dumps to a GraphViz file + ## A SHELL INIT view that dumps bash/csh script setting up to use packages in the view + return + + +### Util functions def assuredir(path): 'Assure path exists as a directory' @@ -79,7 +128,6 @@ def relative_to(prefix, path): reldir = reldir[1:] return reldir - def transform_path(spec, path, prefix=None): 'Return the a relative path corresponding to given path spec.prefix' if os.path.isabs(path): @@ -92,66 +140,6 @@ def transform_path(spec, path, prefix=None): path = os.path.join(prefix, path) return path -def action_status(spec, prefix): - 'Check status of view in prefix against spec' - dotspack = os.path.join(prefix, '.spack', spec.name) - if os.path.exists(os.path.join(dotspack)): - tty.info("Package added: %s"%spec.name) - return - tty.info("Package missing: %s"%spec.name) - return - -def action_remove(spec, prefix): - 'Remove any files found in spec from prefix and purge empty directories.' - - if not os.path.exists(prefix): - return - - dotspack = transform_path(spec, '.spack', prefix) - if not os.path.exists(dotspack): - tty.info("Skipping nonexistent package %s"%spec.name) - return - - tty.info("remove %s"%spec.name) - for dirpath,dirnames,filenames in os.walk(spec.prefix): - if not filenames: - continue - - targdir = transform_path(spec, dirpath, prefix) - for fname in filenames: - src = os.path.join(dirpath, fname) - dst = os.path.join(targdir, fname) - if not os.path.exists(dst): - #tty.warn("Skipping nonexistent file for view: %s" % dst) - continue - os.unlink(dst) - -def action_link(spec, prefix): - 'Symlink all files in `spec` into directory `prefix`.' - - dotspack = transform_path(spec, '.spack', prefix) - if os.path.exists(dotspack): - tty.warn("Skipping previously added package %s"%spec.name) - return - - tty.info("link %s" % spec.name) - for dirpath,dirnames,filenames in os.walk(spec.prefix): - if not filenames: - continue # avoid explicitly making empty dirs - - targdir = transform_path(spec, dirpath, prefix) - assuredir(targdir) - - for fname in filenames: - src = os.path.join(dirpath, fname) - dst = os.path.join(targdir, fname) - if os.path.exists(dst): - if '.spack' in dst.split(os.path.sep): - continue # silence these - tty.warn("Skipping existing file for view: %s" % dst) - continue - os.symlink(src,dst) - def purge_empty_directories(path): 'Ascend up from the leaves accessible from `path` and remove empty directories.' for dirpath, subdirs, files in os.walk(path, topdown=False): @@ -160,58 +148,144 @@ def purge_empty_directories(path): try: os.rmdir(sdp) except OSError: - #tty.warn("Not removing directory with contents: %s" % sdp) pass - - - -def view_action(action, parser, args): - 'The view command parameterized by the action.' - to_exclude = [re.compile(e) for e in args.exclude] +def filter_exclude(specs, exclude): + 'Filter specs given sequence of exclude regex' + to_exclude = [re.compile(e) for e in exclude] def exclude(spec): for e in to_exclude: if e.match(spec.name): return True return False + return [s for s in specs if not exclude(s)] - specs = spack.cmd.parse_specs(args.specs, normalize=True, concretize=True) - if not specs: - parser.print_help() - return 1 - - prefix = args.prefix[0] - assuredir(prefix) - +def flatten(seeds, descend=True): + 'Normalize and flattend seed specs and descend hiearchy' flat = set() - for spec in specs: - if args.no_dependencies: + for spec in seeds: + if not descend: flat.add(spec) continue flat.update(spec.normalized().traverse()) + return flat - for spec in flat: - if exclude(spec): - tty.info('Skipping excluded package: "%s"' % spec.name) + +### Action-specific helpers + +def check_one(spec, path, verbose=False): + 'Check status of view in path against spec' + dotspack = os.path.join(path, '.spack', spec.name) + if os.path.exists(os.path.join(dotspack)): + tty.info('Package in view: "%s"'%spec.name) + return + tty.info('Package not in view: "%s"'%spec.name) + return + +def remove_one(spec, path, verbose=False): + 'Remove any files found in `spec` from `path` and purge empty directories.' + + if not os.path.exists(path): + return # done, short circuit + + dotspack = transform_path(spec, '.spack', path) + if not os.path.exists(dotspack): + if verbose: + tty.info('Skipping nonexistent package: "%s"'%spec.name) + return + + if verbose: + tty.info('Removing package: "%s"'%spec.name) + for dirpath,dirnames,filenames in os.walk(spec.prefix): + if not filenames: continue - if not os.path.exists(spec.prefix): - tty.warn('Skipping unknown package: %s in %s' % (spec.name, spec.prefix)) - continue - action(spec, prefix) + targdir = transform_path(spec, dirpath, path) + for fname in filenames: + dst = os.path.join(targdir, fname) + if not os.path.exists(dst): + continue + os.unlink(dst) - if args.action in ['remove','rm']: - purge_empty_directories(prefix) +def link_one(spec, path, link = os.symlink, verbose=False): + 'Link all files in `spec` into directory `path`.' + dotspack = transform_path(spec, '.spack', path) + if os.path.exists(dotspack): + tty.warn('Skipping existing package: "%s"'%spec.name) + return + + if verbose: + tty.info('Linking package: "%s"' % spec.name) + for dirpath,dirnames,filenames in os.walk(spec.prefix): + if not filenames: + continue # avoid explicitly making empty dirs + + targdir = transform_path(spec, dirpath, path) + assuredir(targdir) + + for fname in filenames: + src = os.path.join(dirpath, fname) + dst = os.path.join(targdir, fname) + if os.path.exists(dst): + if '.spack' in dst.split(os.path.sep): + continue # silence these + tty.warn("Skipping existing file: %s" % dst) + continue + link(src,dst) + + +### The canonically named visitor_* functions and their alias maps. +### One for each action. + +def visitor_symlink(specs, args): + 'Symlink all files found in specs' + path = args.path[0] + assuredir(path) + for spec in specs: + link_one(spec, path, verbose=args.verbose) +visitor_add = visitor_symlink +visitor_soft = visitor_symlink + +def visitor_hardlink(specs, args): + 'Hardlink all files found in specs' + path = args.path[0] + assuredir(path) + for spec in specs: + link_one(spec, path, os.link, verbose=args.verbose) +visitor_hard = visitor_hardlink + +def visitor_remove(specs, args): + 'Remove all files and directories found in specs from args.path' + path = args.path[0] + for spec in specs: + remove_one(spec, path, verbose=args.verbose) + purge_empty_directories(path) +visitor_rm = visitor_remove + +def visitor_statlink(specs, args): + 'Give status of view in args.path relative to specs' + path = args.path[0] + for spec in specs: + check_one(spec, path, verbose=args.verbose) +visitor_status = visitor_statlink +visitor_check = visitor_statlink + + +# Finally, the actual "view" command. There should be no need to +# modify anything below when new actions are added. def view(parser, args): - 'The view command.' - action = { - 'add': action_link, - 'link': action_link, - 'remove': action_remove, - 'rm': action_remove, - 'status': action_status, - 'check': action_status - }[args.action] - view_action(action, parser, args) + 'Produce a view of a set of packages.' + + # Process common args + seeds = [spack.cmd.disambiguate_spec(s) for s in args.specs] + specs = flatten(seeds, args.dependencies.lower() in ['yes','true']) + specs = filter_exclude(specs, args.exclude) + + # Execute the visitation. + try: + visitor = globals()['visitor_' + args.action] + except KeyError: + tty.error('Unknown action: "%s"' % args.action) + visitor(specs, args) diff --git a/share/spack/examples/test_view.sh b/share/spack/examples/test_view.sh new file mode 100755 index 0000000000..b3e7102192 --- /dev/null +++ b/share/spack/examples/test_view.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# This will install a few bogus/test packages in order to test the +# `spack view` command. It assumes you have "spack" in your path. + +# It makes sub-directories in your CWD and installs and uninstalls +# Spack packages named test-*. + +set -x +set -e + +view="spack -m view -v" +for variant in +nom ~nom+var +nom+var +do + spack -m uninstall -f -a -y test-d + spack -m install test-d$variant + testdir=test_view + rm -rf $testdir + echo "hardlink may fail if Spack install area and CWD are not same FS" + for action in symlink hardlink + do + $view --dependencies=no $action $testdir test-d + $view -e test-a -e test-b $action $testdir test-d + $view $action $testdir test-d + $view status $testdir test-d + $view -d false remove $testdir test-a + $view remove $testdir test-d + rmdir $testdir # should not fail + done +done +echo "Warnings about skipping existing in the above are okay" diff --git a/var/spack/repos/builtin.mock/packages/test-a/package.py b/var/spack/repos/builtin.mock/packages/test-a/package.py index 2f72370580..ff45344b7a 100644 --- a/var/spack/repos/builtin.mock/packages/test-a/package.py +++ b/var/spack/repos/builtin.mock/packages/test-a/package.py @@ -9,6 +9,7 @@ class TestA(Package): """The test-a package""" url = 'file://'+source + homepage = "http://www.example.com/" version('0.0', '4e823d0af4154fcf52b75dad41b7fd63') diff --git a/var/spack/repos/builtin.mock/packages/test-b/package.py b/var/spack/repos/builtin.mock/packages/test-b/package.py index db64b0a556..f6d30bb85f 100644 --- a/var/spack/repos/builtin.mock/packages/test-b/package.py +++ b/var/spack/repos/builtin.mock/packages/test-b/package.py @@ -9,6 +9,7 @@ class TestB(Package): """The test-b package""" url = 'file://'+source + homepage = "http://www.example.com/" version('0.0', '4e823d0af4154fcf52b75dad41b7fd63') diff --git a/var/spack/repos/builtin.mock/packages/test-c/package.py b/var/spack/repos/builtin.mock/packages/test-c/package.py index 0b03037466..641e46ee50 100644 --- a/var/spack/repos/builtin.mock/packages/test-c/package.py +++ b/var/spack/repos/builtin.mock/packages/test-c/package.py @@ -9,6 +9,7 @@ class TestC(Package): """The test-c package""" url = 'file://'+source + homepage = "http://www.example.com/" version('0.0', '4e823d0af4154fcf52b75dad41b7fd63') diff --git a/var/spack/repos/builtin.mock/packages/test-d/package.py b/var/spack/repos/builtin.mock/packages/test-d/package.py index 5cb7dcb2cb..b46c680edd 100644 --- a/var/spack/repos/builtin.mock/packages/test-d/package.py +++ b/var/spack/repos/builtin.mock/packages/test-d/package.py @@ -9,6 +9,7 @@ class TestD(Package): """The test-d package""" url = 'file://'+source + homepage = "http://www.example.com/" version('0.0', '4e823d0af4154fcf52b75dad41b7fd63') From f1900f6a7b080a6b45392d356749e2a98de46398 Mon Sep 17 00:00:00 2001 From: Brett Viren Date: Sun, 8 May 2016 10:25:21 -0400 Subject: [PATCH 217/481] Add a 'print' view allowing genreation of arbitrary strings based on format using package/spec parameters. --- lib/spack/spack/cmd/view.py | 64 +++++++++++++++++++++++++++++++++++-- 1 file changed, 61 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/cmd/view.py b/lib/spack/spack/cmd/view.py index 31937b39a2..b504dfd2b1 100644 --- a/lib/spack/spack/cmd/view.py +++ b/lib/spack/spack/cmd/view.py @@ -38,11 +38,12 @@ the union of the hierarchies of the installed packages in the DAG where installed files are referenced via symlinks. -- hardlink :: like the symlink view but hardlinks are used +- hardlink :: like the symlink view but hardlinks are used. - statlink :: a view producing a status report of a symlink or hardlink view. +- format :: a view printing one string per spec following a given format. The file system view concept is imspired by Nix, implemented by brett.viren@gmail.com ca 2016. @@ -64,6 +65,7 @@ import os import re +import sys import argparse import spack @@ -86,6 +88,9 @@ def setup_parser(sp): ssp = sp.add_subparsers(metavar='ACTION', dest='action') + specs_opts = dict(metavar='spec', nargs='+', + help="Seed specs of the packages to view.") + # The action parameterizes the command but in keeping with Spack # patterns we make it a subcommand. file_system_view_actions = [ @@ -102,9 +107,15 @@ def setup_parser(sp): for act in file_system_view_actions: act.add_argument('path', nargs=1, help="Path to file system view directory.") - act.add_argument('specs', metavar='spec', nargs='+', - help="Seed specs of the packages to view.") + act.add_argument('specs', **specs_opts) + # The formatted print action. + act = ssp.add_parser('print', + help="Print a string to stdout based on given format") + act.add_argument('format', nargs=1, + help="Format describing per-package printout.") + act.add_argument('specs', **specs_opts) + ## Other VIEW ACTIONS might be added here. ## Some ideas are the following (and some are redundant with existing cmds) ## A JSON view that dumps a DAG to a JSON file @@ -170,6 +181,39 @@ def flatten(seeds, descend=True): flat.update(spec.normalized().traverse()) return flat +def spec2dict(spec): + 'Convert info in a spec into a simple dictionary.' + + # Expclitly convert instead of just returning spec.__dict__ as + # some things need processing or are properties. + # + pkg = spec.package + ret = dict(name = spec.name, + short = spec.short_spec, + cshort = spec.cshort_spec, # color + root = spec.root, + prefix = spec.prefix, + version = spec.version, + variants = spec.variants, + namespace = spec.namespace, + compiler = spec.compiler, + architecture = spec.architecture, + dependencies = ','.join(spec.dependencies.keys()), + dependents = ','.join(spec.dependents.keys()), + external = spec.external or "False", + hash = spec.dag_hash(), + + # package related: + url = pkg.url, + stage = pkg.stage.path, + installed = pkg.installed, + installed_dependents = ','.join([s.name for s in pkg.installed_dependents]), + build_log = pkg.build_log_path, + rpath = ':'.join(pkg.rpath), + + # ... + ) + return ret ### Action-specific helpers @@ -270,6 +314,20 @@ def visitor_statlink(specs, args): visitor_status = visitor_statlink visitor_check = visitor_statlink +def visitor_print(specs, args): + 'Print a string for each spec using args.format.' + fmt = args.format[0] + for spec in specs: + kwds = spec2dict(spec) + try: + string = fmt.format(**kwds) + except KeyError: + tty.error("Format error, use keywords: %s" % (', '.join(kwds.keys()), )) + raise + # argparser escapes these + string = string.replace(r'\n', '\n').replace(r'\t', '\t') + sys.stdout.write(string) + # Finally, the actual "view" command. There should be no need to # modify anything below when new actions are added. From 68c681bb788c86dc45c4df042f513fbf8a3e2c5f Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Thu, 12 May 2016 08:59:17 +0200 Subject: [PATCH 218/481] document blas and lapack usage in packages --- lib/spack/docs/packaging_guide.rst | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index 650e0ee3b2..f7f39d0d7b 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -1994,6 +1994,19 @@ instead of hard-coding ``join_path(self.spec['mpi'].prefix.bin, 'mpicc')`` for the reasons outlined above. +Blas and Lapack libraries +~~~~~~~~~~~~~~~~~~~~~~~~~ +Different packages provide implementation of ``Blas`` and ``Lapack`` routines. +The names of the resulting static and/or shared libraries differ from package +to package. In order to make ``install()`` method indifferent to the +choice of ``Blas`` implementation, each package which provides it +sets up ``self.spec.blas_shared_lib`` and ``self.spec.blas_static_lib `` to +point to the shared and static ``Blas`` libraries, respectively. The same +applies to packages which provide ``Lapack``. Package developers are advised to +use these variables, for example ``spec['blas'].blas_shared_lib`` instead of +hard-coding ``join_path(spec['blas'].prefix.lib, 'libopenblas.so')``. + + Forking ``install()`` ~~~~~~~~~~~~~~~~~~~~~ From ab41fd6692feb9cdaa29dedf81689e3ff04a5f26 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Thu, 12 May 2016 16:21:06 -0500 Subject: [PATCH 219/481] Add bertini package Also add missing dependency. Flex would not build for me without bison. It complained about not being able to find `yacc`. --- .../repos/builtin/packages/bertini/package.py | 49 +++++++++++++++++++ .../repos/builtin/packages/flex/package.py | 2 + 2 files changed, 51 insertions(+) create mode 100644 var/spack/repos/builtin/packages/bertini/package.py diff --git a/var/spack/repos/builtin/packages/bertini/package.py b/var/spack/repos/builtin/packages/bertini/package.py new file mode 100644 index 0000000000..67e4a945ff --- /dev/null +++ b/var/spack/repos/builtin/packages/bertini/package.py @@ -0,0 +1,49 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class Bertini(Package): + """Bertini is a general-purpose solver, written in C, that was created + for research about polynomial continuation. It solves for the numerical + solution of systems of polynomial equations using homotopy continuation.""" + + homepage = "https://bertini.nd.edu/" + url = "https://bertini.nd.edu/BertiniSource_v1.5.tar.gz" + + version('1.5', 'e3f6cc6e7f9a0cf1d73185e8671af707') + + variant('mpi', default=True, description='Compile in parallel') + + depends_on('flex') + depends_on('bison') + depends_on('gmp') + depends_on('mpfr') + depends_on('mpi', when='+mpi') + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix) + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/flex/package.py b/var/spack/repos/builtin/packages/flex/package.py index 926651010f..99ccf70024 100644 --- a/var/spack/repos/builtin/packages/flex/package.py +++ b/var/spack/repos/builtin/packages/flex/package.py @@ -33,6 +33,8 @@ class Flex(Package): version('2.6.0', '5724bcffed4ebe39e9b55a9be80859ec') version('2.5.39', 'e133e9ead8ec0a58d81166b461244fde') + depends_on('bison') + def install(self, spec, prefix): configure("--prefix=%s" % prefix) From 088dce709e85d0f522ac2d90c22618a20d421cbc Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Thu, 12 May 2016 16:33:37 -0500 Subject: [PATCH 220/481] Make flake8 happy --- var/spack/repos/builtin/packages/bertini/package.py | 1 + var/spack/repos/builtin/packages/flex/package.py | 1 + 2 files changed, 2 insertions(+) diff --git a/var/spack/repos/builtin/packages/bertini/package.py b/var/spack/repos/builtin/packages/bertini/package.py index 67e4a945ff..8d7da705e4 100644 --- a/var/spack/repos/builtin/packages/bertini/package.py +++ b/var/spack/repos/builtin/packages/bertini/package.py @@ -24,6 +24,7 @@ ############################################################################## from spack import * + class Bertini(Package): """Bertini is a general-purpose solver, written in C, that was created for research about polynomial continuation. It solves for the numerical diff --git a/var/spack/repos/builtin/packages/flex/package.py b/var/spack/repos/builtin/packages/flex/package.py index 99ccf70024..f41af061a5 100644 --- a/var/spack/repos/builtin/packages/flex/package.py +++ b/var/spack/repos/builtin/packages/flex/package.py @@ -24,6 +24,7 @@ ############################################################################## from spack import * + class Flex(Package): """Flex is a tool for generating scanners.""" From 4c9dd028d2305360329767831e65bafe403c6e8a Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Thu, 12 May 2016 17:20:48 -0700 Subject: [PATCH 221/481] Fixed error causing hang in env/cc --- lib/spack/env/cc | 46 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index 4a3e6eddc9..91916c4532 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -153,6 +153,29 @@ fi # Save original command for debug logging input_command="$@" +# +# Filter '.' and Spack environment directories out of PATH so that +# this script doesn't just call itself +# +IFS=':' read -ra env_path <<< "$PATH" +IFS=':' read -ra spack_env_dirs <<< "$SPACK_ENV_PATH" +spack_env_dirs+=(".") +PATH="" +for dir in "${env_path[@]}"; do + remove="" + for rm_dir in "${spack_env_dirs[@]}"; do + if [ "$dir" = "$rm_dir" ]; then remove=True; fi + done + if [ -z "$remove" ]; then + if [ -z "$PATH" ]; then + PATH="$dir" + else + PATH="$PATH:$dir" + fi + fi +done +export PATH + if [ "$mode" == vcheck ] ; then exec ${command} "$@" fi @@ -317,29 +340,6 @@ unset LD_LIBRARY_PATH unset LD_RUN_PATH unset DYLD_LIBRARY_PATH -# -# Filter '.' and Spack environment directories out of PATH so that -# this script doesn't just call itself -# -IFS=':' read -ra env_path <<< "$PATH" -IFS=':' read -ra spack_env_dirs <<< "$SPACK_ENV_PATH" -spack_env_dirs+=(".") -PATH="" -for dir in "${env_path[@]}"; do - remove="" - for rm_dir in "${spack_env_dirs[@]}"; do - if [ "$dir" = "$rm_dir" ]; then remove=True; fi - done - if [ -z "$remove" ]; then - if [ -z "$PATH" ]; then - PATH="$dir" - else - PATH="$PATH:$dir" - fi - fi -done -export PATH - full_command=("$command") full_command+=("${args[@]}") From b8a91db08948440100a2d4d6a0cad01f1eb9b33a Mon Sep 17 00:00:00 2001 From: "Gregory L. Lee" Date: Thu, 12 May 2016 20:04:45 -0700 Subject: [PATCH 222/481] Intel software packages and license enhancements --- lib/spack/spack/hooks/licensing.py | 5 +- lib/spack/spack/package.py | 10 +- .../repos/builtin/packages/daal/package.py | 27 ++++ .../packages/intel-parallelstudio/package.py | 112 +++++++++++++++++ .../repos/builtin/packages/intel/package.py | 117 ++++++++++++++++++ .../repos/builtin/packages/ipp/package.py | 25 ++++ .../repos/builtin/packages/mkl/package.py | 27 ++++ 7 files changed, 319 insertions(+), 4 deletions(-) create mode 100644 var/spack/repos/builtin/packages/daal/package.py create mode 100644 var/spack/repos/builtin/packages/intel-parallelstudio/package.py create mode 100644 var/spack/repos/builtin/packages/intel/package.py create mode 100644 var/spack/repos/builtin/packages/ipp/package.py create mode 100644 var/spack/repos/builtin/packages/mkl/package.py diff --git a/lib/spack/spack/hooks/licensing.py b/lib/spack/spack/hooks/licensing.py index 0f63b0e05a..f5d3ebd98c 100644 --- a/lib/spack/spack/hooks/licensing.py +++ b/lib/spack/spack/hooks/licensing.py @@ -26,7 +26,7 @@ import spack import llnl.util.tty as tty -from llnl.util.filesystem import join_path +from llnl.util.filesystem import join_path, mkdirp def pre_install(pkg): @@ -154,6 +154,9 @@ def symlink_license(pkg): target = pkg.global_license_file for filename in pkg.license_files: link_name = join_path(pkg.prefix, filename) + license_dir = os.path.split(link_name)[0] + if not os.path.exists(license_dir): + mkdirp(license_dir) if os.path.exists(target): os.symlink(target, link_name) tty.msg("Added local symlink %s to global license file" % diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 8167341127..cda6a4be18 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -397,14 +397,18 @@ def __init__(self, spec): if self.is_extension: spack.repo.get(self.extendee_spec)._check_extendable() + @property + def global_license_dir(self): + """Returns the path where a global license file should be stored.""" + spack_root = ancestor(__file__, 4) + return join_path(spack_root, 'etc', 'spack', 'licenses') + @property def global_license_file(self): """Returns the path where a global license file should be stored.""" if not self.license_files: return - spack_root = ancestor(__file__, 4) - global_license_dir = join_path(spack_root, 'etc', 'spack', 'licenses') - return join_path(global_license_dir, self.name, + return join_path(self.global_license_dir, self.name, os.path.basename(self.license_files[0])) @property diff --git a/var/spack/repos/builtin/packages/daal/package.py b/var/spack/repos/builtin/packages/daal/package.py new file mode 100644 index 0000000000..16ae048a4c --- /dev/null +++ b/var/spack/repos/builtin/packages/daal/package.py @@ -0,0 +1,27 @@ +from spack import * +import sys, os, re + +from spack.pkg.builtin.intel import IntelInstaller + +class Daal(IntelInstaller): + """Intel Data Analytics Accesleration Library. + + Note: You will have to add the download file to a + mirror so that Spack can find it. For instructions on how to set up a + mirror, see http://software.llnl.gov/spack/mirrors.html""" + + homepage = "https://software.intel.com/en-us/daal" + + version('2016.2.181', 'aad2aa70e5599ebfe6f85b29d8719d46', + url="file://%s/l_daal_2016.2.181.tgz" % os.getcwd()) + version('2016.3.210', 'ad747c0dd97dace4cad03cf2266cad28', + url="file://%s/l_daal_2016.3.210.tgz" % os.getcwd()) + + def install(self, spec, prefix): + + self.intel_prefix = os.path.join(prefix, "pkg") + IntelInstaller.install(self, spec, prefix) + + daal_dir = os.path.join(self.intel_prefix, "daal") + for f in os.listdir(daal_dir): + os.symlink(os.path.join(daal_dir, f), os.path.join(self.prefix, f)) diff --git a/var/spack/repos/builtin/packages/intel-parallelstudio/package.py b/var/spack/repos/builtin/packages/intel-parallelstudio/package.py new file mode 100644 index 0000000000..127a89654d --- /dev/null +++ b/var/spack/repos/builtin/packages/intel-parallelstudio/package.py @@ -0,0 +1,112 @@ +from spack import * +import sys, os, re + +from spack.pkg.builtin.intel import IntelInstaller, filter_pick + +class IntelParallelstudio(IntelInstaller): + """Intel Parallel Studio. + + Note: You will have to add the download file to a + mirror so that Spack can find it. For instructions on how to set up a + mirror, see http://software.llnl.gov/spack/mirrors.html""" + + homepage = "https://software.intel.com/en-us/intel-parallel-studio-xe" + + # TODO: can also try the online installer (will download files on demand) + version('composer.2016.2', '1133fb831312eb519f7da897fec223fa', + url="file://%s/parallel_studio_xe_2016_composer_edition_update2.tgz" % os.getcwd()) + version('professional.2016.2', '70be832f2d34c9bf596a5e99d5f2d832', + url="file://%s/parallel_studio_xe_2016_update2.tgz" % os.getcwd()) + version('cluster.2016.2', '70be832f2d34c9bf596a5e99d5f2d832', + url="file://%s/parallel_studio_xe_2016_update2.tgz" % os.getcwd()) + version('composer.2016.3', '3208eeabee951fc27579177b593cefe9', + url="file://%s/parallel_studio_xe_2016_composer_edition_update3.tgz" % os.getcwd()) + version('professional.2016.3', 'eda19bb0d0d19709197ede58f13443f3', + url="file://%s/parallel_studio_xe_2016_update3.tgz" % os.getcwd()) + version('cluster.2016.3', 'eda19bb0d0d19709197ede58f13443f3', + url="file://%s/parallel_studio_xe_2016_update3.tgz" % os.getcwd()) + + variant('rpath', default=True, description="Add rpath to .cfg files") + variant('all', default=False, description="Install all files associated with the requested edition") + variant('mpi', default=True, description="Install the Intel MPI library and ITAC tool") + variant('mkl', default=True, description="Install the Intel MKL library") + variant('daal', default=True, description="Install the Intel DAAL libraries") + variant('ipp', default=True, description="Install the Intel IPP libraries") + variant('tools', default=True, description="Install the Intel Advisor, VTune Amplifier, and Inspector tools") + + provides('mpi', when='@cluster:+mpi') + provides('mkl', when='+mkl') + provides('daal', when='+daal') + provides('ipp', when='+ipp') + + def install(self, spec, prefix): + + # remove the installation DB, otherwise it will try to install into location of other Intel builds + try: + os.remove(os.path.join(os.environ["HOME"], "intel", "intel_sdp_products.db")) + except OSError: + pass # if the file does not exist + + base_components = "ALL" # when in doubt, install everything + mpi_components = "" + mkl_components = "" + daal_components = "" + ipp_components = "" + tools_components = "" + + if spec.satisfies('+all'): + base_components = "ALL" + else: + with open("pset/mediaconfig.xml", "r") as f: + lines = f.readlines() + all_components = [] + for line in lines: + if line.find('') != -1: + component = line[line.find('') + 6:line.find('')] + all_components.append(component) + base_components = filter_pick(all_components, re.compile('(comp|openmp|intel-tbb|icc|ifort|psxe|icsxe-pset)').search) + mpi_components = filter_pick(all_components, re.compile('(icsxe|imb|mpi|itac|intel-tc|clck)').search) + mkl_components = filter_pick(all_components, re.compile('(mkl)').search) + daal_components = filter_pick(all_components, re.compile('(daal)').search) + ipp_components = filter_pick(all_components, re.compile('(ipp)').search) + tool_components = filter_pick(all_components, re.compile('(gdb|vtune|inspector|advisor)').search) + + components = base_components + if not spec.satisfies('+all'): + if spec.satisfies('+mpi') and 'cluster' in str(spec.version): + components += mpi_components + if spec.satisfies('+mkl'): + components += mkl_components + if spec.satisfies('+daal'): + components += daal_components + if spec.satisfies('+ipp'): + components += ipp_components + if spec.satisfies('+tools') and (spec.satisfies('@cluster') or spec.satisfies('@professional')): + components += tool_components + + self.intel_components = ';'.join(components) + IntelInstaller.install(self, spec, prefix) + + absbindir = os.path.split(os.path.realpath(os.path.join(self.prefix.bin, "icc")))[0] + abslibdir = os.path.split(os.path.realpath(os.path.join(self.prefix.lib, "intel64", "libimf.a")))[0] + + # symlink or copy? + os.symlink(self.global_license_file, os.path.join(absbindir, "license.lic")) + if spec.satisfies('+tools') and (spec.satisfies('@cluster') or spec.satisfies('@professional')): + os.mkdir(os.path.join(self.prefix, "inspector_xe/licenses")) + os.symlink(self.global_license_file, os.path.join(self.prefix, "inspector_xe/licenses", "license.lic")) + os.mkdir(os.path.join(self.prefix, "advisor_xe/licenses")) + os.symlink(self.global_license_file, os.path.join(self.prefix, "advisor_xe/licenses", "license.lic")) + os.mkdir(os.path.join(self.prefix, "vtune_amplifier_xe/licenses")) + os.symlink(self.global_license_file, os.path.join(self.prefix, "vtune_amplifier_xe/licenses", "license.lic")) + + if (spec.satisfies('+all') or spec.satisfies('+mpi')) and spec.satisfies('@cluster'): + os.symlink(self.global_license_file, os.path.join(self.prefix, "itac_latest", "license.lic")) + + if spec.satisfies('+rpath'): + for compiler_command in ["icc", "icpc", "ifort"]: + cfgfilename = os.path.join(absbindir, "%s.cfg" %(compiler_command)) + with open(cfgfilename, "w") as f: + f.write('-Xlinker -rpath -Xlinker %s\n' %(abslibdir)) + + os.symlink(os.path.join(self.prefix.man, "common", "man1"), os.path.join(self.prefix.man, "man1")) diff --git a/var/spack/repos/builtin/packages/intel/package.py b/var/spack/repos/builtin/packages/intel/package.py new file mode 100644 index 0000000000..e51cc773af --- /dev/null +++ b/var/spack/repos/builtin/packages/intel/package.py @@ -0,0 +1,117 @@ +from spack import * +import sys, os, re + +def filter_pick(input_list, regex_filter): + return [l for l in input_list for m in (regex_filter(l),) if m] + +def unfilter_pick(input_list, regex_filter): + return [l for l in input_list for m in (regex_filter(l),) if not m] + +def get_all_components(): + all_components = [] + with open("pset/mediaconfig.xml", "r") as f: + lines = f.readlines() + for line in lines: + if line.find('') != -1: + component = line[line.find('') + 6:line.find('')] + all_components.append(component) + return all_components + +class IntelInstaller(Package): + """Base package containing common methods for installing Intel software""" + + homepage = "https://software.intel.com/en-us" + intel_components = "ALL" + license_required = True + license_comment = '#' + license_files = ['Licenses/license.lic'] + license_vars = ['INTEL_LICENSE_FILE'] + license_url = 'https://software.intel.com/en-us/articles/intel-license-manager-faq' + + @property + def global_license_file(self): + """Returns the path where a global license file should be stored.""" + if not self.license_files: + return + return join_path(self.global_license_dir, "intel", + os.path.basename(self.license_files[0])) + + def install(self, spec, prefix): + + # remove the installation DB, otherwise it will try to install into location of other Intel builds + try: + os.remove(os.path.join(os.environ["HOME"], "intel", "intel_sdp_products.db")) + except OSError: + pass # if the file does not exist + + if not hasattr(self, "intel_prefix"): + self.intel_prefix = self.prefix + + silent_config_filename = 'silent.cfg' + with open(silent_config_filename, 'w') as f: + f.write(""" +ACCEPT_EULA=accept +PSET_MODE=install +CONTINUE_WITH_INSTALLDIR_OVERWRITE=yes +PSET_INSTALL_DIR=%s +ACTIVATION_LICENSE_FILE=%s +ACTIVATION_TYPE=license_file +PHONEHOME_SEND_USAGE_DATA=no +COMPONENTS=%s +""" %(self.intel_prefix, self.global_license_file, self.intel_components)) + + install_script = which("install.sh") + install_script('--silent', silent_config_filename) + + +class Intel(IntelInstaller): + """Intel Compilers. + + Note: You will have to add the download file to a + mirror so that Spack can find it. For instructions on how to set up a + mirror, see http://software.llnl.gov/spack/mirrors.html""" + + homepage = "https://software.intel.com/en-us/intel-parallel-studio-xe" + + # TODO: can also try the online installer (will download files on demand) + version('16.0.2', '1133fb831312eb519f7da897fec223fa', + url="file://%s/parallel_studio_xe_2016_composer_edition_update2.tgz" % os.getcwd()) + version('16.0.3', '3208eeabee951fc27579177b593cefe9', + url="file://%s/parallel_studio_xe_2016_composer_edition_update3.tgz" % os.getcwd()) + + variant('rpath', default=True, description="Add rpath to .cfg files") + + def install(self, spec, prefix): + + # remove the installation DB, otherwise it will try to install into location of other Intel builds + try: + os.remove(os.path.join(os.environ["HOME"], "intel", "intel_sdp_products.db")) + except OSError: + pass # if the file does not exist + + components = [] + with open("pset/mediaconfig.xml", "r") as f: + lines = f.readlines() + all_components = [] + for line in lines: + if line.find('') != -1: + component = line[line.find('') + 6:line.find('')] + all_components.append(component) + components = filter_pick(all_components, re.compile('(comp|openmp|intel-tbb|icc|ifort|psxe|icsxe-pset)').search) + + self.intel_components = ';'.join(components) + IntelInstaller.install(self, spec, prefix) + + absbindir = os.path.split(os.path.realpath(os.path.join(self.prefix.bin, "icc")))[0] + abslibdir = os.path.split(os.path.realpath(os.path.join(self.prefix.lib, "intel64", "libimf.a")))[0] + + # symlink or copy? + os.symlink(self.global_license_file, os.path.join(absbindir, "license.lic")) + + if spec.satisfies('+rpath'): + for compiler_command in ["icc", "icpc", "ifort"]: + cfgfilename = os.path.join(absbindir, "%s.cfg" %(compiler_command)) + with open(cfgfilename, "w") as f: + f.write('-Xlinker -rpath -Xlinker %s\n' %(abslibdir)) + + os.symlink(os.path.join(self.prefix.man, "common", "man1"), os.path.join(self.prefix.man, "man1")) diff --git a/var/spack/repos/builtin/packages/ipp/package.py b/var/spack/repos/builtin/packages/ipp/package.py new file mode 100644 index 0000000000..23c944c2d5 --- /dev/null +++ b/var/spack/repos/builtin/packages/ipp/package.py @@ -0,0 +1,25 @@ +from spack import * +import sys, os, re + +from spack.pkg.builtin.intel import IntelInstaller + +class Ipp(IntelInstaller): + """Intel Integrated Performance Primitives. + + Note: You will have to add the download file to a + mirror so that Spack can find it. For instructions on how to set up a + mirror, see http://software.llnl.gov/spack/mirrors.html""" + + homepage = "https://software.intel.com/en-us/intel-ipp" + + version('9.0.3.210', '0e1520dd3de7f811a6ef6ebc7aa429a3', + url="file://%s/l_ipp_9.0.3.210.tgz" % os.getcwd()) + + def install(self, spec, prefix): + + self.intel_prefix = os.path.join(prefix, "pkg") + IntelInstaller.install(self, spec, prefix) + + ipp_dir = os.path.join(self.intel_prefix, "ipp") + for f in os.listdir(ipp_dir): + os.symlink(os.path.join(ipp_dir, f), os.path.join(self.prefix, f)) diff --git a/var/spack/repos/builtin/packages/mkl/package.py b/var/spack/repos/builtin/packages/mkl/package.py new file mode 100644 index 0000000000..1d4553b7f5 --- /dev/null +++ b/var/spack/repos/builtin/packages/mkl/package.py @@ -0,0 +1,27 @@ +from spack import * +import os, re, sys + +from spack.pkg.builtin.intel import IntelInstaller + +class Mkl(IntelInstaller): + """Intel Math Kernel Library. + + Note: You will have to add the download file to a + mirror so that Spack can find it. For instructions on how to set up a + mirror, see http://software.llnl.gov/spack/mirrors.html""" + + homepage = "https://software.intel.com/en-us/intel-mkl" + + version('11.3.2.181', '536dbd82896d6facc16de8f961d17d65', + url="file://%s/l_mkl_11.3.2.181.tgz" % os.getcwd()) + version('11.3.3.210', 'f72546df27f5ebb0941b5d21fd804e34', + url="file://%s/l_mkl_11.3.3.210.tgz" % os.getcwd()) + + def install(self, spec, prefix): + + self.intel_prefix = os.path.join(prefix, "pkg") + IntelInstaller.install(self, spec, prefix) + + mkl_dir = os.path.join(self.intel_prefix, "mkl") + for f in os.listdir(mkl_dir): + os.symlink(os.path.join(mkl_dir, f), os.path.join(self.prefix, f)) From f43cab0951e5023736d3e332d747e0db212acb72 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 13 May 2016 14:39:20 -0700 Subject: [PATCH 223/481] fixed pkgconfig issue --- lib/spack/spack/build_environment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 82d0792e26..8016bc4151 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -232,7 +232,7 @@ def add_env_path(path): pcdir = join_path(p, libdir, 'pkgconfig') if os.path.isdir(pcdir): pkg_config_dirs.append(pcdir) - path_set("PKG_CONFIG_PATH", pkg_config_dirs) + path_put_first("PKG_CONFIG_PATH", pkg_config_dirs) if pkg.spec.architecture.target.module_name: load_module(pkg.spec.architecture.target.module_name) From fce6ecb05c2aa8f025c27f4245994efef2987b38 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 13 May 2016 15:04:06 -0700 Subject: [PATCH 224/481] Changed _proper_compiler_spec to return filtered list. Apparently filter returns a list rather than filtering the original list. Lead to compiler finding problems if user specified frontend compilers --- lib/spack/spack/concretize.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 58ea83ac4a..9ad5271542 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -301,8 +301,9 @@ def concretize_compiler(self, spec): # Should think whether this can be more efficient def _proper_compiler_style(cspec, architecture): compilers = spack.compilers.compilers_for_spec(cspec) - filter(lambda c: c.operating_system == architecture.platform_os, compilers) - return compilers + return filter(lambda c: c.operating_system == + architecture.platform_os, compilers) + #return compilers all_compilers = spack.compilers.all_compilers() From e9b71872a8d590d342e4cfcc78ff9855cff7fc24 Mon Sep 17 00:00:00 2001 From: "Gregory L. Lee" Date: Fri, 13 May 2016 15:16:07 -0700 Subject: [PATCH 225/481] addressed PR#946 comments from @adamjstewart --- lib/spack/spack/hooks/licensing.py | 2 +- lib/spack/spack/package.py | 6 ++-- .../repos/builtin/packages/daal/package.py | 2 +- .../package.py | 36 +++++++------------ .../repos/builtin/packages/intel/package.py | 17 ++++----- 5 files changed, 24 insertions(+), 39 deletions(-) rename var/spack/repos/builtin/packages/{intel-parallelstudio => intel-parallel-studio}/package.py (73%) diff --git a/lib/spack/spack/hooks/licensing.py b/lib/spack/spack/hooks/licensing.py index f5d3ebd98c..9010b84154 100644 --- a/lib/spack/spack/hooks/licensing.py +++ b/lib/spack/spack/hooks/licensing.py @@ -154,7 +154,7 @@ def symlink_license(pkg): target = pkg.global_license_file for filename in pkg.license_files: link_name = join_path(pkg.prefix, filename) - license_dir = os.path.split(link_name)[0] + license_dir = os.path.dirname(link_name) if not os.path.exists(license_dir): mkdirp(license_dir) if os.path.exists(target): diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index cda6a4be18..fa3ece47f7 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -399,13 +399,15 @@ def __init__(self, spec): @property def global_license_dir(self): - """Returns the path where a global license file should be stored.""" + """Returns the directory where global license files for all + packages are stored.""" spack_root = ancestor(__file__, 4) return join_path(spack_root, 'etc', 'spack', 'licenses') @property def global_license_file(self): - """Returns the path where a global license file should be stored.""" + """Returns the path where a global license file for this + particular package should be stored.""" if not self.license_files: return return join_path(self.global_license_dir, self.name, diff --git a/var/spack/repos/builtin/packages/daal/package.py b/var/spack/repos/builtin/packages/daal/package.py index 16ae048a4c..9fdbd7bf5e 100644 --- a/var/spack/repos/builtin/packages/daal/package.py +++ b/var/spack/repos/builtin/packages/daal/package.py @@ -4,7 +4,7 @@ from spack.pkg.builtin.intel import IntelInstaller class Daal(IntelInstaller): - """Intel Data Analytics Accesleration Library. + """Intel Data Analytics Acceleration Library. Note: You will have to add the download file to a mirror so that Spack can find it. For instructions on how to set up a diff --git a/var/spack/repos/builtin/packages/intel-parallelstudio/package.py b/var/spack/repos/builtin/packages/intel-parallel-studio/package.py similarity index 73% rename from var/spack/repos/builtin/packages/intel-parallelstudio/package.py rename to var/spack/repos/builtin/packages/intel-parallel-studio/package.py index 127a89654d..3a44cb32a3 100644 --- a/var/spack/repos/builtin/packages/intel-parallelstudio/package.py +++ b/var/spack/repos/builtin/packages/intel-parallel-studio/package.py @@ -1,9 +1,9 @@ from spack import * import sys, os, re -from spack.pkg.builtin.intel import IntelInstaller, filter_pick +from spack.pkg.builtin.intel import IntelInstaller, filter_pick, get_all_components -class IntelParallelstudio(IntelInstaller): +class IntelParallelStudio(IntelInstaller): """Intel Parallel Studio. Note: You will have to add the download file to a @@ -41,12 +41,6 @@ class IntelParallelstudio(IntelInstaller): def install(self, spec, prefix): - # remove the installation DB, otherwise it will try to install into location of other Intel builds - try: - os.remove(os.path.join(os.environ["HOME"], "intel", "intel_sdp_products.db")) - except OSError: - pass # if the file does not exist - base_components = "ALL" # when in doubt, install everything mpi_components = "" mkl_components = "" @@ -57,19 +51,13 @@ def install(self, spec, prefix): if spec.satisfies('+all'): base_components = "ALL" else: - with open("pset/mediaconfig.xml", "r") as f: - lines = f.readlines() - all_components = [] - for line in lines: - if line.find('') != -1: - component = line[line.find('') + 6:line.find('')] - all_components.append(component) - base_components = filter_pick(all_components, re.compile('(comp|openmp|intel-tbb|icc|ifort|psxe|icsxe-pset)').search) - mpi_components = filter_pick(all_components, re.compile('(icsxe|imb|mpi|itac|intel-tc|clck)').search) - mkl_components = filter_pick(all_components, re.compile('(mkl)').search) - daal_components = filter_pick(all_components, re.compile('(daal)').search) - ipp_components = filter_pick(all_components, re.compile('(ipp)').search) - tool_components = filter_pick(all_components, re.compile('(gdb|vtune|inspector|advisor)').search) + all_components = get_all_components() + base_components = filter_pick(all_components, re.compile('(comp|openmp|intel-tbb|icc|ifort|psxe|icsxe-pset)').search) + mpi_components = filter_pick(all_components, re.compile('(icsxe|imb|mpi|itac|intel-tc|clck)').search) + mkl_components = filter_pick(all_components, re.compile('(mkl)').search) + daal_components = filter_pick(all_components, re.compile('(daal)').search) + ipp_components = filter_pick(all_components, re.compile('(ipp)').search) + tool_components = filter_pick(all_components, re.compile('(gdb|vtune|inspector|advisor)').search) components = base_components if not spec.satisfies('+all'): @@ -87,10 +75,10 @@ def install(self, spec, prefix): self.intel_components = ';'.join(components) IntelInstaller.install(self, spec, prefix) - absbindir = os.path.split(os.path.realpath(os.path.join(self.prefix.bin, "icc")))[0] - abslibdir = os.path.split(os.path.realpath(os.path.join(self.prefix.lib, "intel64", "libimf.a")))[0] + absbindir = os.path.dirname(os.path.realpath(os.path.join(self.prefix.bin, "icc"))) + abslibdir = os.path.dirname(os.path.realpath(os.path.join(self.prefix.lib, "intel64", "libimf.a"))) - # symlink or copy? + relbindir = absbindir.strip(os.path.commonprefix([self.prefix, absbindir])) os.symlink(self.global_license_file, os.path.join(absbindir, "license.lic")) if spec.satisfies('+tools') and (spec.satisfies('@cluster') or spec.satisfies('@professional')): os.mkdir(os.path.join(self.prefix, "inspector_xe/licenses")) diff --git a/var/spack/repos/builtin/packages/intel/package.py b/var/spack/repos/builtin/packages/intel/package.py index e51cc773af..a547ab56db 100644 --- a/var/spack/repos/builtin/packages/intel/package.py +++ b/var/spack/repos/builtin/packages/intel/package.py @@ -2,12 +2,15 @@ import sys, os, re def filter_pick(input_list, regex_filter): + """Returns the items in input_list that are found in the regex_filter""" return [l for l in input_list for m in (regex_filter(l),) if m] def unfilter_pick(input_list, regex_filter): + """Returns the items in input_list that are not found in the regex_filter""" return [l for l in input_list for m in (regex_filter(l),) if not m] def get_all_components(): + """Returns a list of all the components associated with the downloaded Intel package""" all_components = [] with open("pset/mediaconfig.xml", "r") as f: lines = f.readlines() @@ -39,10 +42,8 @@ def global_license_file(self): def install(self, spec, prefix): # remove the installation DB, otherwise it will try to install into location of other Intel builds - try: + if os.path.exists(os.path.join(os.environ["HOME"], "intel", "intel_sdp_products.db")): os.remove(os.path.join(os.environ["HOME"], "intel", "intel_sdp_products.db")) - except OSError: - pass # if the file does not exist if not hasattr(self, "intel_prefix"): self.intel_prefix = self.prefix @@ -90,14 +91,8 @@ def install(self, spec, prefix): pass # if the file does not exist components = [] - with open("pset/mediaconfig.xml", "r") as f: - lines = f.readlines() - all_components = [] - for line in lines: - if line.find('') != -1: - component = line[line.find('') + 6:line.find('')] - all_components.append(component) - components = filter_pick(all_components, re.compile('(comp|openmp|intel-tbb|icc|ifort|psxe|icsxe-pset)').search) + all_components = get_all_components() + components = filter_pick(all_components, re.compile('(comp|openmp|intel-tbb|icc|ifort|psxe|icsxe-pset)').search) self.intel_components = ';'.join(components) IntelInstaller.install(self, spec, prefix) From e3cd0a67d020b73e5fdc6cfeb4ee874d49f2c305 Mon Sep 17 00:00:00 2001 From: "Gregory L. Lee" Date: Fri, 13 May 2016 16:12:48 -0700 Subject: [PATCH 226/481] address flake8 issues --- .../repos/builtin/packages/daal/package.py | 3 +- .../packages/intel-parallel-studio/package.py | 93 ++++++++++++------- .../repos/builtin/packages/intel/package.py | 60 +++++++----- .../repos/builtin/packages/ipp/package.py | 3 +- .../repos/builtin/packages/mkl/package.py | 3 +- 5 files changed, 103 insertions(+), 59 deletions(-) diff --git a/var/spack/repos/builtin/packages/daal/package.py b/var/spack/repos/builtin/packages/daal/package.py index 9fdbd7bf5e..e13dd80e19 100644 --- a/var/spack/repos/builtin/packages/daal/package.py +++ b/var/spack/repos/builtin/packages/daal/package.py @@ -1,8 +1,9 @@ from spack import * -import sys, os, re +import os from spack.pkg.builtin.intel import IntelInstaller + class Daal(IntelInstaller): """Intel Data Analytics Acceleration Library. diff --git a/var/spack/repos/builtin/packages/intel-parallel-studio/package.py b/var/spack/repos/builtin/packages/intel-parallel-studio/package.py index 3a44cb32a3..db6cc888dd 100644 --- a/var/spack/repos/builtin/packages/intel-parallel-studio/package.py +++ b/var/spack/repos/builtin/packages/intel-parallel-studio/package.py @@ -1,8 +1,10 @@ from spack import * -import sys, os, re +import os +import re from spack.pkg.builtin.intel import IntelInstaller, filter_pick, get_all_components + class IntelParallelStudio(IntelInstaller): """Intel Parallel Studio. @@ -14,25 +16,31 @@ class IntelParallelStudio(IntelInstaller): # TODO: can also try the online installer (will download files on demand) version('composer.2016.2', '1133fb831312eb519f7da897fec223fa', - url="file://%s/parallel_studio_xe_2016_composer_edition_update2.tgz" % os.getcwd()) + url="file://%s/parallel_studio_xe_2016_composer_edition_update2.tgz" + % os.getcwd()) version('professional.2016.2', '70be832f2d34c9bf596a5e99d5f2d832', - url="file://%s/parallel_studio_xe_2016_update2.tgz" % os.getcwd()) + url="file://%s/parallel_studio_xe_2016_update2.tgz" % os.getcwd()) version('cluster.2016.2', '70be832f2d34c9bf596a5e99d5f2d832', - url="file://%s/parallel_studio_xe_2016_update2.tgz" % os.getcwd()) + url="file://%s/parallel_studio_xe_2016_update2.tgz" % os.getcwd()) version('composer.2016.3', '3208eeabee951fc27579177b593cefe9', - url="file://%s/parallel_studio_xe_2016_composer_edition_update3.tgz" % os.getcwd()) + url="file://%s/parallel_studio_xe_2016_composer_edition_update3.tgz" + % os.getcwd()) version('professional.2016.3', 'eda19bb0d0d19709197ede58f13443f3', - url="file://%s/parallel_studio_xe_2016_update3.tgz" % os.getcwd()) + url="file://%s/parallel_studio_xe_2016_update3.tgz" % os.getcwd()) version('cluster.2016.3', 'eda19bb0d0d19709197ede58f13443f3', - url="file://%s/parallel_studio_xe_2016_update3.tgz" % os.getcwd()) + url="file://%s/parallel_studio_xe_2016_update3.tgz" % os.getcwd()) variant('rpath', default=True, description="Add rpath to .cfg files") - variant('all', default=False, description="Install all files associated with the requested edition") - variant('mpi', default=True, description="Install the Intel MPI library and ITAC tool") + variant('all', default=False, + description="Install all files associated with the requested edition") + variant('mpi', default=True, + description="Install the Intel MPI library and ITAC tool") variant('mkl', default=True, description="Install the Intel MKL library") - variant('daal', default=True, description="Install the Intel DAAL libraries") + variant('daal', + default=True, description="Install the Intel DAAL libraries") variant('ipp', default=True, description="Install the Intel IPP libraries") - variant('tools', default=True, description="Install the Intel Advisor, VTune Amplifier, and Inspector tools") + variant('tools', default=True, description="""Install the Intel Advisor,\ +VTune Amplifier, and Inspector tools""") provides('mpi', when='@cluster:+mpi') provides('mkl', when='+mkl') @@ -41,23 +49,29 @@ class IntelParallelStudio(IntelInstaller): def install(self, spec, prefix): - base_components = "ALL" # when in doubt, install everything + base_components = "ALL" # when in doubt, install everything mpi_components = "" mkl_components = "" daal_components = "" ipp_components = "" - tools_components = "" if spec.satisfies('+all'): base_components = "ALL" else: all_components = get_all_components() - base_components = filter_pick(all_components, re.compile('(comp|openmp|intel-tbb|icc|ifort|psxe|icsxe-pset)').search) - mpi_components = filter_pick(all_components, re.compile('(icsxe|imb|mpi|itac|intel-tc|clck)').search) - mkl_components = filter_pick(all_components, re.compile('(mkl)').search) - daal_components = filter_pick(all_components, re.compile('(daal)').search) - ipp_components = filter_pick(all_components, re.compile('(ipp)').search) - tool_components = filter_pick(all_components, re.compile('(gdb|vtune|inspector|advisor)').search) + base_components = filter_pick(all_components, + re.compile('(comp|openmp|intel-tbb|icc|ifort|psxe|icsxe-pset)' + ).search) + mpi_components = filter_pick(all_components, + re.compile('(icsxe|imb|mpi|itac|intel-tc|clck)').search) + mkl_components = filter_pick(all_components, + re.compile('(mkl)').search) + daal_components = filter_pick(all_components, + re.compile('(daal)').search) + ipp_components = filter_pick(all_components, + re.compile('(ipp)').search) + tool_components = filter_pick(all_components, + re.compile('(gdb|vtune|inspector|advisor)').search) components = base_components if not spec.satisfies('+all'): @@ -69,32 +83,45 @@ def install(self, spec, prefix): components += daal_components if spec.satisfies('+ipp'): components += ipp_components - if spec.satisfies('+tools') and (spec.satisfies('@cluster') or spec.satisfies('@professional')): + if spec.satisfies('+tools') and (spec.satisfies('@cluster') or\ + spec.satisfies('@professional')): components += tool_components self.intel_components = ';'.join(components) IntelInstaller.install(self, spec, prefix) - absbindir = os.path.dirname(os.path.realpath(os.path.join(self.prefix.bin, "icc"))) - abslibdir = os.path.dirname(os.path.realpath(os.path.join(self.prefix.lib, "intel64", "libimf.a"))) + absbindir = os.path.dirname(os.path.realpath(os.path.join( + self.prefix.bin, "icc"))) + abslibdir = os.path.dirname(os.path.realpath(os.path.join + (self.prefix.lib, "intel64", "libimf.a"))) - relbindir = absbindir.strip(os.path.commonprefix([self.prefix, absbindir])) - os.symlink(self.global_license_file, os.path.join(absbindir, "license.lic")) - if spec.satisfies('+tools') and (spec.satisfies('@cluster') or spec.satisfies('@professional')): + relbindir = absbindir.strip(os.path.commonprefix([self.prefix, + absbindir])) + os.symlink(self.global_license_file, os.path.join(absbindir, + "license.lic")) + if spec.satisfies('+tools') and (spec.satisfies('@cluster') or\ + spec.satisfies('@professional')): os.mkdir(os.path.join(self.prefix, "inspector_xe/licenses")) - os.symlink(self.global_license_file, os.path.join(self.prefix, "inspector_xe/licenses", "license.lic")) + os.symlink(self.global_license_file, os.path.join( + self.prefix, "inspector_xe/licenses", "license.lic")) os.mkdir(os.path.join(self.prefix, "advisor_xe/licenses")) - os.symlink(self.global_license_file, os.path.join(self.prefix, "advisor_xe/licenses", "license.lic")) + os.symlink(self.global_license_file, os.path.join( + self.prefix, "advisor_xe/licenses", "license.lic")) os.mkdir(os.path.join(self.prefix, "vtune_amplifier_xe/licenses")) - os.symlink(self.global_license_file, os.path.join(self.prefix, "vtune_amplifier_xe/licenses", "license.lic")) + os.symlink(self.global_license_file, os.path.join( + self.prefix, "vtune_amplifier_xe/licenses", "license.lic")) - if (spec.satisfies('+all') or spec.satisfies('+mpi')) and spec.satisfies('@cluster'): - os.symlink(self.global_license_file, os.path.join(self.prefix, "itac_latest", "license.lic")) + if (spec.satisfies('+all') or spec.satisfies('+mpi')) and\ + spec.satisfies('@cluster'): + os.symlink(self.global_license_file, os.path.join( + self.prefix, "itac_latest", "license.lic")) if spec.satisfies('+rpath'): for compiler_command in ["icc", "icpc", "ifort"]: - cfgfilename = os.path.join(absbindir, "%s.cfg" %(compiler_command)) + cfgfilename = os.path.join(absbindir, "%s.cfg" %\ + compiler_command) with open(cfgfilename, "w") as f: - f.write('-Xlinker -rpath -Xlinker %s\n' %(abslibdir)) + f.write('-Xlinker -rpath -Xlinker %s\n' % abslibdir) - os.symlink(os.path.join(self.prefix.man, "common", "man1"), os.path.join(self.prefix.man, "man1")) + os.symlink(os.path.join(self.prefix.man, "common", "man1"), + os.path.join(self.prefix.man, "man1")) diff --git a/var/spack/repos/builtin/packages/intel/package.py b/var/spack/repos/builtin/packages/intel/package.py index a547ab56db..2d6929fc73 100644 --- a/var/spack/repos/builtin/packages/intel/package.py +++ b/var/spack/repos/builtin/packages/intel/package.py @@ -1,16 +1,22 @@ from spack import * -import sys, os, re +import os +import re + def filter_pick(input_list, regex_filter): """Returns the items in input_list that are found in the regex_filter""" return [l for l in input_list for m in (regex_filter(l),) if m] + def unfilter_pick(input_list, regex_filter): - """Returns the items in input_list that are not found in the regex_filter""" + """Returns the items in input_list that are not found in the + regex_filter""" return [l for l in input_list for m in (regex_filter(l),) if not m] + def get_all_components(): - """Returns a list of all the components associated with the downloaded Intel package""" + """Returns a list of all the components associated with the downloaded + Intel package""" all_components = [] with open("pset/mediaconfig.xml", "r") as f: lines = f.readlines() @@ -20,6 +26,7 @@ def get_all_components(): all_components.append(component) return all_components + class IntelInstaller(Package): """Base package containing common methods for installing Intel software""" @@ -29,7 +36,8 @@ class IntelInstaller(Package): license_comment = '#' license_files = ['Licenses/license.lic'] license_vars = ['INTEL_LICENSE_FILE'] - license_url = 'https://software.intel.com/en-us/articles/intel-license-manager-faq' + license_url = \ + 'https://software.intel.com/en-us/articles/intel-license-manager-faq' @property def global_license_file(self): @@ -41,9 +49,12 @@ def global_license_file(self): def install(self, spec, prefix): - # remove the installation DB, otherwise it will try to install into location of other Intel builds - if os.path.exists(os.path.join(os.environ["HOME"], "intel", "intel_sdp_products.db")): - os.remove(os.path.join(os.environ["HOME"], "intel", "intel_sdp_products.db")) + # Remove the installation DB, otherwise it will try to install into + # location of other Intel builds + if os.path.exists(os.path.join(os.environ["HOME"], "intel", + "intel_sdp_products.db")): + os.remove(os.path.join(os.environ["HOME"], "intel", + "intel_sdp_products.db")) if not hasattr(self, "intel_prefix"): self.intel_prefix = self.prefix @@ -58,6 +69,7 @@ def install(self, spec, prefix): ACTIVATION_LICENSE_FILE=%s ACTIVATION_TYPE=license_file PHONEHOME_SEND_USAGE_DATA=no +CONTINUE_WITH_OPTIONAL_ERROR=yes COMPONENTS=%s """ %(self.intel_prefix, self.global_license_file, self.intel_components)) @@ -76,37 +88,39 @@ class Intel(IntelInstaller): # TODO: can also try the online installer (will download files on demand) version('16.0.2', '1133fb831312eb519f7da897fec223fa', - url="file://%s/parallel_studio_xe_2016_composer_edition_update2.tgz" % os.getcwd()) + url="file://%s/parallel_studio_xe_2016_composer_edition_update2.tgz"\ + % os.getcwd()) version('16.0.3', '3208eeabee951fc27579177b593cefe9', - url="file://%s/parallel_studio_xe_2016_composer_edition_update3.tgz" % os.getcwd()) + url="file://%s/parallel_studio_xe_2016_composer_edition_update3.tgz"\ + % os.getcwd()) variant('rpath', default=True, description="Add rpath to .cfg files") def install(self, spec, prefix): - - # remove the installation DB, otherwise it will try to install into location of other Intel builds - try: - os.remove(os.path.join(os.environ["HOME"], "intel", "intel_sdp_products.db")) - except OSError: - pass # if the file does not exist - components = [] all_components = get_all_components() - components = filter_pick(all_components, re.compile('(comp|openmp|intel-tbb|icc|ifort|psxe|icsxe-pset)').search) + components = filter_pick(all_components, + re.compile('(comp|openmp|intel-tbb|icc|ifort|psxe|icsxe-pset)' + ).search) self.intel_components = ';'.join(components) IntelInstaller.install(self, spec, prefix) - absbindir = os.path.split(os.path.realpath(os.path.join(self.prefix.bin, "icc")))[0] - abslibdir = os.path.split(os.path.realpath(os.path.join(self.prefix.lib, "intel64", "libimf.a")))[0] + absbindir = os.path.split(os.path.realpath(os.path.join( + self.prefix.bin, "icc")))[0] + abslibdir = os.path.split(os.path.realpath(os.path.join( + self.prefix.lib, "intel64", "libimf.a")))[0] # symlink or copy? - os.symlink(self.global_license_file, os.path.join(absbindir, "license.lic")) + os.symlink(self.global_license_file, os.path.join(absbindir, + "license.lic")) if spec.satisfies('+rpath'): for compiler_command in ["icc", "icpc", "ifort"]: - cfgfilename = os.path.join(absbindir, "%s.cfg" %(compiler_command)) + cfgfilename = os.path.join(absbindir, "%s.cfg" %\ + compiler_command) with open(cfgfilename, "w") as f: - f.write('-Xlinker -rpath -Xlinker %s\n' %(abslibdir)) + f.write('-Xlinker -rpath -Xlinker %s\n' % abslibdir) - os.symlink(os.path.join(self.prefix.man, "common", "man1"), os.path.join(self.prefix.man, "man1")) + os.symlink(os.path.join(self.prefix.man, "common", "man1"), + os.path.join(self.prefix.man, "man1")) diff --git a/var/spack/repos/builtin/packages/ipp/package.py b/var/spack/repos/builtin/packages/ipp/package.py index 23c944c2d5..2bd931d5bd 100644 --- a/var/spack/repos/builtin/packages/ipp/package.py +++ b/var/spack/repos/builtin/packages/ipp/package.py @@ -1,8 +1,9 @@ from spack import * -import sys, os, re +import os from spack.pkg.builtin.intel import IntelInstaller + class Ipp(IntelInstaller): """Intel Integrated Performance Primitives. diff --git a/var/spack/repos/builtin/packages/mkl/package.py b/var/spack/repos/builtin/packages/mkl/package.py index 1d4553b7f5..454e78d29c 100644 --- a/var/spack/repos/builtin/packages/mkl/package.py +++ b/var/spack/repos/builtin/packages/mkl/package.py @@ -1,8 +1,9 @@ from spack import * -import os, re, sys +import os from spack.pkg.builtin.intel import IntelInstaller + class Mkl(IntelInstaller): """Intel Math Kernel Library. From 8ddc1f89778335dda3d1dd213e75bc3a92d99712 Mon Sep 17 00:00:00 2001 From: Brett Viren Date: Sat, 14 May 2016 09:11:04 -0400 Subject: [PATCH 227/481] Move from str.format() to string.Template. --- lib/spack/spack/cmd/view.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/cmd/view.py b/lib/spack/spack/cmd/view.py index b504dfd2b1..caf14bfa53 100644 --- a/lib/spack/spack/cmd/view.py +++ b/lib/spack/spack/cmd/view.py @@ -317,16 +317,19 @@ def visitor_statlink(specs, args): def visitor_print(specs, args): 'Print a string for each spec using args.format.' fmt = args.format[0] + from string import Template + t = Template(fmt) + for spec in specs: kwds = spec2dict(spec) try: - string = fmt.format(**kwds) + text = t.substitute(kwds) except KeyError: tty.error("Format error, use keywords: %s" % (', '.join(kwds.keys()), )) raise # argparser escapes these - string = string.replace(r'\n', '\n').replace(r'\t', '\t') - sys.stdout.write(string) + text = text.replace(r'\n', '\n').replace(r'\t', '\t') + sys.stdout.write(text) # Finally, the actual "view" command. There should be no need to From 8078845273021dae01e3fdde2a5e3b10aea2f845 Mon Sep 17 00:00:00 2001 From: Brett Viren Date: Sat, 14 May 2016 09:11:41 -0400 Subject: [PATCH 228/481] Remove problematic tests, deal with this issue outside of PR #869. --- share/spack/examples/test_view.sh | 31 ---------------- .../builtin.mock/packages/test-a/package.py | 30 ---------------- .../packages/test-a/test-a-0.0.tar.gz | Bin 112 -> 0 bytes .../builtin.mock/packages/test-b/package.py | 32 ----------------- .../packages/test-b/test-b-0.0.tar.gz | Bin 112 -> 0 bytes .../builtin.mock/packages/test-c/package.py | 32 ----------------- .../packages/test-c/test-c-0.0.tar.gz | Bin 112 -> 0 bytes .../builtin.mock/packages/test-d/package.py | 33 ------------------ .../packages/test-d/test-d-0.0.tar.gz | Bin 112 -> 0 bytes 9 files changed, 158 deletions(-) delete mode 100755 share/spack/examples/test_view.sh delete mode 100644 var/spack/repos/builtin.mock/packages/test-a/package.py delete mode 100644 var/spack/repos/builtin.mock/packages/test-a/test-a-0.0.tar.gz delete mode 100644 var/spack/repos/builtin.mock/packages/test-b/package.py delete mode 100644 var/spack/repos/builtin.mock/packages/test-b/test-b-0.0.tar.gz delete mode 100644 var/spack/repos/builtin.mock/packages/test-c/package.py delete mode 100644 var/spack/repos/builtin.mock/packages/test-c/test-c-0.0.tar.gz delete mode 100644 var/spack/repos/builtin.mock/packages/test-d/package.py delete mode 100644 var/spack/repos/builtin.mock/packages/test-d/test-d-0.0.tar.gz diff --git a/share/spack/examples/test_view.sh b/share/spack/examples/test_view.sh deleted file mode 100755 index b3e7102192..0000000000 --- a/share/spack/examples/test_view.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -# This will install a few bogus/test packages in order to test the -# `spack view` command. It assumes you have "spack" in your path. - -# It makes sub-directories in your CWD and installs and uninstalls -# Spack packages named test-*. - -set -x -set -e - -view="spack -m view -v" -for variant in +nom ~nom+var +nom+var -do - spack -m uninstall -f -a -y test-d - spack -m install test-d$variant - testdir=test_view - rm -rf $testdir - echo "hardlink may fail if Spack install area and CWD are not same FS" - for action in symlink hardlink - do - $view --dependencies=no $action $testdir test-d - $view -e test-a -e test-b $action $testdir test-d - $view $action $testdir test-d - $view status $testdir test-d - $view -d false remove $testdir test-a - $view remove $testdir test-d - rmdir $testdir # should not fail - done -done -echo "Warnings about skipping existing in the above are okay" diff --git a/var/spack/repos/builtin.mock/packages/test-a/package.py b/var/spack/repos/builtin.mock/packages/test-a/package.py deleted file mode 100644 index ff45344b7a..0000000000 --- a/var/spack/repos/builtin.mock/packages/test-a/package.py +++ /dev/null @@ -1,30 +0,0 @@ -from spack import * - -import os - -mydir = os.path.dirname(__file__) -source = os.path.join(mydir,'test-a-0.0.tar.gz') - -class TestA(Package): - """The test-a package""" - - url = 'file://'+source - homepage = "http://www.example.com/" - - version('0.0', '4e823d0af4154fcf52b75dad41b7fd63') - - variant('nom', default=True, description='Nominal variant') - variant('var', default=False, description='Variant variant') - - def install(self, spec, prefix): - bindir = os.path.join(prefix,'bin') - os.makedirs(bindir) - script = os.path.join(bindir, 'test-a') - with open(script,'w') as fp: - fp.write("""#!/bin/bash -echo 'name: %s' -echo 'prefix: %s' -echo 'spec: %s' - """ % (spec.name, prefix, spec)) - - os.chmod(script, 0555) diff --git a/var/spack/repos/builtin.mock/packages/test-a/test-a-0.0.tar.gz b/var/spack/repos/builtin.mock/packages/test-a/test-a-0.0.tar.gz deleted file mode 100644 index 3588e05765cd799dfe19e1f77e057bf7508ec47e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 112 zcmb2|=3uzVq!!M={Pvt8-ysJPmJ7D0<~95|Kkb{1f}-PrX$3W18BI)~SE|#F^vw8N zv-i!f&C_|0CdssyKPx$LK4$72|C!Dyr^A#M>z4NF`mcRfZGSIIclq(}ch}1?z`(y3 OtXZo*ykgK`U;qF>c`_gX diff --git a/var/spack/repos/builtin.mock/packages/test-b/package.py b/var/spack/repos/builtin.mock/packages/test-b/package.py deleted file mode 100644 index f6d30bb85f..0000000000 --- a/var/spack/repos/builtin.mock/packages/test-b/package.py +++ /dev/null @@ -1,32 +0,0 @@ -from spack import * - -import os - -mydir = os.path.dirname(__file__) -source = os.path.join(mydir,'test-b-0.0.tar.gz') - -class TestB(Package): - """The test-b package""" - - url = 'file://'+source - homepage = "http://www.example.com/" - - version('0.0', '4e823d0af4154fcf52b75dad41b7fd63') - - variant('nom', default=True, description='Nominal variant') - variant('var', default=False, description='Variant variant') - - depends_on('test-a') - - def install(self, spec, prefix): - bindir = os.path.join(prefix,'bin') - os.makedirs(bindir) - script = os.path.join(bindir, 'test-b') - with open(script,'w') as fp: - fp.write("""#!/bin/bash -echo 'name: %s' -echo 'prefix: %s' -echo 'spec: %s' - """ % (spec.name, prefix, spec)) - - os.chmod(script, 0555) diff --git a/var/spack/repos/builtin.mock/packages/test-b/test-b-0.0.tar.gz b/var/spack/repos/builtin.mock/packages/test-b/test-b-0.0.tar.gz deleted file mode 100644 index 3588e05765cd799dfe19e1f77e057bf7508ec47e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 112 zcmb2|=3uzVq!!M={Pvt8-ysJPmJ7D0<~95|Kkb{1f}-PrX$3W18BI)~SE|#F^vw8N zv-i!f&C_|0CdssyKPx$LK4$72|C!Dyr^A#M>z4NF`mcRfZGSIIclq(}ch}1?z`(y3 OtXZo*ykgK`U;qF>c`_gX diff --git a/var/spack/repos/builtin.mock/packages/test-c/package.py b/var/spack/repos/builtin.mock/packages/test-c/package.py deleted file mode 100644 index 641e46ee50..0000000000 --- a/var/spack/repos/builtin.mock/packages/test-c/package.py +++ /dev/null @@ -1,32 +0,0 @@ -from spack import * - -import os - -mydir = os.path.dirname(__file__) -source = os.path.join(mydir,'test-c-0.0.tar.gz') - -class TestC(Package): - """The test-c package""" - - url = 'file://'+source - homepage = "http://www.example.com/" - - version('0.0', '4e823d0af4154fcf52b75dad41b7fd63') - - variant('nom', default=True, description='Nominal variant') - variant('var', default=False, description='Variant variant') - - depends_on('test-a+var',when='+var') - - def install(self, spec, prefix): - bindir = os.path.join(prefix,'bin') - os.makedirs(bindir) - script = os.path.join(bindir, 'test-c') - with open(script,'w') as fp: - fp.write("""#!/bin/bash -echo 'name: %s' -echo 'prefix: %s' -echo 'spec: %s' - """ % (spec.name, prefix, spec)) - - os.chmod(script, 0555) diff --git a/var/spack/repos/builtin.mock/packages/test-c/test-c-0.0.tar.gz b/var/spack/repos/builtin.mock/packages/test-c/test-c-0.0.tar.gz deleted file mode 100644 index 3588e05765cd799dfe19e1f77e057bf7508ec47e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 112 zcmb2|=3uzVq!!M={Pvt8-ysJPmJ7D0<~95|Kkb{1f}-PrX$3W18BI)~SE|#F^vw8N zv-i!f&C_|0CdssyKPx$LK4$72|C!Dyr^A#M>z4NF`mcRfZGSIIclq(}ch}1?z`(y3 OtXZo*ykgK`U;qF>c`_gX diff --git a/var/spack/repos/builtin.mock/packages/test-d/package.py b/var/spack/repos/builtin.mock/packages/test-d/package.py deleted file mode 100644 index b46c680edd..0000000000 --- a/var/spack/repos/builtin.mock/packages/test-d/package.py +++ /dev/null @@ -1,33 +0,0 @@ -from spack import * - -import os - -mydir = os.path.dirname(__file__) -source = os.path.join(mydir,'test-d-0.0.tar.gz') - -class TestD(Package): - """The test-d package""" - - url = 'file://'+source - homepage = "http://www.example.com/" - - version('0.0', '4e823d0af4154fcf52b75dad41b7fd63') - - variant('nom', default=True, description='Nominal variant') - variant('var', default=False, description='Variant variant') - - depends_on('test-b') - depends_on('test-c') - - def install(self, spec, prefix): - bindir = os.path.join(prefix,'bin') - os.makedirs(bindir) - script = os.path.join(bindir, 'test-d') - with open(script,'w') as fp: - fp.write("""#!/bin/bash -echo 'name: %s' -echo 'prefix: %s' -echo 'spec: %s' - """ % (spec.name, prefix, spec)) - - os.chmod(script, 0555) diff --git a/var/spack/repos/builtin.mock/packages/test-d/test-d-0.0.tar.gz b/var/spack/repos/builtin.mock/packages/test-d/test-d-0.0.tar.gz deleted file mode 100644 index 3588e05765cd799dfe19e1f77e057bf7508ec47e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 112 zcmb2|=3uzVq!!M={Pvt8-ysJPmJ7D0<~95|Kkb{1f}-PrX$3W18BI)~SE|#F^vw8N zv-i!f&C_|0CdssyKPx$LK4$72|C!Dyr^A#M>z4NF`mcRfZGSIIclq(}ch}1?z`(y3 OtXZo*ykgK`U;qF>c`_gX From c98afe2205ec90bc98c7f1c33b3e8e05403c7c22 Mon Sep 17 00:00:00 2001 From: Brett Viren Date: Sat, 14 May 2016 13:14:51 -0400 Subject: [PATCH 229/481] Pick a better name for the 'spec' parameters. --- lib/spack/spack/cmd/view.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/cmd/view.py b/lib/spack/spack/cmd/view.py index caf14bfa53..8767459c08 100644 --- a/lib/spack/spack/cmd/view.py +++ b/lib/spack/spack/cmd/view.py @@ -189,8 +189,8 @@ def spec2dict(spec): # pkg = spec.package ret = dict(name = spec.name, - short = spec.short_spec, - cshort = spec.cshort_spec, # color + spec = spec.short_spec, + colorspec = spec.cshort_spec, # color root = spec.root, prefix = spec.prefix, version = spec.version, From b3ede099e2d2dd684ebe478dd711a979ebb3bf98 Mon Sep 17 00:00:00 2001 From: Brett Viren Date: Sat, 14 May 2016 13:15:25 -0400 Subject: [PATCH 230/481] Document command. --- lib/spack/docs/basic_usage.rst | 233 +++++++++++++++++++++++++++++++++ 1 file changed, 233 insertions(+) diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst index 29791d98c4..e340881a93 100644 --- a/lib/spack/docs/basic_usage.rst +++ b/lib/spack/docs/basic_usage.rst @@ -330,6 +330,125 @@ of libelf would look like this: The full spec syntax is discussed in detail in :ref:`sec-specs`. +``spack view print`` +~~~~~~~~~~~~~~~~~~~~~~ + +``spack view print`` is a subcommand of ``spack view`` which displays information about one or more installed packages and their dependencies using a user-provided format string. The string can reverence variables in a shell-like manner for example ``$variable`` or ``${variable}text``. It can also include ``\t`` for tabs and ``\n`` for new lines. + +Some of the supported variables are: + +``name`` + The package name. + +``version`` + The package version. + +``spec`` + The package specification. + +``root`` + The root specification. + +``prefix`` + The installation directory. + +``variants`` + The collection of variants, if any. + +``namespace`` + The package repository name space. + +``compiler`` + The compiler \`name@versoin\` used to build the package. + +``architecture`` + The architecture targeted by the compiler. + +``dependencies`` + A comma-separated list of names of packages on which the package depends. + +``dependents`` + A comma-separated list of names of packages which depend on the package. + +``hash`` + The Spack hash for the package. + +``url`` + The source URL for the package. + +``stage`` + The directory for staging the build. + +``build_log`` + The path to the build log file. + +``rpath`` + The colon-separated library \`RPATH\` used in building the package. + +Here are some example uses of `spack view print`. A simple line-oriented report of information can be produced: + +.. code-block:: sh + + $ spack view print '$name\t$version\t$hash\n' cmake@3.5.2 + ncurses 6.0 bvbu4ixbnvtodpik4qzljlx3ukpyfrcz + zlib 1.2.8 ckki7zlryxrsetfqkgoxxahlhqqjni7n + openssl 1.0.2g 6zbar63sciso253nptxyrnhupymo7oyi + cmake 3.5.2 wprvmoczkpw4tiy5ybuk5zr7saus2d7g + +There are better ways to do this but a slow-and-dirty shell init procedure can be formed: + +.. code-block:: sh + + $ spack view print 'export PATH="${prefix}/bin:$$PATH"\n' m4 + export PATH="/spack/opt/spack/linux-x86_64/gcc-5.2.1/libsigsegv-2.10-h6hsv76hffcjoe3nsaihzxemniwiedu2/bin:$PATH" + export PATH="/spack/opt/spack/linux-x86_64/gcc-5.2.1/m4-1.4.17-a4ikhddnk2zonr66mbwaqb226uhigcrp/bin:$PATH" + $ eval $(spack view print 'export PATH="${prefix}/bin:$$PATH"\n' m4) + +Or, maybe you have some external application that can chew on Spack data in which case you can dump it to some easily parsed markup syntax such as YAML: + +.. code-block:: sh + + + $ spack view print '${name}:\n - version: ${version}\n - url : ${url}\n - spec: ${spec}\n - prefix: ${prefix}\n - root : ${root}\n - stage : ${stage}\n - log: ${build_log}\n' m4 + +Which might produce something like: + +.. code-block:: yaml + + libsigsegv: + - version: 2.10 + - url : ftp://ftp.gnu.org/gnu/libsigsegv/libsigsegv-2.10.tar.gz + - spec: libsigsegv@2.10%gcc@5.2.1=linux-x86_64-h6hsv76 + - prefix: /spack/opt/spack/linux-x86_64/gcc-5.2.1/libsigsegv-2.10-h6hsv76hffcjoe3nsaihzxemniwiedu2 + - root : m4@1.4.17%gcc@5.2.1+sigsegv=linux-x86_64^libsigsegv@2.10%gcc@5.2.1=linux-x86_64 + - stage : /spack/var/spack/stage/libsigsegv-2.10-h6hsv76hffcjoe3nsaihzxemniwiedu2 + - log: /spack/opt/spack/linux-x86_64/gcc-5.2.1/libsigsegv-2.10-h6hsv76hffcjoe3nsaihzxemniwiedu2/.spack/build.out + m4: + - version: 1.4.17 + - url : ftp://ftp.gnu.org/gnu/m4/m4-1.4.17.tar.gz + - spec: m4@1.4.17%gcc@5.2.1+sigsegv=linux-x86_64-a4ikhdd + - prefix: /spack/opt/spack/linux-x86_64/gcc-5.2.1/m4-1.4.17-a4ikhddnk2zonr66mbwaqb226uhigcrp + - root : m4@1.4.17%gcc@5.2.1+sigsegv=linux-x86_64^libsigsegv@2.10%gcc@5.2.1=linux-x86_64 + - stage : /spack/var/spack/stage/m4-1.4.17-a4ikhddnk2zonr66mbwaqb226uhigcrp + - log: /spack/opt/spack/linux-x86_64/gcc-5.2.1/m4-1.4.17-a4ikhddnk2zonr66mbwaqb226uhigcrp/.spack/build.out + +Or, maybe you want to do something with information about package dependencies by generating some ready-to-import Python code holding the tree: + +.. code-block:: sh + + $ spack view print '$name = dict(name = "$name", parents = [$dependencies], children = [$dependents])\n' cmake@3.5.2 + +Producing this Python code + +.. code-block:: python + + ncurses = dict(name = "ncurses", parents = []]) + zlib = dict(name = "zlib", parents = []]) + openssl = dict(name = "openssl", parents = [zlib]]) + cmake = dict(name = "cmake", parents = [ncurses,openssl]]) + + + Compiler configuration ----------------------------------- @@ -1035,6 +1154,120 @@ regenerate all module and dotkit files from scratch: .. _extensions: +Filesystem Views +------------------------------- + +.. Maybe this is not the right location for this documentation. + +The Spack installation area allows for many package installation trees +to coexist and gives the user choices as to what versions and variants +of packages to use. To use them, the user must rely on a way to +aggregate a subset of those packages. The section on Environment +Modules gives one good way to do that which relies on setting various +environment variables. An alternative way to aggregate is through +**filesystem views**. + +A filesystem view is a single directory tree which is the union of the +directory hierarchies of the individual package installation trees +that have been included. The files of the view's installed packages +are brought into the view by symbolic or hard links back to their +location in the original Spack installation area. As the view is +formed, any clashes due to a file having the exact same path in its +package installation tree are handled in a first-come-first-served +basis and a warning is printed. Packages and their dependencies can +be both added and removed. During removal, empty directories will be +purged. These operations can be limited to pertain to just the +packages listed by the user or to exclude specific dependencies and +they allow for software installed outside of Spack to coexist inside +the filesystem view tree. + +By its nature, a filesystem view represents a particular choice of one +set of packages among all the versions and variants that are available +in the Spack installation area. It is thus equivalent to the +directory hiearchy that might exist under ``/usr/local``. While this +limits a view to including only one version/variant of any package, it +provides the benefits of having a simpler and traditional layout which +may be used without any particular knowledge that its packages were +built by Spack. + +Views can be used for a variety of purposes including: + +- A central installation in a traditional layout, eg ``/usr/local`` maintained over time by the sysadmin. +- A self-contained installation area which may for the basis of a top-level atomic versioning scheme, eg ``/opt/pro`` vs ``/opt/dev``. +- Providing an atomic and monolithic binary distribution, eg for delivery as a single tarball. +- Producing ephemeral testing or developing environments. + +Using Filesystem Views +~~~~~~~~~~~~~~~~~~~~~~ + +A filesystem view is created and packages are linked in by the ``spack +view`` command's ``symlink`` and ``hardlink`` sub-commands. The +``spack view remove`` command can be used to unlink some or all of the +filesystem view. + +The following example creates a filesystem view based +on an installed ``cmake`` package and then removes from the view the +files in the ``cmake`` package while retaining its dependencies. + +.. code-block:: sh + + + $ spack view -v symlink myview cmake@3.5.2 + ==> Linking package: "ncurses" + ==> Linking package: "zlib" + ==> Linking package: "openssl" + ==> Linking package: "cmake" + + $ ls myview/ + bin doc etc include lib share + + $ ls myview/bin/ + captoinfo clear cpack ctest infotocap openssl tabs toe tset + ccmake cmake c_rehash infocmp ncurses6-config reset tic tput + + $ spack view -v -d false rm myview cmake@3.5.2 + ==> Removing package: "cmake" + + $ ls myview/bin/ + captoinfo c_rehash infotocap openssl tabs toe tset + clear infocmp ncurses6-config reset tic tput + + +Limitations of Filesystem Views +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This section describes some limitations that should be considered in +using filesystems views. + +Filesystem views are merely organizational. The binary executable +programs, shared libraries and other build products found in a view +are mere links into the "real" Spack installation area. If a view is +built with symbolic links it requires the Spack-installed package to +be kept in place. Building a view with hardlinks removes this +requirement but any internal paths (eg, rpath or ``#!`` interpreter +specifications) will still require the Spack-installed package files +to be in place. + +.. FIXME: reference the relocation work of Hegner and Gartung. + +As described above, when a view is built only a single instance of a +file may exist in the unified filesystem tree. If more than one +package provides a file at the same path (relative to its own root) +then it is the first package added to the view that "wins". A warning +is printed and it is up to the user to determine if the conflict +matters. + +It is up to the user to assure a consistent view is produced. In +particular if the user excludes packages, limits the following of +dependencies or removes packages the view may become inconsistent. In +particular, if two packages require the same sub-tree of dependencies, +removing one package (recursively) will remove its dependencies and +leave the other package broken. + + + + + Extensions & Python support ------------------------------------ From c2ca5f44b4091821fbda36ce2dce77af3e4087f5 Mon Sep 17 00:00:00 2001 From: Greg Lee Date: Sun, 15 May 2016 08:19:55 -0700 Subject: [PATCH 231/481] flake8 fixes --- .../packages/intel-parallel-studio/package.py | 77 ++++++++++--------- .../repos/builtin/packages/intel/package.py | 23 +++--- 2 files changed, 50 insertions(+), 50 deletions(-) diff --git a/var/spack/repos/builtin/packages/intel-parallel-studio/package.py b/var/spack/repos/builtin/packages/intel-parallel-studio/package.py index db6cc888dd..a176af101d 100644 --- a/var/spack/repos/builtin/packages/intel-parallel-studio/package.py +++ b/var/spack/repos/builtin/packages/intel-parallel-studio/package.py @@ -2,7 +2,8 @@ import os import re -from spack.pkg.builtin.intel import IntelInstaller, filter_pick, get_all_components +from spack.pkg.builtin.intel import IntelInstaller, filter_pick, \ + get_all_components class IntelParallelStudio(IntelInstaller): @@ -16,28 +17,28 @@ class IntelParallelStudio(IntelInstaller): # TODO: can also try the online installer (will download files on demand) version('composer.2016.2', '1133fb831312eb519f7da897fec223fa', - url="file://%s/parallel_studio_xe_2016_composer_edition_update2.tgz" + url="file://%s/parallel_studio_xe_2016_composer_edition_update2.tgz" # NOQA: ignore=E501 % os.getcwd()) version('professional.2016.2', '70be832f2d34c9bf596a5e99d5f2d832', - url="file://%s/parallel_studio_xe_2016_update2.tgz" % os.getcwd()) + url="file://%s/parallel_studio_xe_2016_update2.tgz" % os.getcwd()) # NOQA: ignore=E501 version('cluster.2016.2', '70be832f2d34c9bf596a5e99d5f2d832', - url="file://%s/parallel_studio_xe_2016_update2.tgz" % os.getcwd()) + url="file://%s/parallel_studio_xe_2016_update2.tgz" % os.getcwd()) # NOQA: ignore=E501 version('composer.2016.3', '3208eeabee951fc27579177b593cefe9', - url="file://%s/parallel_studio_xe_2016_composer_edition_update3.tgz" + url="file://%s/parallel_studio_xe_2016_composer_edition_update3.tgz" # NOQA: ignore=E501 % os.getcwd()) version('professional.2016.3', 'eda19bb0d0d19709197ede58f13443f3', - url="file://%s/parallel_studio_xe_2016_update3.tgz" % os.getcwd()) + url="file://%s/parallel_studio_xe_2016_update3.tgz" % os.getcwd()) # NOQA: ignore=E501 version('cluster.2016.3', 'eda19bb0d0d19709197ede58f13443f3', - url="file://%s/parallel_studio_xe_2016_update3.tgz" % os.getcwd()) + url="file://%s/parallel_studio_xe_2016_update3.tgz" % os.getcwd()) # NOQA: ignore=E501 variant('rpath', default=True, description="Add rpath to .cfg files") variant('all', default=False, - description="Install all files associated with the requested edition") + description="Install all files with the requested edition") variant('mpi', default=True, - description="Install the Intel MPI library and ITAC tool") + description="Install the Intel MPI library and ITAC tool") variant('mkl', default=True, description="Install the Intel MKL library") variant('daal', - default=True, description="Install the Intel DAAL libraries") + default=True, description="Install the Intel DAAL libraries") variant('ipp', default=True, description="Install the Intel IPP libraries") variant('tools', default=True, description="""Install the Intel Advisor,\ VTune Amplifier, and Inspector tools""") @@ -59,19 +60,21 @@ def install(self, spec, prefix): base_components = "ALL" else: all_components = get_all_components() - base_components = filter_pick(all_components, - re.compile('(comp|openmp|intel-tbb|icc|ifort|psxe|icsxe-pset)' - ).search) - mpi_components = filter_pick(all_components, - re.compile('(icsxe|imb|mpi|itac|intel-tc|clck)').search) - mkl_components = filter_pick(all_components, - re.compile('(mkl)').search) - daal_components = filter_pick(all_components, - re.compile('(daal)').search) - ipp_components = filter_pick(all_components, - re.compile('(ipp)').search) - tool_components = filter_pick(all_components, - re.compile('(gdb|vtune|inspector|advisor)').search) + regex = '(comp|openmp|intel-tbb|icc|ifort|psxe|icsxe-pset)' + base_components = \ + filter_pick(all_components, re.compile(regex).search) + regex = '(icsxe|imb|mpi|itac|intel-tc|clck)' + mpi_components = \ + filter_pick(all_components, re.compile(regex).search) + mkl_components = \ + filter_pick(all_components, re.compile('(mkl)').search) + daal_components = \ + filter_pick(all_components, re.compile('(daal)').search) + ipp_components = \ + filter_pick(all_components, re.compile('(ipp)').search) + regex = '(gdb|vtune|inspector|advisor)' + tool_components = \ + filter_pick(all_components, re.compile(regex).search) components = base_components if not spec.satisfies('+all'): @@ -83,8 +86,8 @@ def install(self, spec, prefix): components += daal_components if spec.satisfies('+ipp'): components += ipp_components - if spec.satisfies('+tools') and (spec.satisfies('@cluster') or\ - spec.satisfies('@professional')): + if spec.satisfies('+tools') and (spec.satisfies('@cluster') or + spec.satisfies('@professional')): components += tool_components self.intel_components = ';'.join(components) @@ -93,14 +96,12 @@ def install(self, spec, prefix): absbindir = os.path.dirname(os.path.realpath(os.path.join( self.prefix.bin, "icc"))) abslibdir = os.path.dirname(os.path.realpath(os.path.join - (self.prefix.lib, "intel64", "libimf.a"))) + (self.prefix.lib, "intel64", "libimf.a"))) - relbindir = absbindir.strip(os.path.commonprefix([self.prefix, - absbindir])) os.symlink(self.global_license_file, os.path.join(absbindir, - "license.lic")) - if spec.satisfies('+tools') and (spec.satisfies('@cluster') or\ - spec.satisfies('@professional')): + "license.lic")) + if spec.satisfies('+tools') and (spec.satisfies('@cluster') or + spec.satisfies('@professional')): os.mkdir(os.path.join(self.prefix, "inspector_xe/licenses")) os.symlink(self.global_license_file, os.path.join( self.prefix, "inspector_xe/licenses", "license.lic")) @@ -111,17 +112,17 @@ def install(self, spec, prefix): os.symlink(self.global_license_file, os.path.join( self.prefix, "vtune_amplifier_xe/licenses", "license.lic")) - if (spec.satisfies('+all') or spec.satisfies('+mpi')) and\ - spec.satisfies('@cluster'): - os.symlink(self.global_license_file, os.path.join( - self.prefix, "itac_latest", "license.lic")) + if (spec.satisfies('+all') or spec.satisfies('+mpi')) and \ + spec.satisfies('@cluster'): + os.symlink(self.global_license_file, os.path.join( + self.prefix, "itac_latest", "license.lic")) if spec.satisfies('+rpath'): for compiler_command in ["icc", "icpc", "ifort"]: - cfgfilename = os.path.join(absbindir, "%s.cfg" %\ - compiler_command) + cfgfilename = os.path.join(absbindir, "%s.cfg" % + compiler_command) with open(cfgfilename, "w") as f: f.write('-Xlinker -rpath -Xlinker %s\n' % abslibdir) os.symlink(os.path.join(self.prefix.man, "common", "man1"), - os.path.join(self.prefix.man, "man1")) + os.path.join(self.prefix.man, "man1")) diff --git a/var/spack/repos/builtin/packages/intel/package.py b/var/spack/repos/builtin/packages/intel/package.py index 2d6929fc73..ec3192380a 100644 --- a/var/spack/repos/builtin/packages/intel/package.py +++ b/var/spack/repos/builtin/packages/intel/package.py @@ -52,9 +52,9 @@ def install(self, spec, prefix): # Remove the installation DB, otherwise it will try to install into # location of other Intel builds if os.path.exists(os.path.join(os.environ["HOME"], "intel", - "intel_sdp_products.db")): + "intel_sdp_products.db")): os.remove(os.path.join(os.environ["HOME"], "intel", - "intel_sdp_products.db")) + "intel_sdp_products.db")) if not hasattr(self, "intel_prefix"): self.intel_prefix = self.prefix @@ -71,7 +71,7 @@ def install(self, spec, prefix): PHONEHOME_SEND_USAGE_DATA=no CONTINUE_WITH_OPTIONAL_ERROR=yes COMPONENTS=%s -""" %(self.intel_prefix, self.global_license_file, self.intel_components)) +""" % (self.intel_prefix, self.global_license_file, self.intel_components)) install_script = which("install.sh") install_script('--silent', silent_config_filename) @@ -88,10 +88,10 @@ class Intel(IntelInstaller): # TODO: can also try the online installer (will download files on demand) version('16.0.2', '1133fb831312eb519f7da897fec223fa', - url="file://%s/parallel_studio_xe_2016_composer_edition_update2.tgz"\ + url="file://%s/parallel_studio_xe_2016_composer_edition_update2.tgz" # NOQA: ignore=E501 % os.getcwd()) version('16.0.3', '3208eeabee951fc27579177b593cefe9', - url="file://%s/parallel_studio_xe_2016_composer_edition_update3.tgz"\ + url="file://%s/parallel_studio_xe_2016_composer_edition_update3.tgz" # NOQA: ignore=E501 % os.getcwd()) variant('rpath', default=True, description="Add rpath to .cfg files") @@ -99,9 +99,8 @@ class Intel(IntelInstaller): def install(self, spec, prefix): components = [] all_components = get_all_components() - components = filter_pick(all_components, - re.compile('(comp|openmp|intel-tbb|icc|ifort|psxe|icsxe-pset)' - ).search) + regex = '(comp|openmp|intel-tbb|icc|ifort|psxe|icsxe-pset)' + components = filter_pick(all_components, re.compile(regex).search) self.intel_components = ';'.join(components) IntelInstaller.install(self, spec, prefix) @@ -113,14 +112,14 @@ def install(self, spec, prefix): # symlink or copy? os.symlink(self.global_license_file, os.path.join(absbindir, - "license.lic")) + "license.lic")) if spec.satisfies('+rpath'): for compiler_command in ["icc", "icpc", "ifort"]: - cfgfilename = os.path.join(absbindir, "%s.cfg" %\ - compiler_command) + cfgfilename = os.path.join(absbindir, "%s.cfg" % + compiler_command) with open(cfgfilename, "w") as f: f.write('-Xlinker -rpath -Xlinker %s\n' % abslibdir) os.symlink(os.path.join(self.prefix.man, "common", "man1"), - os.path.join(self.prefix.man, "man1")) + os.path.join(self.prefix.man, "man1")) From 77a34ebdf94dd39be338ab85deb45e564c0a563b Mon Sep 17 00:00:00 2001 From: Brett Viren Date: Mon, 16 May 2016 10:08:59 -0400 Subject: [PATCH 232/481] flake8-clean. --- lib/spack/spack/cmd/view.py | 159 +++++++++++++++++++----------------- 1 file changed, 84 insertions(+), 75 deletions(-) diff --git a/lib/spack/spack/cmd/view.py b/lib/spack/spack/cmd/view.py index 8767459c08..4861383eaa 100644 --- a/lib/spack/spack/cmd/view.py +++ b/lib/spack/spack/cmd/view.py @@ -29,14 +29,17 @@ This set consists of: - specs resolved from the package names given by the user (the seeds) + - all depenencies of the seeds unless user specifies `--no-depenencies` -- less any specs with names matching the regular expressions given by `--exclude` + +- less any specs with names matching the regular expressions given by + `--exclude` The `view` command provides a number of functions (the "actions"): - symlink :: a file system view which is a directory hierarchy that is the union of the hierarchies of the installed packages in the DAG - where installed files are referenced via symlinks. + where installed files are referenced via symlinks. - hardlink :: like the symlink view but hardlinks are used. @@ -66,71 +69,70 @@ import os import re import sys -import argparse - import spack import spack.cmd import llnl.util.tty as tty description = "Produce a single-rooted directory view of a spec." + def setup_parser(sp): setup_parser.parser = sp - sp.add_argument('-v','--verbose', action='store_true', default=False, - help="Display verbose output.") - sp.add_argument('-e','--exclude', action='append', default=[], - help="Exclude packages with names matching the given regex pattern.") - sp.add_argument('-d', '--dependencies', choices=['true','false','yes','no'], - default='true', - help="Follow dependencies.") - + sp.add_argument( + '-v', '--verbose', action='store_true', default=False, + help="Display verbose output.") + sp.add_argument( + '-e', '--exclude', action='append', default=[], + help="Exclude packages with names matching the given regex pattern.") + sp.add_argument( + '-d', '--dependencies', choices=['true', 'false', 'yes', 'no'], + default='true', + help="Follow dependencies.") ssp = sp.add_subparsers(metavar='ACTION', dest='action') - specs_opts = dict(metavar='spec', nargs='+', + specs_opts = dict(metavar='spec', nargs='+', help="Seed specs of the packages to view.") # The action parameterizes the command but in keeping with Spack # patterns we make it a subcommand. file_system_view_actions = [ - ssp.add_parser('symlink', aliases=['add','soft'], - help='Add package files to a filesystem view via symbolic links.'), - ssp.add_parser('hardlink', aliases=['hard'], - help='Add packages files to a filesystem via via hard links.'), - ssp.add_parser('remove', aliases=['rm'], - help='Remove packages from a filesystem view.'), - ssp.add_parser('statlink', aliases=['status','check'], - help='Check status of packages in a filesystem view.') + ssp.add_parser( + 'symlink', aliases=['add', 'soft'], + help='Add package files to a filesystem view via symbolic links.'), + ssp.add_parser( + 'hardlink', aliases=['hard'], + help='Add packages files to a filesystem via via hard links.'), + ssp.add_parser( + 'remove', aliases=['rm'], + help='Remove packages from a filesystem view.'), + ssp.add_parser( + 'statlink', aliases=['status', 'check'], + help='Check status of packages in a filesystem view.') ] # All these options and arguments are common to every action. for act in file_system_view_actions: act.add_argument('path', nargs=1, help="Path to file system view directory.") act.add_argument('specs', **specs_opts) - + # The formatted print action. act = ssp.add_parser('print', help="Print a string to stdout based on given format") - act.add_argument('format', nargs=1, + act.add_argument('format', nargs=1, help="Format describing per-package printout.") act.add_argument('specs', **specs_opts) - ## Other VIEW ACTIONS might be added here. - ## Some ideas are the following (and some are redundant with existing cmds) - ## A JSON view that dumps a DAG to a JSON file - ## A DOT view that dumps to a GraphViz file - ## A SHELL INIT view that dumps bash/csh script setting up to use packages in the view return -### Util functions - def assuredir(path): 'Assure path exists as a directory' if not os.path.exists(path): os.makedirs(path) - + + def relative_to(prefix, path): 'Return end of `path` relative to `prefix`' assert 0 == path.find(prefix) @@ -139,31 +141,36 @@ def relative_to(prefix, path): reldir = reldir[1:] return reldir + def transform_path(spec, path, prefix=None): 'Return the a relative path corresponding to given path spec.prefix' if os.path.isabs(path): path = relative_to(spec.prefix, path) subdirs = path.split(os.path.sep) if subdirs[0] == '.spack': - lst = ['.spack',spec.name]+subdirs[1:] + lst = ['.spack', spec.name] + subdirs[1:] path = os.path.join(*lst) if prefix: path = os.path.join(prefix, path) return path + def purge_empty_directories(path): - 'Ascend up from the leaves accessible from `path` and remove empty directories.' + '''Ascend up from the leaves accessible from `path` + and remove empty directories.''' for dirpath, subdirs, files in os.walk(path, topdown=False): for sd in subdirs: - sdp = os.path.join(dirpath,sd) + sdp = os.path.join(dirpath, sd) try: os.rmdir(sdp) except OSError: pass + def filter_exclude(specs, exclude): 'Filter specs given sequence of exclude regex' to_exclude = [re.compile(e) for e in exclude] + def exclude(spec): for e in to_exclude: if e.match(spec.name): @@ -171,6 +178,7 @@ def exclude(spec): return False return [s for s in specs if not exclude(s)] + def flatten(seeds, descend=True): 'Normalize and flattend seed specs and descend hiearchy' flat = set() @@ -181,6 +189,7 @@ def flatten(seeds, descend=True): flat.update(spec.normalized().traverse()) return flat + def spec2dict(spec): 'Convert info in a spec into a simple dictionary.' @@ -188,44 +197,45 @@ def spec2dict(spec): # some things need processing or are properties. # pkg = spec.package - ret = dict(name = spec.name, - spec = spec.short_spec, - colorspec = spec.cshort_spec, # color - root = spec.root, - prefix = spec.prefix, - version = spec.version, - variants = spec.variants, - namespace = spec.namespace, - compiler = spec.compiler, - architecture = spec.architecture, - dependencies = ','.join(spec.dependencies.keys()), - dependents = ','.join(spec.dependents.keys()), - external = spec.external or "False", - hash = spec.dag_hash(), + inst_deps = ','.join([s.name for s in pkg.installed_dependents]), + ret = dict(name=spec.name, + spec=spec.short_spec, + colorspec=spec.cshort_spec, # color + root=spec.root, + prefix=spec.prefix, + version=spec.version, + variants=spec.variants, + namespace=spec.namespace, + compiler=spec.compiler, + architecture=spec.architecture, + dependencies=','.join(spec.dependencies.keys()), + dependents=','.join(spec.dependents.keys()), + external=spec.external or "False", + hash=spec.dag_hash(), # package related: - url = pkg.url, - stage = pkg.stage.path, - installed = pkg.installed, - installed_dependents = ','.join([s.name for s in pkg.installed_dependents]), - build_log = pkg.build_log_path, - rpath = ':'.join(pkg.rpath), + url=pkg.url, + stage=pkg.stage.path, + installed=pkg.installed, + installed_dependents=inst_deps, + build_log=pkg.build_log_path, + rpath=':'.join(pkg.rpath), # ... ) return ret -### Action-specific helpers def check_one(spec, path, verbose=False): 'Check status of view in path against spec' dotspack = os.path.join(path, '.spack', spec.name) if os.path.exists(os.path.join(dotspack)): - tty.info('Package in view: "%s"'%spec.name) + tty.info('Package in view: "%s"' % spec.name) return - tty.info('Package not in view: "%s"'%spec.name) + tty.info('Package not in view: "%s"' % spec.name) return + def remove_one(spec, path, verbose=False): 'Remove any files found in `spec` from `path` and purge empty directories.' @@ -235,12 +245,12 @@ def remove_one(spec, path, verbose=False): dotspack = transform_path(spec, '.spack', path) if not os.path.exists(dotspack): if verbose: - tty.info('Skipping nonexistent package: "%s"'%spec.name) + tty.info('Skipping nonexistent package: "%s"' % spec.name) return if verbose: - tty.info('Removing package: "%s"'%spec.name) - for dirpath,dirnames,filenames in os.walk(spec.prefix): + tty.info('Removing package: "%s"' % spec.name) + for dirpath, dirnames, filenames in os.walk(spec.prefix): if not filenames: continue targdir = transform_path(spec, dirpath, path) @@ -250,17 +260,18 @@ def remove_one(spec, path, verbose=False): continue os.unlink(dst) -def link_one(spec, path, link = os.symlink, verbose=False): + +def link_one(spec, path, link=os.symlink, verbose=False): 'Link all files in `spec` into directory `path`.' dotspack = transform_path(spec, '.spack', path) if os.path.exists(dotspack): - tty.warn('Skipping existing package: "%s"'%spec.name) + tty.warn('Skipping existing package: "%s"' % spec.name) return if verbose: tty.info('Linking package: "%s"' % spec.name) - for dirpath,dirnames,filenames in os.walk(spec.prefix): + for dirpath, dirnames, filenames in os.walk(spec.prefix): if not filenames: continue # avoid explicitly making empty dirs @@ -275,12 +286,9 @@ def link_one(spec, path, link = os.symlink, verbose=False): continue # silence these tty.warn("Skipping existing file: %s" % dst) continue - link(src,dst) + link(src, dst) -### The canonically named visitor_* functions and their alias maps. -### One for each action. - def visitor_symlink(specs, args): 'Symlink all files found in specs' path = args.path[0] @@ -290,6 +298,7 @@ def visitor_symlink(specs, args): visitor_add = visitor_symlink visitor_soft = visitor_symlink + def visitor_hardlink(specs, args): 'Hardlink all files found in specs' path = args.path[0] @@ -298,6 +307,7 @@ def visitor_hardlink(specs, args): link_one(spec, path, os.link, verbose=args.verbose) visitor_hard = visitor_hardlink + def visitor_remove(specs, args): 'Remove all files and directories found in specs from args.path' path = args.path[0] @@ -306,6 +316,7 @@ def visitor_remove(specs, args): purge_empty_directories(path) visitor_rm = visitor_remove + def visitor_statlink(specs, args): 'Give status of view in args.path relative to specs' path = args.path[0] @@ -314,6 +325,7 @@ def visitor_statlink(specs, args): visitor_status = visitor_statlink visitor_check = visitor_statlink + def visitor_print(specs, args): 'Print a string for each spec using args.format.' fmt = args.format[0] @@ -325,25 +337,22 @@ def visitor_print(specs, args): try: text = t.substitute(kwds) except KeyError: - tty.error("Format error, use keywords: %s" % (', '.join(kwds.keys()), )) + tty.error("Format error, use keywords: %s" % + (', '.join(kwds.keys()), )) raise # argparser escapes these text = text.replace(r'\n', '\n').replace(r'\t', '\t') sys.stdout.write(text) -# Finally, the actual "view" command. There should be no need to -# modify anything below when new actions are added. - - def view(parser, args): 'Produce a view of a set of packages.' - # Process common args + # Process common args seeds = [spack.cmd.disambiguate_spec(s) for s in args.specs] - specs = flatten(seeds, args.dependencies.lower() in ['yes','true']) + specs = flatten(seeds, args.dependencies.lower() in ['yes', 'true']) specs = filter_exclude(specs, args.exclude) - + # Execute the visitation. try: visitor = globals()['visitor_' + args.action] From 7705603f7302f7e7c1076d3aff53cb1829fb8e74 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Mon, 16 May 2016 09:43:46 -0700 Subject: [PATCH 233/481] Added more compilers to mock packages tests --- lib/spack/spack/test/mock_packages_test.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index 612008ac58..81b56daacd 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -47,6 +47,17 @@ f77: None fc: None modules: 'None' + clang3.3SUSE: + spec: clang@3.3 + operating_system: + name: SuSE + version: '11' + paths: + cc: /path/to/clang + cxx: /path/to/clang++ + f77: None + fc: None + modules: 'None' clang3.3RHL: spec: clang@3.3 operating_system: @@ -69,6 +80,17 @@ version: '10' spec: gcc@4.5.0 modules: 'None' + gcc4.5.0SUSE: + paths: + cc: /path/to/gcc + cxx: /path/to/g++ + f77: /path/to/gfortran + fc: /path/to/gfortran + operating_system: + name: SuSE + version: '11' + spec: gcc@4.5.0 + modules: 'None' gcc4.5.0RHL: paths: cc: /path/to/gcc From 0fd9cdb861b9b794cc6aa20cd5ecca1989a43820 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Mon, 16 May 2016 09:56:07 -0700 Subject: [PATCH 234/481] Fixed config tests for new compiler config format --- lib/spack/spack/test/config.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index 2a8d642584..daa4c572bd 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -138,11 +138,17 @@ def check_config(self, comps, *compiler_names): """Check that named compilers in comps match Spack's config.""" config = spack.config.get_config('compilers') compiler_list = ['cc', 'cxx', 'f77', 'fc'] - for key in compiler_names: - for c in compiler_list: - expected = comps['all'][key][c] - actual = config['all'][key][c] - self.assertEqual(expected, actual) + param_list = ['modules', 'paths', 'spec', 'operating_system'] + for alias, compiler in config.items(): + if compiler['spec'] in compiler_names: + for p in param_list: + expected = comps[alias][p] + actual = config[alias][p] + self.assertEqual(expected, actual) + for c in compiler_list: + expected = comps[alias]['paths'][c] + actual = config[alias]['paths'][c] + self.assertEqual(expected, actual) def test_write_key_in_memory(self): # Write b_comps "on top of" a_comps. From 7b9b18f03c7654271b865e4a3a0117124331aa0c Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 17 May 2016 13:14:59 -0500 Subject: [PATCH 235/481] Add HPL benchmark package --- .../repos/builtin/packages/hpl/package.py | 111 ++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 var/spack/repos/builtin/packages/hpl/package.py diff --git a/var/spack/repos/builtin/packages/hpl/package.py b/var/spack/repos/builtin/packages/hpl/package.py new file mode 100644 index 0000000000..1fe3e6ea32 --- /dev/null +++ b/var/spack/repos/builtin/packages/hpl/package.py @@ -0,0 +1,111 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * +import os +import platform + + +class Hpl(Package): + """HPL is a software package that solves a (random) dense linear system + in double precision (64 bits) arithmetic on distributed-memory computers. + It can thus be regarded as a portable as well as freely available + implementation of the High Performance Computing Linpack Benchmark.""" + + homepage = "http://www.netlib.org/benchmark/hpl/" + url = "http://www.netlib.org/benchmark/hpl/hpl-2.2.tar.gz" + + version('2.2', '0eb19e787c3dc8f4058db22c9e0c5320') + + variant('openmp', default=False, description='Enable OpenMP support') + + depends_on('mpi@1.1:') + depends_on('blas') + + def configure(self, spec, arch): + # List of configuration options + # Order is important + config = [] + + # OpenMP support + if '+openmp' in spec: + config.append( + 'OMP_DEFS = {0}'.format(self.compiler.openmp_flag) + ) + + config.extend([ + # Shell + 'SHELL = /bin/sh', + 'CD = cd', + 'CP = cp', + 'LN_S = ln -fs', + 'MKDIR = mkdir -p', + 'RM = /bin/rm -f', + 'TOUCH = touch', + # Platform identifier + 'ARCH = {0}'.format(arch), + # HPL Directory Structure / HPL library + 'TOPdir = {0}'.format(os.getcwd()), + 'INCdir = $(TOPdir)/include', + 'BINdir = $(TOPdir)/bin/$(ARCH)', + 'LIBdir = $(TOPdir)/lib/$(ARCH)', + 'HPLlib = $(LIBdir)/libhpl.a', + # Message Passing library (MPI) + 'MPinc = -I{0}'.format(spec['mpi'].prefix.include), + 'MPlib = -L{0}'.format(spec['mpi'].prefix.lib), + # Linear Algebra library (BLAS or VSIPL) + 'LAinc = {0}'.format(spec['blas'].prefix.include), + 'LAlib = {0}'.format(spec['blas'].blas_shared_lib), + # F77 / C interface + 'F2CDEFS = -DAdd_ -DF77_INTEGER=int -DStringSunStyle', + # HPL includes / libraries / specifics + 'HPL_INCLUDES = -I$(INCdir) -I$(INCdir)/$(ARCH) ' + + '-I$(LAinc) -I$(MPinc)', + 'HPL_LIBS = $(HPLlib) $(LAlib) $(MPlib)', + 'HPL_OPTS = -DHPL_DETAILED_TIMING -DHPL_PROGRESS_REPORT', + 'HPL_DEFS = $(F2CDEFS) $(HPL_OPTS) $(HPL_INCLUDES)', + # Compilers / linkers - Optimization flags + 'CC = {0}'.format(spec['mpi'].mpicc), + 'CCNOOPT = $(HPL_DEFS)', + 'CCFLAGS = $(HPL_DEFS) -O3', + 'LINKER = $(CC)', + 'LINKFLAGS = $(CCFLAGS) $(OMP_DEFS)', + 'ARCHIVER = ar', + 'ARFLAGS = r', + 'RANLIB = echo' + ]) + + # Write configuration options to include file + with open('Make.{0}'.format(arch), 'w') as makefile: + for var in config: + makefile.write('{0}\n'.format(var)) + + + def install(self, spec, prefix): + arch = '{0}-{1}'.format(platform.system(), platform.processor()) + + self.configure(spec, arch) + + make('arch={0}'.format(arch)) + install_tree(join_path('bin', arch), prefix.bin) From 631cbdbda89a1a227dcb0277478195a950984240 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 17 May 2016 13:22:00 -0500 Subject: [PATCH 236/481] Flake8 --- var/spack/repos/builtin/packages/hpl/package.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/hpl/package.py b/var/spack/repos/builtin/packages/hpl/package.py index 1fe3e6ea32..3a469d842e 100644 --- a/var/spack/repos/builtin/packages/hpl/package.py +++ b/var/spack/repos/builtin/packages/hpl/package.py @@ -81,7 +81,7 @@ def configure(self, spec, arch): 'F2CDEFS = -DAdd_ -DF77_INTEGER=int -DStringSunStyle', # HPL includes / libraries / specifics 'HPL_INCLUDES = -I$(INCdir) -I$(INCdir)/$(ARCH) ' + - '-I$(LAinc) -I$(MPinc)', + '-I$(LAinc) -I$(MPinc)', 'HPL_LIBS = $(HPLlib) $(LAlib) $(MPlib)', 'HPL_OPTS = -DHPL_DETAILED_TIMING -DHPL_PROGRESS_REPORT', 'HPL_DEFS = $(F2CDEFS) $(HPL_OPTS) $(HPL_INCLUDES)', @@ -101,7 +101,6 @@ def configure(self, spec, arch): for var in config: makefile.write('{0}\n'.format(var)) - def install(self, spec, prefix): arch = '{0}-{1}'.format(platform.system(), platform.processor()) From f16228d811e4be68f4e4704ff3c2eaee86c936d6 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 17 May 2016 13:36:38 -0500 Subject: [PATCH 237/481] Add more install directories --- var/spack/repos/builtin/packages/hpl/package.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/var/spack/repos/builtin/packages/hpl/package.py b/var/spack/repos/builtin/packages/hpl/package.py index 3a469d842e..3d6e996323 100644 --- a/var/spack/repos/builtin/packages/hpl/package.py +++ b/var/spack/repos/builtin/packages/hpl/package.py @@ -102,9 +102,16 @@ def configure(self, spec, arch): makefile.write('{0}\n'.format(var)) def install(self, spec, prefix): + # Arch used for file naming purposes only arch = '{0}-{1}'.format(platform.system(), platform.processor()) + # Generate Makefile include self.configure(spec, arch) make('arch={0}'.format(arch)) + + # Manual installation install_tree(join_path('bin', arch), prefix.bin) + install_tree(join_path('lib', arch), prefix.lib) + install_tree(join_path('include', arch), prefix.include) + install_tree('man', prefix.man) From 582d01784aa44024da598187c654947bfd2e55de Mon Sep 17 00:00:00 2001 From: Greg Lee Date: Tue, 17 May 2016 21:18:34 -0700 Subject: [PATCH 238/481] added newdtags variant, default to false (i.e., remove --enable-new-dtags) --- .../packages/intel-parallel-studio/package.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/var/spack/repos/builtin/packages/intel-parallel-studio/package.py b/var/spack/repos/builtin/packages/intel-parallel-studio/package.py index a176af101d..493ca16417 100644 --- a/var/spack/repos/builtin/packages/intel-parallel-studio/package.py +++ b/var/spack/repos/builtin/packages/intel-parallel-studio/package.py @@ -32,6 +32,8 @@ class IntelParallelStudio(IntelInstaller): url="file://%s/parallel_studio_xe_2016_update3.tgz" % os.getcwd()) # NOQA: ignore=E501 variant('rpath', default=True, description="Add rpath to .cfg files") + variant('newdtags', default=False, + description="Allow use of --enable-new-dtags in MPI wrappers") variant('all', default=False, description="Install all files with the requested edition") variant('mpi', default=True, @@ -116,6 +118,20 @@ def install(self, spec, prefix): spec.satisfies('@cluster'): os.symlink(self.global_license_file, os.path.join( self.prefix, "itac_latest", "license.lic")) + if spec.satisfies('~newdtags'): + wrappers = ["mpif77", "mpif77", "mpif90", "mpif90", + "mpigcc", "mpigcc", "mpigxx", "mpigxx", + "mpiicc", "mpiicc", "mpiicpc", "mpiicpc", + "mpiifort", "mpiifort"] + wrapper_paths = [] + for root, dirs, files in os.walk(spec.prefix): + for name in files: + if name in wrappers: + wrapper_paths.append(os.path.join(spec.prefix, + root, name)) + for wrapper in wrapper_paths: + filter_file(r'-Xlinker --enable-new-dtags', r' ', + wrapper) if spec.satisfies('+rpath'): for compiler_command in ["icc", "icpc", "ifort"]: From 80310a3b7c09c8b59b3414b0c20aed6c36620675 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 18 May 2016 16:50:50 -0700 Subject: [PATCH 239/481] Finished merge on these files --- lib/spack/env/cc | 44 +++++++-------- lib/spack/spack/compilers/pgi.py | 13 ++--- lib/spack/spack/config.py | 46 +++++++++------ lib/spack/spack/spec.py | 97 ++++++++++++++++---------------- 4 files changed, 105 insertions(+), 95 deletions(-) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index 9758b74f37..1e405ae6e9 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -174,6 +174,28 @@ if [[ -z $command ]]; then die "ERROR: Compiler '$SPACK_COMPILER_SPEC' does not support compiling $language programs." fi +# +# Filter '.' and Spack environment directories out of PATH so that +# this script doesn't just call itself +# +IFS=':' read -ra env_path <<< "$PATH" +IFS=':' read -ra spack_env_dirs <<< "$SPACK_ENV_PATH" +spack_env_dirs+=("" ".") +PATH="" +for dir in "${env_path[@]}"; do + addpath=true + for env_dir in "${spack_env_dirs[@]}"; do + if [[ $dir == $env_dir ]]; then + addpath=false + break + fi + done + if $addpath; then + PATH="${PATH:+$PATH:}$dir" + fi +done +export PATH + if [[ $mode == vcheck ]]; then exec ${command} "$@" fi @@ -286,28 +308,6 @@ unset LD_LIBRARY_PATH unset LD_RUN_PATH unset DYLD_LIBRARY_PATH -# -# Filter '.' and Spack environment directories out of PATH so that -# this script doesn't just call itself -# -IFS=':' read -ra env_path <<< "$PATH" -IFS=':' read -ra spack_env_dirs <<< "$SPACK_ENV_PATH" -spack_env_dirs+=("" ".") -PATH="" -for dir in "${env_path[@]}"; do - addpath=true - for env_dir in "${spack_env_dirs[@]}"; do - if [[ $dir == $env_dir ]]; then - addpath=false - break - fi - done - if $addpath; then - PATH="${PATH:+$PATH:}$dir" - fi -done -export PATH - full_command=("$command" "${args[@]}") # In test command mode, write out full command for Spack tests. diff --git a/lib/spack/spack/compilers/pgi.py b/lib/spack/spack/compilers/pgi.py index d3fad15b0c..6d36d8bfa6 100644 --- a/lib/spack/spack/compilers/pgi.py +++ b/lib/spack/spack/compilers/pgi.py @@ -44,7 +44,12 @@ class Pgi(Compiler): 'f77' : 'pgi/pgfortran', 'fc' : 'pgi/pgfortran' } -#ifdef NEW + + + PrgEnv = 'PrgEnv-pgi' + PrgEnv_compiler = 'pgi' + + @property def openmp_flag(self): return "-mp" @@ -53,12 +58,6 @@ def openmp_flag(self): def cxx11_flag(self): return "-std=c++11" - -#else /* not NEW */ - PrgEnv = 'PrgEnv-pgi' - PrgEnv_compiler = 'pgi' - -#endif /* not NEW */ @classmethod def default_version(cls, comp): """The '-V' option works for all the PGI compilers. diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 3d6c175c7c..ec04c81787 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -167,6 +167,18 @@ 'f77': { 'anyOf': [ {'type' : 'string' }, {'type' : 'null' }]}, 'fc': { 'anyOf': [ {'type' : 'string' }, + {'type' : 'null' }]}, + 'cflags': { 'anyOf': [ {'type' : 'string' }, + {'type' : 'null' }]}, + 'cxxflags': { 'anyOf': [ {'type' : 'string' }, + {'type' : 'null' }]}, + 'fflags': { 'anyOf': [ {'type' : 'string' }, + {'type' : 'null' }]}, + 'cppflags': { 'anyOf': [ {'type' : 'string' }, + {'type' : 'null' }]}, + 'ldflags': { 'anyOf': [ {'type' : 'string' }, + {'type' : 'null' }]}, + 'ldlibs': { 'anyOf': [ {'type' : 'string' }, {'type' : 'null' }]}}}, 'spec': { 'type': 'string'},#r'\w[\w-]*@\w[\w-]*' 'operating_system': { @@ -234,15 +246,11 @@ 'items' : { 'type' : 'string' } }, #compiler specs 'buildable': { 'type': 'boolean', -#ifdef NEW 'default': True, -#else /* not NEW */ - 'default': False, }, - 'module': { - 'anyOf' : [{'type': 'string'}, - {'type': 'null'}] -#endif /* not NEW */ + 'modules': { + 'type' : 'object', + 'default' : {}, }, 'providers': { 'type': 'object', @@ -687,7 +695,8 @@ def spec_externals(spec): external_specs = [] pkg_paths = allpkgs.get(name, {}).get('paths', None) - if not pkg_paths: + pkg_modules = allpkgs.get(name, {}).get('modules', None) + if (not pkg_paths) and (not pkg_modules): return [] for external_spec, path in pkg_paths.iteritems(): @@ -695,20 +704,21 @@ def spec_externals(spec): # skip entries without paths (avoid creating extra Specs) continue -#ifdef NEW external_spec = spack.spec.Spec(external_spec, external=path) if external_spec.satisfies(spec): external_specs.append(external_spec) + + for external_spec, module in pkg_modules.iteritems(): + if not module: + continue + + path = get_path_from_module(module) + + external_spec = spack.spec.Spec(external_spec, external=path, external_module=module) + if external_spec.satisfies(spec): + external_specs.append(external_spec) + return external_specs -#else /* not NEW */ - module = allpkgs.get(pkg, {}).get('module', None) - if not path: - if not module: - continue - path = get_path_from_module(module) - spec_locations.append( (spack.spec.Spec(pkg), path, module) ) - return spec_locations -#endif /* not NEW */ def is_spec_buildable(spec): diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index f71d56435e..e61ffc0912 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -497,14 +497,10 @@ def __init__(self, spec_like, *dep_like, **kwargs): # package.py files for. self._normal = kwargs.get('normal', False) self._concrete = kwargs.get('concrete', False) -#ifdef NEW # Allow a spec to be constructed with an external path. self.external = kwargs.get('external', None) -#else /* not NEW */ - self.external = None - self.external_module = None -#endif /* not NEW */ + self.external_module = kwargs.get('external_module', None) # This allows users to construct a spec DAG with literals. # Note that given two specs a and b, Spec(a) copies a, but @@ -538,8 +534,10 @@ def _add_flag(self, name, value): Known flags currently include "arch" """ valid_flags = FlagMap.valid_compiler_flags() - if name == 'arch': - self._set_architecture(value) + if name == 'os' or name == 'operating_system': + self._set_os(value) + elif name == 'target': + self._set_target(value) elif name in valid_flags: assert(self.compiler_flags is not None) self.compiler_flags[name] = value.split() @@ -553,12 +551,13 @@ def _set_compiler(self, compiler): self.compiler = compiler - def _set_architecture(self, architecture): - """Called by the parser to set the architecture.""" - if self.architecture: raise DuplicateArchitectureError( - "Spec for '%s' cannot have two architectures." % self.name) - self.architecture = architecture + def _set_os(self, value): + """Called by the parser to set the architecture operating system""" + self.architecture.platform_os = self.architecture.platform.operating_system(value) + def _set_target(self, value): + """Called by the parser to set the architecture target""" + self.architecture.target = self.architecture.platform.target(value) def _add_dependency(self, spec): """Called by the parser to add another spec as a dependency.""" @@ -793,9 +792,9 @@ def to_node_dict(self): if self.architecture: # TODO: Fix the target.to_dict to account for the tuple # Want it to be a dict of dicts - d['architecture'] = self.architecture.to_dict() + d['arch'] = self.architecture.to_dict() else: - d['architecture'] = None + d['arch'] = None if self.compiler: d.update(self.compiler.to_dict()) @@ -824,17 +823,12 @@ def from_node_dict(node): spec = Spec(name) spec.namespace = node.get('namespace', None) spec.versions = VersionList.from_dict(node) -#ifdef NEW - spec.architecture = node['arch'] if 'hash' in node: spec._hash = node['hash'] -#else /* not NEW */ - # TODO: Need to fix the architecture.Target.from_dict - spec.architecture = spack.architecture.arch_from_dict(node['architecture']) + spec.architecture = spack.architecture.arch_from_dict(node['arch']) -#endif /* not NEW */ if node['compiler'] is None: spec.compiler = None else: @@ -1423,9 +1417,19 @@ def constrain(self, other, deps=True): # TODO: Check out the logic here if self.architecture is not None and other.architecture is not None: - if self.architecture != other.architecture: - raise UnsatisfiableArchitectureSpecError(self.architecture, - other.architecture) + if self.architecture.platform is not None and other.architecture.platform is not None: + if self.architecture.platform != other.architecture.platform: + raise UnsatisfiableArchitectureSpecError(self.architecture, + other.architecture) + if self.architecture.platform_os is not None and other.architecture.platform_os is not None: + if self.architecture.platform_os != other.architecture.platform_os: + raise UnsatisfiableArchitectureSpecError(self.architecture, + other.architecture) + if self.architecture.target is not None and other.architecture.target is not None: + if self.architecture.target != other.architecture.target: + raise UnsatisfiableArchitectureSpecError(self.architecture, + other.architecture) + changed = False if self.compiler is not None and other.compiler is not None: @@ -1440,7 +1444,14 @@ def constrain(self, other, deps=True): changed |= self.compiler_flags.constrain(other.compiler_flags) old = self.architecture - self.architecture = self.architecture or other.architecture + if self.architecture is None or other.architecture is None: + self.architecture = self.architecture or other.architecture + elif self.architecture.platform is None or other.architecture.platform is None: + self.architecture.platform = self.architecture.platform or other.architecture.platform + elif self.architecture.platform_os is None of other.architecture.platform_os is None: + self.architecture.platform_os = self.architecture.platform_os or other.architecture.platform_os + elif self.architecture.target is None or other.architecture.target is None: + self.architecture.target = self.architecture.target or other.architecture.target changed |= (self.architecture != old) if deps: @@ -1572,16 +1583,18 @@ def satisfies(self, other, deps=True, strict=False): # Architecture satisfaction is currently just string equality. # If not strict, None means unconstrained. - if isinstance(self.architecture, basestring): - self.add_architecture_from_string(self.architecture) - if isinstance(other.architecture, basestring): - other.add_architecture_from_string(other.architecture) + # TODO: Need to make sure that comparisons can be made via classes if self.architecture and other.architecture: - if self.architecture != other.architecture: + if ((self.architecture.platform and other.architecture.platform and self.architecture.platform != other.architecture.platform) or + (self.architecture.platform_os and other.architecture.platform_os and self.architecture.platform_os != other.architecture.platform_os) or + (self.architecture.target and other.architecture.target and self.architecture.target != other.architecture.target)): return False - elif strict and (other.architecture and not self.architecture): + elif strict and ((other.architecture and not self.architecture) or + (other.architecture.platform and not self.architecture.platform) or + (other.architecture.platform_os and not self.architecture.platform_os) or + (other.architecture.target and not self.architecture.target)): return False if not self.compiler_flags.satisfies(other.compiler_flags, strict=strict): @@ -1663,7 +1676,7 @@ def _dup(self, other, **kwargs): self.architecture != other.architecture and self.compiler != other.compiler and \ self.variants != other.variants and self._normal != other._normal and \ self.concrete != other.concrete and self.external != other.external and \ - self.external_module != other.external_module) + self.external_module != other.external_module and self.compiler_flags != other.compiler_flags) # Local node attributes get copied first. self.name = other.name @@ -1677,12 +1690,9 @@ def _dup(self, other, **kwargs): self.variants = other.variants.copy() self.variants.spec = self self.external = other.external - self.namespace = other.namespace -#ifdef NEW - self._hash = other._hash -#else /* not NEW */ self.external_module = other.external_module -#endif /* not NEW */ + self.namespace = other.namespace + self._hash = other._hash # If we copy dependencies, preserve DAG structure in the new spec if kwargs.get('deps', True): @@ -1983,7 +1993,7 @@ def write(s, c): write(fmt % str(self.variants), '+') elif named_str == 'ARCHITECTURE': if self.architecture: - write(fmt % str(self.architecture), '=') + write(fmt % str(self.architecture), ' arch=') elif named_str == 'SHA1': if self.dependencies: out.write(fmt % str(self.dag_hash(7))) @@ -2202,14 +2212,11 @@ def spec(self, name, check_valid_token = False): spec.name = spec_name spec.versions = VersionList() spec.variants = VariantMap(spec) - spec.architecture = None + spec.architecture = spack.architecture.Arch() spec.compiler = None spec.external = None -#ifdef NEW - spec.compiler_flags = FlagMap(spec) -#else /* not NEW */ spec.external_module = None -#endif /* not NEW */ + spec.compiler_flags = FlagMap(spec) spec.dependents = DependencyMap() spec.dependencies = DependencyMap() spec.namespace = spec_namespace @@ -2284,12 +2291,6 @@ def variant(self,name=None): self.check_identifier() return self.token.value - def architecture(self): - #TODO: Make this work properly as a subcase of variant (includes adding names to grammar) - self.expect(ID) - return self.token.value - - def version(self): start = None end = None From faa0f2a13c01a9f279d59912cff5d00c499cb990 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 18 May 2016 17:01:59 -0700 Subject: [PATCH 240/481] got rid of ifdef --- lib/spack/spack/cmd/find.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/spack/spack/cmd/find.py b/lib/spack/spack/cmd/find.py index c768faea91..8b91f45cfd 100644 --- a/lib/spack/spack/cmd/find.py +++ b/lib/spack/spack/cmd/find.py @@ -239,8 +239,5 @@ def find(parser, args): mode=args.mode, long=args.long, very_long=args.very_long, -#ifdef NEW show_flags=args.show_flags) -#else /* not NEW */ namespace=args.namespace) -#endif /* not NEW */ From 9f8ff32bcce99c8a188f285e235e9c32ee245f97 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 18 May 2016 17:05:11 -0700 Subject: [PATCH 241/481] Got rid of ifdefs, changed parameters for Compiler: added kwargs --- lib/spack/spack/compiler.py | 69 +++++++++---------------------------- 1 file changed, 17 insertions(+), 52 deletions(-) diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index dfae72010a..ce4555bc56 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -101,7 +101,6 @@ def cc_rpath_arg(self): def cxx_rpath_arg(self): return '-Wl,-rpath,' -#ifdef NEW @property def f77_rpath_arg(self): return '-Wl,-rpath,' @@ -109,45 +108,19 @@ def f77_rpath_arg(self): @property def fc_rpath_arg(self): return '-Wl,-rpath,' -#else /* not NEW */ # Cray PrgEnv name that can be used to load this compiler PrgEnv = None - # Name of module used to switch versions of this compiler PrgEnv_compiler = None -#endif /* not NEW */ -#ifdef NEW - - def __init__(self, cspec, cc, cxx, f77, fc, **kwargs): -#else /* not NEW */ - - def __init__(self, cspec, operating_system, paths, modules=[], alias=None): -#endif /* not NEW */ + def __init__(self, cspec, operating_system, + paths, modules=[], alias=None, **kwargs): def check(exe): if exe is None: return None _verify_executables(exe) return exe -#ifdef NEW - self.cc = check(cc) - self.cxx = check(cxx) - self.f77 = check(f77) - self.fc = check(fc) - - # Unfortunately have to make sure these params are accepted - # in the same order they are returned by sorted(flags) - # in compilers/__init__.py - self.flags = {} - for flag in spack.spec.FlagMap.valid_compiler_flags(): - value = kwargs.get(flag, None) - if value is not None: - self.flags[flag] = value.split() - -#else /* not NEW */ - self.operating_system = operating_system - self.cc = check(paths[0]) self.cxx = check(paths[1]) if len(paths) > 2: @@ -157,7 +130,21 @@ def check(exe): else: self.fc = check(paths[3]) -#endif /* not NEW */ + #self.cc = check(cc) + #self.cxx = check(cxx) + #self.f77 = check(f77) + #self.fc = check(fc) + + # Unfortunately have to make sure these params are accepted + # in the same order they are returned by sorted(flags) + # in compilers/__init__.py + self.flags = {} + for flag in spack.spec.FlagMap.valid_compiler_flags(): + value = kwargs.get(flag, None) + if value is not None: + self.flags[flag] = value.split() + + self.operating_system = operating_system self.spec = cspec self.modules = modules self.alias = alias @@ -287,28 +274,6 @@ def check(key): successful.reverse() return dict(((v, p, s), path) for v, p, s, path in successful) - @classmethod - def default_version(cls, cc): - """Override just this to override all compiler version functions.""" - return dumpversion(cc) - - @classmethod - def cc_version(cls, cc): - return cls.default_version(cc) - - @classmethod - def cxx_version(cls, cxx): - return cls.default_version(cxx) - - @classmethod - def f77_version(cls, f77): - return cls.default_version(f77) - - @classmethod - def fc_version(cls, fc): - return cls.default_version(fc) - - def __repr__(self): """Return a string representation of the compiler toolchain.""" return self.__str__() From b4b794223065b3301da9f89eab536bf8d162c150 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 18 May 2016 17:07:23 -0700 Subject: [PATCH 242/481] Moved searching of flags into get compilers in compilers_for_spec --- lib/spack/spack/compilers/__init__.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 000ef2058e..3432c142bf 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -254,17 +254,18 @@ def get_compilers(cspec): else: operating_system = None - compilers.append(cls(cspec, operating_system, compiler_paths, mods, alias)) + flags = {} + for f in spack.spec.FlagMap.valid_compiler_flags(): + if f in items: + flags[f] = items[f] -#ifdef NEW - flags = {} - for f in spack.spec.FlagMap.valid_compiler_flags(): - if f in items: - flags[f] = items[f] - return cls(cspec, *compiler_paths, **flags) -#else /* not NEW */ - return compilers -#endif /* not NEW */ + compilers.append(cls(cspec, operating_system, compiler_paths, mods, alias, **flags)) + +##ifdef NEW +# return cls(cspec, *compiler_paths, **flags) +##else /* not NEW */ +# return compilers +##endif /* not NEW */ matches = set(find(compiler_spec, scope)) compilers = [] From deb5011d08467c0a19b52e5ee9fefa3fb0854b89 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 18 May 2016 17:14:16 -0700 Subject: [PATCH 243/481] more merge work --- lib/spack/spack/build_environment.py | 23 ++++++++--------------- lib/spack/spack/spec.py | 10 ++++++++++ 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 4974cc1e76..7aeea5c672 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -213,13 +213,12 @@ def set_compiler_environment_variables(pkg): if flags[flag] != []: env.set('SPACK_' + flag.upper(), ' '.join(f for f in flags[flag])) -#ifdef NEW env.set('SPACK_COMPILER_SPEC', str(pkg.spec.compiler)) - return env -#else /* not NEW */ + for mod in compiler.modules: load_module(mod) -#endif /* not NEW */ + + return env def set_build_environment_variables(pkg, env): @@ -283,16 +282,13 @@ def set_build_environment_variables(pkg, env): pcdir = join_path(p, maybe, 'pkgconfig') if os.path.isdir(pcdir): pkg_config_dirs.append(pcdir) -#ifdef NEW - env.set_path('PKG_CONFIG_PATH', pkg_config_dirs) - return env -#else /* not NEW */ - path_put_first("PKG_CONFIG_PATH", pkg_config_dirs) + env.prepend_path('PKG_CONFIG_PATH', pkg_config_dirs) if pkg.spec.architecture.target.module_name: load_module(pkg.spec.architecture.target.module_name) -#endif /* not NEW */ + + return env def set_module_variables_for_package(pkg, module): @@ -368,16 +364,13 @@ def set_module_variables_for_package(pkg, module): def get_rpaths(pkg): """Get a list of all the rpaths for a package.""" - for spec in pkg.spec.traverse(root=False): - if spec.external_module: - load_module(spec.external_module) - spec.external = get_path_from_module(spec.external_module) - rpaths = [pkg.prefix.lib, pkg.prefix.lib64] rpaths.extend(d.prefix.lib for d in pkg.spec.dependencies.values() if os.path.isdir(d.prefix.lib)) rpaths.extend(d.prefix.lib64 for d in pkg.spec.dependencies.values() if os.path.isdir(d.prefix.lib64)) + for mod in pkg.spec.compiler.modules: + rpaths.append(get_path_for_module(mod)) return rpaths diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index e61ffc0912..9d217d17ca 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -120,6 +120,7 @@ from spack.util.string import * from spack.util.prefix import Prefix from spack.virtual import ProviderIndex +from spack.build_environment import get_path_from_module, load_module # Valid pattern for an identifier in Spack identifier_re = r'\w[\w-]*' @@ -1085,6 +1086,15 @@ def concretize(self): if s.namespace is None: s.namespace = spack.repo.repo_for_pkg(s.name).namespace + + for s in self.traverse(root=False): + if spec.external_module: + compiler = spack.compilers.compiler_for_spec(spec.compiler, spec.architecture.platform_os) + for mod in compiler.modules: + load_module(mod) + + spec.external = get_path_from_module(spec.external_module) + # Mark everything in the spec as concrete, as well. self._mark_concrete() From b1e5eafb80c8cd87f4d496fe7fdb7a74a77da603 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 18 May 2016 17:18:26 -0700 Subject: [PATCH 244/481] more merge work --- lib/spack/spack/test/config.py | 29 ++++++++--------------------- 1 file changed, 8 insertions(+), 21 deletions(-) diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index 7d71cc1c7f..8fffc09437 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -142,19 +142,6 @@ def check_config(self, comps, *compiler_names): """Check that named compilers in comps match Spack's config.""" config = spack.config.get_config('compilers') compiler_list = ['cc', 'cxx', 'f77', 'fc'] -#ifdef NEW - for key in compiler_names: - for c in compiler_list: - expected = comps[arch][key][c] - actual = config[arch][key][c] - self.assertEqual(expected, actual) - - def test_write_list_in_memory(self): - spack.config.update_config('repos', repos_low, 'test_low_priority') - spack.config.update_config('repos', repos_high, 'test_high_priority') - config = spack.config.get_config('repos') - self.assertEqual(config, repos_high+repos_low) -#else /* not NEW */ param_list = ['modules', 'paths', 'spec', 'operating_system'] for alias, compiler in config.items(): if compiler['spec'] in compiler_names: @@ -166,7 +153,12 @@ def test_write_list_in_memory(self): expected = comps[alias]['paths'][c] actual = config[alias]['paths'][c] self.assertEqual(expected, actual) -#endif /* not NEW */ + + def test_write_list_in_memory(self): + spack.config.update_config('repos', repos_low, 'test_low_priority') + spack.config.update_config('repos', repos_high, 'test_high_priority') + config = spack.config.get_config('repos') + self.assertEqual(config, repos_high+repos_low) def test_write_key_in_memory(self): # Write b_comps "on top of" a_comps. @@ -187,9 +179,8 @@ def test_write_key_to_disk(self): spack.config.clear_config_caches() # Same check again, to ensure consistency. -#ifdef NEW - self.check_config(a_comps, 'x86_64_E5v2_IntelIB', 'gcc@4.7.3', 'gcc@4.5.0') - self.check_config(b_comps, 'x86_64_E5v3', 'icc@10.0', 'icc@11.1', 'clang@3.3') + self.check_config(a_comps, 'gcc@4.7.3', 'gcc@4.5.0') + self.check_config(b_comps, 'icc@10.0', 'icc@11.1', 'clang@3.3') def test_write_to_same_priority_file(self): # Write b_comps in the same file as a_comps. @@ -200,9 +191,5 @@ def test_write_to_same_priority_file(self): spack.config.clear_config_caches() # Same check again, to ensure consistency. - self.check_config(a_comps, 'x86_64_E5v2_IntelIB', 'gcc@4.7.3', 'gcc@4.5.0') - self.check_config(b_comps, 'x86_64_E5v3', 'icc@10.0', 'icc@11.1', 'clang@3.3') -#else /* not NEW */ self.check_config(a_comps, 'gcc@4.7.3', 'gcc@4.5.0') self.check_config(b_comps, 'icc@10.0', 'icc@11.1', 'clang@3.3') -#endif /* not NEW */ From 5417f1cdc676ab6c244cce1f3594a9620ba24b16 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 18 May 2016 17:21:03 -0700 Subject: [PATCH 245/481] more merge work --- lib/spack/spack/test/__init__.py | 40 ++--------------------------- lib/spack/spack/test/multimethod.py | 22 ++-------------- 2 files changed, 4 insertions(+), 58 deletions(-) diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py index cac7c2f058..480e6290e7 100644 --- a/lib/spack/spack/test/__init__.py +++ b/lib/spack/spack/test/__init__.py @@ -31,8 +31,8 @@ from llnl.util.tty.colify import colify from spack.test.tally_plugin import Tally """Names of tests to be included in Spack's test suite""" -#ifdef NEW -test_names = ['versions', 'url_parse', 'url_substitution', 'packages', 'stage', + +test_names = ['architecture', 'versions', 'url_parse', 'url_substitution', 'packages', 'stage', 'spec_syntax', 'spec_semantics', 'spec_dag', 'concretize', 'multimethod', 'install', 'package_sanity', 'config', 'directory_layout', 'pattern', 'python_version', 'git_fetch', @@ -41,42 +41,6 @@ 'make_executable', 'configure_guess', 'lock', 'database', 'namespace_trie', 'yaml', 'sbang', 'environment', 'cmd.uninstall', 'cmd.test_install'] -#else /* not NEW */ -test_names = ['architecture', - 'versions', - 'url_parse', - 'url_substitution', - 'packages', - 'stage', - 'spec_syntax', - 'spec_semantics', - 'spec_dag', - 'concretize', - 'multimethod', - 'install', - 'package_sanity', - 'config', - 'directory_layout', - 'pattern', - 'python_version', - 'git_fetch', - 'svn_fetch', - 'hg_fetch', - 'mirror', - 'url_extrapolate', - 'cc', - 'link_tree', - 'spec_yaml', - 'optional_deps', - 'make_executable', - 'configure_guess', - 'unit_install', - 'lock', - 'database', - 'namespace_trie', - 'yaml', - 'sbang'] -#endif /* not NEW */ def list_tests(): diff --git a/lib/spack/spack/test/multimethod.py b/lib/spack/spack/test/multimethod.py index 7636bd6c38..034e6b3923 100644 --- a/lib/spack/spack/test/multimethod.py +++ b/lib/spack/spack/test/multimethod.py @@ -92,36 +92,18 @@ def test_default_works(self): self.assertEqual(pkg.has_a_default(), 'default') -#ifdef NEW - def test_architecture_match(self): - pkg = spack.repo.get('multimethod arch=x86_64') - self.assertEqual(pkg.different_by_architecture(), 'x86_64') - - pkg = spack.repo.get('multimethod arch=ppc64') - self.assertEqual(pkg.different_by_architecture(), 'ppc64') - - pkg = spack.repo.get('multimethod arch=ppc32') - self.assertEqual(pkg.different_by_architecture(), 'ppc32') - - pkg = spack.repo.get('multimethod arch=arm64') - self.assertEqual(pkg.different_by_architecture(), 'arm64') - - pkg = spack.repo.get('multimethod arch=macos') - self.assertRaises(NoSuchMethodError, pkg.different_by_architecture) -#else /* not NEW */ def test_target_match(self): platform = spack.architecture.sys_type() targets = platform.targets.values() for target in targets[:-1]: - pkg = spack.repo.get('multimethod='+target.name) + pkg = spack.repo.get('multimethod target='+target.name) self.assertEqual(pkg.different_by_target(), target.name) - pkg = spack.repo.get('multimethod='+targets[-1].name) + pkg = spack.repo.get('multimethod target='+targets[-1].name) if len(targets) == 1: self.assertEqual(pkg.different_by_target(), targets[-1].name) else: self.assertRaises(NoSuchMethodError, pkg.different_by_target) -#endif /* not NEW */ def test_dependency_match(self): From 6926f4d0da05ee2839e455af32d52098d9828be7 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 18 May 2016 17:23:10 -0700 Subject: [PATCH 246/481] Changed arch tests to new spec target= os= --- lib/spack/spack/test/spec_semantics.py | 109 ++++++++++++------------- 1 file changed, 50 insertions(+), 59 deletions(-) diff --git a/lib/spack/spack/test/spec_semantics.py b/lib/spack/spack/test/spec_semantics.py index b6c9d61293..8cdb91e206 100644 --- a/lib/spack/spack/test/spec_semantics.py +++ b/lib/spack/spack/test/spec_semantics.py @@ -23,6 +23,7 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import unittest +import spack.architecture from spack.spec import * from spack.test.mock_packages_test import * @@ -138,23 +139,31 @@ def test_satisfies_compiler_version(self): self.check_unsatisfiable('foo %gcc@4.7', '%gcc@4.7.3') -#ifdef NEW def test_satisfies_architecture(self): - self.check_satisfies('foo arch=chaos_5_x86_64_ib', ' arch=chaos_5_x86_64_ib') - self.check_satisfies('foo arch=bgqos_0', ' arch=bgqos_0') + platform = self.architecture.sys_type() + if platform.name == 'crayxc': + self.check_satisfies('foo target=frontend os=frontend', 'target=frontend os=frontend') + self.check_satisfies('foo target=backend os=backend', 'target=backend', 'os=backend') + self.check_satisfies('foo target=default_target os=default_os','target=default_target os=default_os') - self.check_unsatisfiable('foo arch=bgqos_0', ' arch=chaos_5_x86_64_ib') - self.check_unsatisfiable('foo arch=chaos_5_x86_64_ib', ' arch=bgqos_0') -#else /* not NEW */ - def test_satisfies_target(self): - platform = spack.architecture.sys_type() - targets = platform.targets.values() - for target in targets: - self.check_satisfies('foo='+target.name, '='+target.name) -#endif /* not NEW */ - for i in range(1,len(targets)): - self.check_unsatisfiable('foo='+targets[i-1].name, '='+targets[i].name) +#ifdef NEW + #def test_satisfies_architecture(self): + # self.check_satisfies('foo arch=chaos_5_x86_64_ib', ' arch=chaos_5_x86_64_ib') + # self.check_satisfies('foo arch=bgqos_0', ' arch=bgqos_0') + + # self.check_unsatisfiable('foo arch=bgqos_0', ' arch=chaos_5_x86_64_ib') + # self.check_unsatisfiable('foo arch=chaos_5_x86_64_ib', ' arch=bgqos_0') +#els#e /* not NEW */ + #def test_satisfies_target(self): + # platform = spack.architecture.sys_type() + # targets = platform.targets.values() + # for target in targets: + # self.check_satisfies('foo='+target.name, '='+target.name) +#end#if /* not NEW */ + + # for i in range(1,len(targets)): + # self.check_unsatisfiable('foo='+targets[i-1].name, '='+targets[i].name) def test_satisfies_dependencies(self): self.check_satisfies('mpileaks^mpich', '^mpich') @@ -355,22 +364,29 @@ def test_constrain_variants(self): self.check_constrain('libelf+debug~foo', 'libelf+debug', 'libelf+debug~foo') -#ifdef NEW def test_constrain_compiler_flags(self): self.check_constrain('libelf cflags="-O3" cppflags="-Wall"', 'libelf cflags="-O3"', 'libelf cppflags="-Wall"') self.check_constrain('libelf cflags="-O3" cppflags="-Wall"', 'libelf cflags="-O3"', 'libelf cflags="-O3" cppflags="-Wall"') - def test_constrain_arch(self): - self.check_constrain('libelf arch=bgqos_0', 'libelf arch=bgqos_0', 'libelf arch=bgqos_0') - self.check_constrain('libelf arch=bgqos_0', 'libelf', 'libelf arch=bgqos_0') -#else /* not NEW */ - def test_constrain_target(self): - platform = spack.architecture.sys_type() - target = platform.target('default_target').name - self.check_constrain('libelf='+target, 'libelf='+target, 'libelf='+target) - self.check_constrain('libelf='+target, 'libelf', 'libelf='+target) -#endif /* not NEW */ + def test_constrain_architecture(self): + self.check_constrain('libelf target=default_target os=default_os', + 'libelf target=default_target os=default_os', + 'libelf target=default_target os=default_os') + self.check_constrain('libelf target=default_target os=default_os', + 'libelf', + 'libelf target=default_target os=default_os') + + #def test_constrain_arch(self): + # self.check_constrain('libelf arch=bgqos_0', 'libelf arch=bgqos_0', 'libelf arch=bgqos_0') + # self.check_constrain('libelf arch=bgqos_0', 'libelf', 'libelf arch=bgqos_0') +#els#e /* not NEW */ + #def test_constrain_target(self): + # platform = spack.architecture.sys_type() + # target = platform.target('default_target').name + # self.check_constrain('libelf='+target, 'libelf='+target, 'libelf='+target) + # self.check_constrain('libelf='+target, 'libelf', 'libelf='+target) +#end#if /* not NEW */ def test_constrain_compiler(self): @@ -384,20 +400,11 @@ def test_invalid_constraint(self): self.check_invalid_constraint('libelf+debug', 'libelf~debug') self.check_invalid_constraint('libelf+debug~foo', 'libelf+debug+foo') -#ifdef NEW self.check_invalid_constraint('libelf debug=2', 'libelf debug=1') self.check_invalid_constraint('libelf cppflags="-O3"', 'libelf cppflags="-O2"') - - self.check_invalid_constraint('libelf arch=bgqos_0', 'libelf arch=x86_54') -#else /* not NEW */ - - platform = spack.architecture.sys_type() - targets = platform.targets.values() - if len(targets) > 1: - self.check_invalid_constraint('libelf='+targets[0].name, 'libelf='+targets[1].name) -#endif /* not NEW */ - + self.check_invalid_constraint('libelf target=default_target os=default_os', + 'libelf target=x86_64 os=ubuntu') def test_constrain_changed(self): self.check_constrain_changed('libelf', '@1.0') @@ -406,14 +413,13 @@ def test_constrain_changed(self): self.check_constrain_changed('libelf%gcc', '%gcc@4.5') self.check_constrain_changed('libelf', '+debug') self.check_constrain_changed('libelf', '~debug') -#ifdef NEW self.check_constrain_changed('libelf', 'debug=2') self.check_constrain_changed('libelf', 'cppflags="-O3"') self.check_constrain_changed('libelf', ' arch=bgqos_0') -#else /* not NEW */ + platform = spack.architecture.sys_type() - self.check_constrain_changed('libelf', '='+platform.target('default_target').name) -#endif /* not NEW */ + self.check_constrain_changed('libelf', 'target='+platform.target('default_target').name) + self.check_constrain_changed('libelf', 'os='+platform.operating_system('default_os').name) def test_constrain_not_changed(self): @@ -424,17 +430,12 @@ def test_constrain_not_changed(self): self.check_constrain_not_changed('libelf%gcc@4.5', '%gcc@4.5') self.check_constrain_not_changed('libelf+debug', '+debug') self.check_constrain_not_changed('libelf~debug', '~debug') -#ifdef NEW self.check_constrain_not_changed('libelf debug=2', 'debug=2') self.check_constrain_not_changed('libelf cppflags="-O3"', 'cppflags="-O3"') - self.check_constrain_not_changed('libelf arch=bgqos_0', ' arch=bgqos_0') -#else /* not NEW */ + platform = spack.architecture.sys_type() default_target = platform.target('default_target').name - self.check_constrain_not_changed('libelf='+default_target, '='+default_target) -#endif /* not NEW */ - self.check_constrain_not_changed('libelf^foo', 'libelf^foo') - self.check_constrain_not_changed('libelf^foo^bar', 'libelf^foo^bar') + self.check_constrain_not_changed('libelf target='+default_target, 'target='+default_target) def test_constrain_dependency_changed(self): @@ -444,14 +445,9 @@ def test_constrain_dependency_changed(self): self.check_constrain_changed('libelf^foo%gcc', 'libelf^foo%gcc@4.5') self.check_constrain_changed('libelf^foo', 'libelf^foo+debug') self.check_constrain_changed('libelf^foo', 'libelf^foo~debug') -#ifdef NEW - self.check_constrain_changed('libelf^foo', 'libelf^foo cppflags="-O3"') - self.check_constrain_changed('libelf^foo', 'libelf^foo arch=bgqos_0') -#else /* not NEW */ platform = spack.architecture.sys_type() default_target = platform.target('default_target').name - self.check_constrain_changed('libelf^foo', 'libelf^foo='+default_target) -#endif /* not NEW */ + self.check_constrain_changed('libelf^foo', 'libelf^foo target='+default_target) def test_constrain_dependency_not_changed(self): @@ -461,13 +457,8 @@ def test_constrain_dependency_not_changed(self): self.check_constrain_not_changed('libelf^foo%gcc@4.5', 'libelf^foo%gcc@4.5') self.check_constrain_not_changed('libelf^foo+debug', 'libelf^foo+debug') self.check_constrain_not_changed('libelf^foo~debug', 'libelf^foo~debug') -#ifdef NEW self.check_constrain_not_changed('libelf^foo cppflags="-O3"', 'libelf^foo cppflags="-O3"') - self.check_constrain_not_changed('libelf^foo arch=bgqos_0', 'libelf^foo arch=bgqos_0') -#else /* not NEW */ platform = spack.architecture.sys_type() default_target = platform.target('default_target').name - self.check_constrain_not_changed('libelf^foo='+default_target, 'libelf^foo='+default_target) - -#endif /* not NEW */ + self.check_constrain_not_changed('libelf^foo target='+default_target, 'libelf^foo target='+default_target) From b25da51638c6c3abd97a1f77cc4feffd144a62d1 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 18 May 2016 17:26:06 -0700 Subject: [PATCH 247/481] Changed unsatisfiable architecture test --- lib/spack/spack/test/spec_dag.py | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/lib/spack/spack/test/spec_dag.py b/lib/spack/spack/test/spec_dag.py index 3c005c4ead..99a986a528 100644 --- a/lib/spack/spack/test/spec_dag.py +++ b/lib/spack/spack/test/spec_dag.py @@ -29,6 +29,7 @@ spack/lib/spack/spack/test/mock_packages """ import spack +import spack.architecture import spack.package from llnl.util.lang import list_modules @@ -240,21 +241,12 @@ def test_unsatisfiable_compiler_version(self): self.assertRaises(spack.spec.UnsatisfiableCompilerSpecError, spec.normalize) -#ifdef NEW def test_unsatisfiable_architecture(self): - self.set_pkg_dep('mpileaks', 'mpich arch=bgqos_0') - spec = Spec('mpileaks ^mpich arch=sles_10_ppc64 ^callpath ^dyninst ^libelf ^libdwarf') - self.assertRaises(spack.spec.UnsatisfiableArchitectureSpecError, spec.normalize) -#else /* not NEW */ - def test_unsatisfiable_target(self): platform = spack.architecture.sys_type() - if len(platform.targets) > 1: - first = platform.targets.values()[0].name - second = platform.targets.values()[1].name - self.set_pkg_dep('mpileaks', 'mpich='+first) - spec = Spec('mpileaks ^mpich='+ second +' ^callpath ^dyninst ^libelf ^libdwarf') - self.assertRaises(spack.spec.UnsatisfiableTargetSpecError, spec.normalize) -#endif /* not NEW */ + + self.set_pkg_dep('mpileaks', 'mpich target=%s' % platform.target('default_target')) + spec = Spec('mpileaks ^mpich target=sles_10_ppc64 ^callpath ^dyninst ^libelf ^libdwarf') + self.assertRaises(spack.spec.UnsatisfiableArchitectureSpecError, spec.normalize) def test_invalid_dep(self): From 01d5ffcd87d4801874dc810fb8123447971b867d Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 18 May 2016 17:42:11 -0700 Subject: [PATCH 248/481] Fixed the first set of merge bugs --- lib/spack/spack/architecture.py | 8 ++++---- lib/spack/spack/cmd/find.py | 2 +- lib/spack/spack/compilers/__init__.py | 6 +----- lib/spack/spack/concretize.py | 11 ++++++----- lib/spack/spack/spec.py | 2 +- lib/spack/spack/test/architecture.py | 7 +++---- 6 files changed, 16 insertions(+), 20 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index d7f908cfb1..41778795c3 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -323,8 +323,8 @@ def to_dict(self): target = self.target d['platform'] = self.platform.name - d['platform_os'] = self.platform_os.to_dict() - d['target'] = self.target.to_dict() + d['platform_os'] = self.platform_os.to_dict() if self.platform_os else None + d['target'] = self.target.to_dict() if self.target else None return d @@ -362,8 +362,8 @@ def arch_from_dict(d): os_dict = d['platform_os'] target_dict = d['target'] - target = _target_from_dict(target_dict) - platform_os = _operating_system_from_dict(os_dict) + target = _target_from_dict(target_dict) if os_dict else None + platform_os = _operating_system_from_dict(os_dict) if os_dict else None arch.target = target arch.platform_os = platform_os diff --git a/lib/spack/spack/cmd/find.py b/lib/spack/spack/cmd/find.py index 8b91f45cfd..c2bba13dc8 100644 --- a/lib/spack/spack/cmd/find.py +++ b/lib/spack/spack/cmd/find.py @@ -239,5 +239,5 @@ def find(parser, args): mode=args.mode, long=args.long, very_long=args.very_long, - show_flags=args.show_flags) + show_flags=args.show_flags, namespace=args.namespace) diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 3432c142bf..8fe7a17116 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -261,11 +261,7 @@ def get_compilers(cspec): compilers.append(cls(cspec, operating_system, compiler_paths, mods, alias, **flags)) -##ifdef NEW -# return cls(cspec, *compiler_paths, **flags) -##else /* not NEW */ -# return compilers -##endif /* not NEW */ + return compilers matches = set(find(compiler_spec, scope)) compilers = [] diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 72b184dd8f..f38afd38dc 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -282,11 +282,8 @@ def _proper_compiler_style(cspec, architecture): return False #Find the another spec that has a compiler, or the root if none do -#ifdef NEW other_spec = spec if spec.compiler else find_spec(spec, lambda(x) : x.compiler) -#else /* not NEW */ - other_spec = find_spec(spec, lambda(x) : x.compiler) -#endif /* not NEW */ + if not other_spec: other_spec = spec.root other_compiler = other_spec.compiler @@ -321,6 +318,10 @@ def concretize_compiler_flags(self, spec): compiler is used, defaulting to no compiler flags in the spec. Default specs set at the compiler level will still be added later. """ + if not spec.architecture.platform_os: + #Although this usually means changed, this means awaiting other changes + return True + ret = False for flag in spack.spec.FlagMap.valid_compiler_flags(): try: @@ -352,7 +353,7 @@ def concretize_compiler_flags(self, spec): # Include the compiler flag defaults from the config files # This ensures that spack will detect conflicts that stem from a change # in default compiler flags. - compiler = spack.compilers.compiler_for_spec(spec.compiler) + compiler = spack.compilers.compiler_for_spec(spec.compiler, spec.architecture.platform_os) for flag in compiler.flags: if flag not in spec.compiler_flags: spec.compiler_flags[flag] = compiler.flags[flag] diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 9d217d17ca..7c04baaa91 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1458,7 +1458,7 @@ def constrain(self, other, deps=True): self.architecture = self.architecture or other.architecture elif self.architecture.platform is None or other.architecture.platform is None: self.architecture.platform = self.architecture.platform or other.architecture.platform - elif self.architecture.platform_os is None of other.architecture.platform_os is None: + elif self.architecture.platform_os is None or other.architecture.platform_os is None: self.architecture.platform_os = self.architecture.platform_os or other.architecture.platform_os elif self.architecture.target is None or other.architecture.target is None: self.architecture.target = self.architecture.target or other.architecture.target diff --git a/lib/spack/spack/test/architecture.py b/lib/spack/spack/test/architecture.py index 4cba2da9a5..2927e468a0 100644 --- a/lib/spack/spack/test/architecture.py +++ b/lib/spack/spack/test/architecture.py @@ -72,7 +72,7 @@ def test_user_front_end_input(self): """ frontend_os = self.platform.operating_system("frontend") frontend_target = self.platform.target("frontend") - frontend_spec = Spec("libelf=frontend") + frontend_spec = Spec("libelf os=frontend target=frontend") frontend_spec.concretize() self.assertEqual(frontend_os, frontend_spec.architecture.platform_os) self.assertEqual(frontend_target, frontend_spec.architecture.target) @@ -83,7 +83,7 @@ def test_user_back_end_input(self): """ backend_os = self.platform.operating_system("backend") backend_target = self.platform.target("backend") - backend_spec = Spec("libelf=backend") + backend_spec = Spec("libelf os=backend target=backend") backend_spec.concretize() self.assertEqual(backend_os, backend_spec.architecture.platform_os) self.assertEqual(backend_target, backend_spec.architecture.target) @@ -109,8 +109,7 @@ def test_user_input_combination(self): results = [] for arch in combinations: o,t = arch - arch_spec = "-".join(arch) - spec = Spec("libelf=%s" % arch_spec) + spec = Spec("libelf os=%s target=%s" % (o, t)) spec.concretize() results.append(spec.architecture.platform_os == self.platform.operating_system(o)) results.append(spec.architecture.target == self.platform.target(t)) From 63459ab0c78512906c35c5d5f9fbd5a072a3203a Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 18 May 2016 18:21:52 -0700 Subject: [PATCH 249/481] Fixed some of the bugs --- lib/spack/spack/spec.py | 24 +++++++++-------- lib/spack/spack/test/spec_semantics.py | 37 +++++++++++--------------- 2 files changed, 29 insertions(+), 32 deletions(-) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 7c04baaa91..f88475d1c8 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1088,12 +1088,12 @@ def concretize(self): for s in self.traverse(root=False): - if spec.external_module: - compiler = spack.compilers.compiler_for_spec(spec.compiler, spec.architecture.platform_os) + if s.external_module: + compiler = spack.compilers.compiler_for_spec(s.compiler, s.architecture.platform_os) for mod in compiler.modules: load_module(mod) - spec.external = get_path_from_module(spec.external_module) + s.external = get_path_from_module(s.external_module) # Mark everything in the spec as concrete, as well. self._mark_concrete() @@ -1426,6 +1426,7 @@ def constrain(self, other, deps=True): other.variants[v]) # TODO: Check out the logic here + print self.architecture, other.architecture, "^^^^^^^^^^^^^^^^^^^^^^^" if self.architecture is not None and other.architecture is not None: if self.architecture.platform is not None and other.architecture.platform is not None: if self.architecture.platform != other.architecture.platform: @@ -1453,16 +1454,17 @@ def constrain(self, other, deps=True): changed |= self.compiler_flags.constrain(other.compiler_flags) - old = self.architecture + old = str(self.architecture) if self.architecture is None or other.architecture is None: self.architecture = self.architecture or other.architecture - elif self.architecture.platform is None or other.architecture.platform is None: - self.architecture.platform = self.architecture.platform or other.architecture.platform - elif self.architecture.platform_os is None or other.architecture.platform_os is None: - self.architecture.platform_os = self.architecture.platform_os or other.architecture.platform_os - elif self.architecture.target is None or other.architecture.target is None: - self.architecture.target = self.architecture.target or other.architecture.target - changed |= (self.architecture != old) + else: + if self.architecture.platform is None or other.architecture.platform is None: + self.architecture.platform = self.architecture.platform or other.architecture.platform + if self.architecture.platform_os is None or other.architecture.platform_os is None: + self.architecture.platform_os = self.architecture.platform_os or other.architecture.platform_os + if self.architecture.target is None or other.architecture.target is None: + self.architecture.target = self.architecture.target or other.architecture.target + changed |= (str(self.architecture) != old) if deps: changed |= self._constrain_dependencies(other) diff --git a/lib/spack/spack/test/spec_semantics.py b/lib/spack/spack/test/spec_semantics.py index 8cdb91e206..9bd32a3d10 100644 --- a/lib/spack/spack/test/spec_semantics.py +++ b/lib/spack/spack/test/spec_semantics.py @@ -140,7 +140,7 @@ def test_satisfies_compiler_version(self): def test_satisfies_architecture(self): - platform = self.architecture.sys_type() + platform = spack.architecture.sys_type() if platform.name == 'crayxc': self.check_satisfies('foo target=frontend os=frontend', 'target=frontend os=frontend') self.check_satisfies('foo target=backend os=backend', 'target=backend', 'os=backend') @@ -377,34 +377,28 @@ def test_constrain_architecture(self): 'libelf', 'libelf target=default_target os=default_os') - #def test_constrain_arch(self): - # self.check_constrain('libelf arch=bgqos_0', 'libelf arch=bgqos_0', 'libelf arch=bgqos_0') - # self.check_constrain('libelf arch=bgqos_0', 'libelf', 'libelf arch=bgqos_0') -#els#e /* not NEW */ - #def test_constrain_target(self): - # platform = spack.architecture.sys_type() - # target = platform.target('default_target').name - # self.check_constrain('libelf='+target, 'libelf='+target, 'libelf='+target) - # self.check_constrain('libelf='+target, 'libelf', 'libelf='+target) -#end#if /* not NEW */ - - def test_constrain_compiler(self): self.check_constrain('libelf %gcc@4.4.7', 'libelf %gcc@4.4.7', 'libelf %gcc@4.4.7') self.check_constrain('libelf %gcc@4.4.7', 'libelf', 'libelf %gcc@4.4.7') def test_invalid_constraint(self): - self.check_invalid_constraint('libelf@0:2.0', 'libelf@2.1:3') - self.check_invalid_constraint('libelf@0:2.5%gcc@4.8:4.9', 'libelf@2.1:3%gcc@4.5:4.7') +# self.check_invalid_constraint('libelf@0:2.0', 'libelf@2.1:3') +# self.check_invalid_constraint('libelf@0:2.5%gcc@4.8:4.9', 'libelf@2.1:3%gcc@4.5:4.7') - self.check_invalid_constraint('libelf+debug', 'libelf~debug') - self.check_invalid_constraint('libelf+debug~foo', 'libelf+debug+foo') - self.check_invalid_constraint('libelf debug=2', 'libelf debug=1') +# self.check_invalid_constraint('libelf+debug', 'libelf~debug') +# self.check_invalid_constraint('libelf+debug~foo', 'libelf+debug+foo') +# self.check_invalid_constraint('libelf debug=2', 'libelf debug=1') - self.check_invalid_constraint('libelf cppflags="-O3"', 'libelf cppflags="-O2"') - self.check_invalid_constraint('libelf target=default_target os=default_os', - 'libelf target=x86_64 os=ubuntu') +# self.check_invalid_constraint('libelf cppflags="-O3"', 'libelf cppflags="-O2"') + platform = spack.architecture.sys_type() + if len(platform.operating_sys.keys()) > 1 or len(platform.targets.keys()) > 1: + os1 = platform.operating_sys.keys()[0] + os2 = platform.operating_sys.keys()[-1] + target1 = platform.targets.keys()[0] + target2 = platform.targets.keys()[-1] + self.check_invalid_constraint('libelf target=%s os=%s' % (target1, os1), + 'libelf target=%s os=%s' % (target2, os2)) def test_constrain_changed(self): self.check_constrain_changed('libelf', '@1.0') @@ -447,6 +441,7 @@ def test_constrain_dependency_changed(self): self.check_constrain_changed('libelf^foo', 'libelf^foo~debug') platform = spack.architecture.sys_type() default_target = platform.target('default_target').name + print default_target self.check_constrain_changed('libelf^foo', 'libelf^foo target='+default_target) From 126fc10b9dc56a51eb75bb7cfb88619b7029097c Mon Sep 17 00:00:00 2001 From: Brett Viren Date: Thu, 19 May 2016 08:35:54 -0600 Subject: [PATCH 250/481] Remove `view print` for now. --- lib/spack/docs/basic_usage.rst | 118 --------------------------------- lib/spack/spack/cmd/view.py | 72 +------------------- 2 files changed, 3 insertions(+), 187 deletions(-) diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst index 4af4387846..e108e393d7 100644 --- a/lib/spack/docs/basic_usage.rst +++ b/lib/spack/docs/basic_usage.rst @@ -330,124 +330,6 @@ of libelf would look like this: The full spec syntax is discussed in detail in :ref:`sec-specs`. -``spack view print`` -~~~~~~~~~~~~~~~~~~~~~~ - -``spack view print`` is a subcommand of ``spack view`` which displays information about one or more installed packages and their dependencies using a user-provided format string. The string can reverence variables in a shell-like manner for example ``$variable`` or ``${variable}text``. It can also include ``\t`` for tabs and ``\n`` for new lines. - -Some of the supported variables are: - -``name`` - The package name. - -``version`` - The package version. - -``spec`` - The package specification. - -``root`` - The root specification. - -``prefix`` - The installation directory. - -``variants`` - The collection of variants, if any. - -``namespace`` - The package repository name space. - -``compiler`` - The compiler \`name@versoin\` used to build the package. - -``architecture`` - The architecture targeted by the compiler. - -``dependencies`` - A comma-separated list of names of packages on which the package depends. - -``dependents`` - A comma-separated list of names of packages which depend on the package. - -``hash`` - The Spack hash for the package. - -``url`` - The source URL for the package. - -``stage`` - The directory for staging the build. - -``build_log`` - The path to the build log file. - -``rpath`` - The colon-separated library \`RPATH\` used in building the package. - -Here are some example uses of `spack view print`. A simple line-oriented report of information can be produced: - -.. code-block:: sh - - $ spack view print '$name\t$version\t$hash\n' cmake@3.5.2 - ncurses 6.0 bvbu4ixbnvtodpik4qzljlx3ukpyfrcz - zlib 1.2.8 ckki7zlryxrsetfqkgoxxahlhqqjni7n - openssl 1.0.2g 6zbar63sciso253nptxyrnhupymo7oyi - cmake 3.5.2 wprvmoczkpw4tiy5ybuk5zr7saus2d7g - -There are better ways to do this but a slow-and-dirty shell init procedure can be formed: - -.. code-block:: sh - - $ spack view print 'export PATH="${prefix}/bin:$$PATH"\n' m4 - export PATH="/spack/opt/spack/linux-x86_64/gcc-5.2.1/libsigsegv-2.10-h6hsv76hffcjoe3nsaihzxemniwiedu2/bin:$PATH" - export PATH="/spack/opt/spack/linux-x86_64/gcc-5.2.1/m4-1.4.17-a4ikhddnk2zonr66mbwaqb226uhigcrp/bin:$PATH" - $ eval $(spack view print 'export PATH="${prefix}/bin:$$PATH"\n' m4) - -Or, maybe you have some external application that can chew on Spack data in which case you can dump it to some easily parsed markup syntax such as YAML: - -.. code-block:: sh - - - $ spack view print '${name}:\n - version: ${version}\n - url : ${url}\n - spec: ${spec}\n - prefix: ${prefix}\n - root : ${root}\n - stage : ${stage}\n - log: ${build_log}\n' m4 - -Which might produce something like: - -.. code-block:: yaml - - libsigsegv: - - version: 2.10 - - url : ftp://ftp.gnu.org/gnu/libsigsegv/libsigsegv-2.10.tar.gz - - spec: libsigsegv@2.10%gcc@5.2.1=linux-x86_64-h6hsv76 - - prefix: /spack/opt/spack/linux-x86_64/gcc-5.2.1/libsigsegv-2.10-h6hsv76hffcjoe3nsaihzxemniwiedu2 - - root : m4@1.4.17%gcc@5.2.1+sigsegv=linux-x86_64^libsigsegv@2.10%gcc@5.2.1=linux-x86_64 - - stage : /spack/var/spack/stage/libsigsegv-2.10-h6hsv76hffcjoe3nsaihzxemniwiedu2 - - log: /spack/opt/spack/linux-x86_64/gcc-5.2.1/libsigsegv-2.10-h6hsv76hffcjoe3nsaihzxemniwiedu2/.spack/build.out - m4: - - version: 1.4.17 - - url : ftp://ftp.gnu.org/gnu/m4/m4-1.4.17.tar.gz - - spec: m4@1.4.17%gcc@5.2.1+sigsegv=linux-x86_64-a4ikhdd - - prefix: /spack/opt/spack/linux-x86_64/gcc-5.2.1/m4-1.4.17-a4ikhddnk2zonr66mbwaqb226uhigcrp - - root : m4@1.4.17%gcc@5.2.1+sigsegv=linux-x86_64^libsigsegv@2.10%gcc@5.2.1=linux-x86_64 - - stage : /spack/var/spack/stage/m4-1.4.17-a4ikhddnk2zonr66mbwaqb226uhigcrp - - log: /spack/opt/spack/linux-x86_64/gcc-5.2.1/m4-1.4.17-a4ikhddnk2zonr66mbwaqb226uhigcrp/.spack/build.out - -Or, maybe you want to do something with information about package dependencies by generating some ready-to-import Python code holding the tree: - -.. code-block:: sh - - $ spack view print '$name = dict(name = "$name", parents = [$dependencies], children = [$dependents])\n' cmake@3.5.2 - -Producing this Python code - -.. code-block:: python - - ncurses = dict(name = "ncurses", parents = []]) - zlib = dict(name = "zlib", parents = []]) - openssl = dict(name = "openssl", parents = [zlib]]) - cmake = dict(name = "cmake", parents = [ncurses,openssl]]) - - Compiler configuration diff --git a/lib/spack/spack/cmd/view.py b/lib/spack/spack/cmd/view.py index 4861383eaa..8f1fc9be74 100644 --- a/lib/spack/spack/cmd/view.py +++ b/lib/spack/spack/cmd/view.py @@ -24,9 +24,8 @@ ############################################################################## '''Produce a "view" of a Spack DAG. -A "view" is the product of applying a function on a set of package specs. - -This set consists of: +A "view" is file hierarchy representing the union of a number of +Spack-installed package file hierarchies. The union is formed from: - specs resolved from the package names given by the user (the seeds) @@ -35,7 +34,7 @@ - less any specs with names matching the regular expressions given by `--exclude` -The `view` command provides a number of functions (the "actions"): +The `view` can be built and tore down via a number of methods (the "actions"): - symlink :: a file system view which is a directory hierarchy that is the union of the hierarchies of the installed packages in the DAG @@ -46,8 +45,6 @@ - statlink :: a view producing a status report of a symlink or hardlink view. -- format :: a view printing one string per spec following a given format. - The file system view concept is imspired by Nix, implemented by brett.viren@gmail.com ca 2016. @@ -68,7 +65,6 @@ import os import re -import sys import spack import spack.cmd import llnl.util.tty as tty @@ -117,13 +113,6 @@ def setup_parser(sp): help="Path to file system view directory.") act.add_argument('specs', **specs_opts) - # The formatted print action. - act = ssp.add_parser('print', - help="Print a string to stdout based on given format") - act.add_argument('format', nargs=1, - help="Format describing per-package printout.") - act.add_argument('specs', **specs_opts) - return @@ -190,42 +179,6 @@ def flatten(seeds, descend=True): return flat -def spec2dict(spec): - 'Convert info in a spec into a simple dictionary.' - - # Expclitly convert instead of just returning spec.__dict__ as - # some things need processing or are properties. - # - pkg = spec.package - inst_deps = ','.join([s.name for s in pkg.installed_dependents]), - ret = dict(name=spec.name, - spec=spec.short_spec, - colorspec=spec.cshort_spec, # color - root=spec.root, - prefix=spec.prefix, - version=spec.version, - variants=spec.variants, - namespace=spec.namespace, - compiler=spec.compiler, - architecture=spec.architecture, - dependencies=','.join(spec.dependencies.keys()), - dependents=','.join(spec.dependents.keys()), - external=spec.external or "False", - hash=spec.dag_hash(), - - # package related: - url=pkg.url, - stage=pkg.stage.path, - installed=pkg.installed, - installed_dependents=inst_deps, - build_log=pkg.build_log_path, - rpath=':'.join(pkg.rpath), - - # ... - ) - return ret - - def check_one(spec, path, verbose=False): 'Check status of view in path against spec' dotspack = os.path.join(path, '.spack', spec.name) @@ -326,25 +279,6 @@ def visitor_statlink(specs, args): visitor_check = visitor_statlink -def visitor_print(specs, args): - 'Print a string for each spec using args.format.' - fmt = args.format[0] - from string import Template - t = Template(fmt) - - for spec in specs: - kwds = spec2dict(spec) - try: - text = t.substitute(kwds) - except KeyError: - tty.error("Format error, use keywords: %s" % - (', '.join(kwds.keys()), )) - raise - # argparser escapes these - text = text.replace(r'\n', '\n').replace(r'\t', '\t') - sys.stdout.write(text) - - def view(parser, args): 'Produce a view of a set of packages.' From 19c8a52fe1c25639c47620bc07f0824c59c921f7 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Thu, 19 May 2016 09:53:49 -0700 Subject: [PATCH 251/481] fixed spec syntax test --- lib/spack/spack/test/spec_syntax.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/test/spec_syntax.py b/lib/spack/spack/test/spec_syntax.py index c4e4c9cdfe..ae19de177b 100644 --- a/lib/spack/spack/test/spec_syntax.py +++ b/lib/spack/spack/test/spec_syntax.py @@ -58,7 +58,7 @@ class SpecSyntaxTest(unittest.TestCase): # ================================================================================ # Parse checks # ================================================================================ - def check_parse(self, expected, spec=None): + def check_parse(self, expected, spec=None, remove_arch=True): """Assert that the provided spec is able to be parsed. If this is called with one argument, it assumes that the string is canonical (i.e., no spaces and ~ instead of - for variants) and that it @@ -70,6 +70,13 @@ def check_parse(self, expected, spec=None): if spec is None: spec = expected output = spack.spec.parse(spec) + + # Remove architectures that get added by parser. + if remove_arch: + for spec in output: + for s in spec.traverse(): + s.architecture = None + parsed = (" ".join(str(spec) for spec in output)) self.assertEqual(expected, parsed) From 36450b9bc9f41ac0f70696352327d8f07d4ee3e3 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 19 May 2016 14:25:50 -0700 Subject: [PATCH 252/481] Adding link_paths to craype compilers --- lib/spack/spack/compilers/craype.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/compilers/craype.py b/lib/spack/spack/compilers/craype.py index e8ae284f5b..4ba8b110ec 100644 --- a/lib/spack/spack/compilers/craype.py +++ b/lib/spack/spack/compilers/craype.py @@ -47,10 +47,11 @@ class Craype(Compiler): PrgEnv = 'PrgEnv-cray' PrgEnv_compiler = 'craype' -# @property -# def cxx11_flag(self): -# return "-hstd=c++11" - + link_paths = { 'cc' : 'cc', + 'cxx' : 'c++', + 'f77' : 'f77', + 'fc' : 'fc'} + @classmethod def default_version(cls, comp): return get_compiler_version(comp, r'([Vv]ersion).*(\d+(\.\d+)+)') From 175a042fd332c13200fb30f7f834541c3bbaa6c4 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 19 May 2016 16:42:13 -0700 Subject: [PATCH 253/481] Changed rpaths method so that it only grabs rpaths using the compiler module name instead of PrgEnv-x module name --- lib/spack/spack/build_environment.py | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 7aeea5c672..4f4ca26cbc 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -113,7 +113,6 @@ def __call__(self, *args, **kwargs): return super(MakeExecutable, self).__call__(*args, **kwargs) - def load_module(mod): """Takes a module name and removes modules until it is possible to load that module. It then loads the provided module. Depends on the @@ -145,7 +144,6 @@ def get_path_from_module(mod): # Read the module text = modulecmd('show', mod, output=str, error=str).split('\n') - # If it lists its package directory, return that for line in text: if line.find(mod.upper()+'_DIR') >= 0: @@ -166,15 +164,14 @@ def get_path_from_module(mod): # If it sets the LD_LIBRARY_PATH or CRAY_LD_LIBRARY_PATH, use that for line in text: - if line.find('LD_LIBRARY_PATH') >= 0: + if line.find('LD_LIBRARY_PATH') >= 0: words = line.split() path = words[2] return path[:path.find('/lib')] - # Unable to find module path return None -def set_compiler_environment_variables(pkg): +def set_compiler_environment_variables(pkg, env): assert(pkg.spec.concrete) compiler = pkg.compiler flags = pkg.spec.compiler_flags @@ -276,14 +273,12 @@ def set_build_environment_variables(pkg, env): env.set(SPACK_DEBUG_LOG_DIR, spack.spack_working_dir) # Add any pkgconfig directories to PKG_CONFIG_PATH - pkg_config_dirs = [] - for p in dep_prefixes: - for maybe in ('lib', 'lib64', 'share'): - pcdir = join_path(p, maybe, 'pkgconfig') + for pre in dep_prefixes: + for directory in ('lib', 'lib64', 'share'): + pcdir = join_path(pre, directory, 'pkgconfig') if os.path.isdir(pcdir): - pkg_config_dirs.append(pcdir) - - env.prepend_path('PKG_CONFIG_PATH', pkg_config_dirs) + #pkg_config_dirs.append(pcdir) + env.prepend_path('PKG_CONFIG_PATH',pcdir) if pkg.spec.architecture.target.module_name: load_module(pkg.spec.architecture.target.module_name) @@ -369,8 +364,9 @@ def get_rpaths(pkg): if os.path.isdir(d.prefix.lib)) rpaths.extend(d.prefix.lib64 for d in pkg.spec.dependencies.values() if os.path.isdir(d.prefix.lib64)) - for mod in pkg.spec.compiler.modules: - rpaths.append(get_path_for_module(mod)) + # Second module is our compiler mod name. We use that to get rpaths from + # module show output. + rpaths.append(get_path_from_module(pkg.compiler.modules[1])) return rpaths From c3767d593d04f990b4344540c95e07f172d54c42 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 20 May 2016 15:48:13 -0700 Subject: [PATCH 254/481] compiler list is sorted in descending order, in order to get most up to date version we check from the beginning of the list --- lib/spack/spack/concretize.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index f38afd38dc..1c451296c6 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -302,10 +302,10 @@ def _proper_compiler_style(cspec, architecture): raise UnavailableCompilerVersionError(other_compiler) # copy concrete version into other_compiler - index = len(matches)-1 + index = 0 while not _proper_compiler_style(matches[index], spec.architecture): - index -= 1 - if index == 0: + index += 1 + if index == len(matches) - 1: raise NoValidVersionError(spec) spec.compiler = matches[index].copy() assert(spec.compiler.concrete) From 6c352132d0dd2c0f05270fa7888d47ca37547674 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 23 May 2016 14:28:00 -0700 Subject: [PATCH 255/481] Started writing documentation for architecture. Might need a couple more drafts --- lib/spack/spack/architecture.py | 52 +++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 41778795c3..45b4cc228d 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -22,6 +22,58 @@ # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +""" +This module contains all the elements that are required to create an +architecture object. These include, the target processor, the operating system, +and the architecture platform (i.e. cray, darwin, linux, bgq, etc) classes. + +On a multiple architecture machine, the architecture spec field can be set to +build a package against any target and operating system that is present on the +platform. On Cray platforms or any other architecture that has different front and +back end environments, the operating system will determine the method of compiler +detection. + +There are two different types of compiler detection: + 1. Through the $PATH env variable (front-end detection) + 2. Through the tcl module system. (back-end detection) + +Depending on which operating system is specified, the compiler will be detected +using one of those methods. + +For platforms such as linux and darwin, the operating system is autodetected and +the target is set to be x86_64. + +The command line syntax for specifying an architecture is as follows: + + target= os= + +If the user wishes to use the defaults, either target or os can be left out of +the command line and Spack will concretize using the default. These defaults are +set in the 'platforms/' directory which contains the different subclasses for +platforms. If the machine has multiple architectures, the user can +also enter front-end, or fe or back-end or be. These settings will concretize +to their respective front-end and back-end targets and operating systems. +Additional platforms can be added by creating a subclass of Platform +and adding it inside the platform directory. + +Platforms are an abstract class that are extended by subclasses. If the user +wants to add a new type of platform (such as cray_xe), they can create a subclass +and set all the class attributes such as priority, front_target ,back_target, +front_os, back_os. Platforms also contain a priority class attribute. A lower +number signifies higher priority. These numbers are arbitrarily set and can be +changed though often there isn't much need unless a new platform is added and +the user wants that to be detected first. + +Targets are created inside the platform subclasses. Most architecture (like linux, +and darwin) will have only one target (x86_64) but in the case of Cray machines, +there is both a frontend and backend processor. The user can specify which targets +are present on front-end and back-end architecture + +Depending on the platform, operating systems are either auto-detected or are +set. The user can set the front-end and back-end operating setting by the class +attributes front_os and back_os. The operating system as described earlier, will +be responsible for compiler detection. +""" import os from collections import namedtuple import imp From d7612e7aaace3b0fa483f17840e20cde5b28f430 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Mon, 23 May 2016 16:54:41 -0700 Subject: [PATCH 256/481] Fixed errors caught by spec and concretize tests --- lib/spack/spack/architecture.py | 78 ++++++++++++++++++++----- lib/spack/spack/concretize.py | 23 ++++++-- lib/spack/spack/spec.py | 81 +++++++++++++++++++++----- lib/spack/spack/test/architecture.py | 2 + lib/spack/spack/test/concretize.py | 4 +- lib/spack/spack/test/spec_semantics.py | 14 ++--- lib/spack/spack/test/spec_syntax.py | 6 -- 7 files changed, 160 insertions(+), 48 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 41778795c3..e4b3dbf9c7 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -105,12 +105,25 @@ def __init__(self, name): self.operating_sys = {} self.name = name + def to_dict(self): + n = {} + n['targets'] = dict((name, target.to_dict()) for (name, target) in self.targets.items()) + n['operating_systems'] = dict((name, os.to_dict()) for (name, os) in self.operating_sys.items()) + n['priority'] = self.priority + n['default_front_end_target'] = self.front_end + n['default_back_end_target'] = self.back_end + n['default_target'] = self.default + n['default_front_end_os'] = self.front_os + n['default_back_end_os'] = self.back_os + n['default_os'] = self.default_os + return {self.name: n} + def add_target(self, name, target): """Used by the platform specific subclass to list available targets. Raises an error if the platform specifies a name that is reserved by spack as an alias. """ - if name in ['front_end', 'fe', 'back_end', 'be', 'default']: + if name in ['frontend', 'fe', 'backend', 'be', 'default_target']: raise ValueError( "%s is a spack reserved alias " "and cannot be the name of a target" % name) @@ -135,6 +148,10 @@ def add_operating_system(self, name, os_class): """ Add the operating_system class object into the platform.operating_sys dictionary """ + if name in ['frontend', 'fe', 'backend', 'be', 'default_os']: + raise ValueError( + "%s is a spack reserved alias " + "and cannot be the name of an OS" % name) self.operating_sys[name] = os_class def operating_system(self, name): @@ -284,15 +301,18 @@ def to_dict(self): class Arch(object): "Architecture is now a class to help with setting attributes" - def __init__(self, platform_os=None, target=None): - self.platform = sys_type() - if platform_os: - platform_os = self.platform.operating_system(platform_os) + def __init__(self, platform=None, platform_os=None, target=None): + self.platform = platform + if platform and platform_os: + platform_os = self.platform.operating_system(platform_os) self.platform_os = platform_os - if target: + if platform and target: target = self.platform.target(target) self.target = target + # Hooks for parser to use when platform is set after target or os + self.target_string = None + self.os_string = None @property def concrete(self): @@ -302,16 +322,19 @@ def concrete(self): def __str__(self): - if self.platform.name == 'darwin': - os_name = self.platform_os.name - else: - os_name = str(self.platform_os) + if self.platform or self.platform_os or self.target: + if self.platform.name == 'darwin': + os_name = self.platform_os.name + else: + os_name = str(self.platform_os) - return (str(self.platform) +"-"+ - os_name + "-" + str(self.target)) + return (str(self.platform) +"-"+ + os_name + "-" + str(self.target)) + else: + return '' def _cmp_key(self): - platform = self.platform.name + platform = self.platform.name if isinstance(self.platform, Platform) else self.platform os = self.platform_os.name if isinstance(self.platform_os, OperatingSystem) else self.platform_os target = self.target.name if isinstance(self.target, Target) else self.target return (platform, os, target) @@ -322,7 +345,7 @@ def to_dict(self): platform_os = self.platform_os target = self.target - d['platform'] = self.platform.name + d['platform'] = self.platform.to_dict() if self.platform else None d['platform_os'] = self.platform_os.to_dict() if self.platform_os else None d['target'] = self.target.to_dict() if self.target else None @@ -350,6 +373,27 @@ def _operating_system_from_dict(os_dict): operating_system.version = os_dict['version'] return operating_system +def _platform_from_dict(platform_dict): + """ Constructs a platform from a dictionary. """ + platform = Platform.__new__(Platform) + name, p_dict = platform_dict.items()[0] + platform.name = name + platform.targets = {} + for name, t_dict in p_dict['targets'].items(): + platform.add_target(name, _target_from_dict(t_dict)) + platform.operating_sys = {} + for name, o_dict in p_dict['operating_systems'].items(): + platform.add_operating_system(name, _operating_system_from_dict(o_dict)) + platform.priority = p_dict['priority'] + platform.front_end = p_dict['default_front_end_target'] + platform.back_end = p_dict['default_back_end_target'] + platform.default = p_dict['default_target'] + platform.front_os = p_dict['default_front_end_os'] + platform.back_os = p_dict['default_back_end_os'] + platform.default_os = p_dict['default_os'] + + return platform + def arch_from_dict(d): """ Uses _platform_from_dict, _operating_system_from_dict, _target_from_dict helper methods to recreate the arch tuple from the dictionary read from @@ -359,14 +403,20 @@ def arch_from_dict(d): if d is None: return None + platform_dict = d['platform'] os_dict = d['platform_os'] target_dict = d['target'] + platform = _platform_from_dict(platform_dict) if platform_dict else None target = _target_from_dict(target_dict) if os_dict else None platform_os = _operating_system_from_dict(os_dict) if os_dict else None + arch.platform = platform arch.target = target arch.platform_os = platform_os + arch.os_string = None + arch.target_string = None + return arch @memoized diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index f38afd38dc..1f5c07779e 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -198,7 +198,7 @@ def _concretize_operating_system(self, spec): spec.architecture.platform_os = spec.root.architecture.platform_os else: spec.architecture.platform_os = spec.architecture.platform.operating_system('default_os') - return True #changed + return True #changed def _concretize_target(self, spec): platform = spec.architecture.platform @@ -210,7 +210,19 @@ def _concretize_target(self, spec): spec.architecture.target = spec.root.architecture.target else: spec.architecture.target = spec.architecture.platform.target('default_target') - return True #changed + print spec.architecture, spec.architecture.platform, spec.architecture.platform_os, spec.architecture.target + return True #changed + + def _concretize_platform(self, spec): + if spec.architecture.platform is not None and isinstance( + spec.architecture.platform, spack.architecture.Platform): + return False + if spec.root.architecture and spec.root.architecture.platform: + if isinstance(spec.root.architecture.platform,spack.architecture.Platform): + spec.architecture.platform = spec.root.architecture.platform + else: + spec.architecture.platform = spack.architecture.sys_type() + return True #changed? def concretize_architecture(self, spec): """If the spec is empty provide the defaults of the platform. If the @@ -227,10 +239,11 @@ def concretize_architecture(self, spec): # Set the architecture to all defaults spec.architecture = spack.architecture.Arch() return True - + # Concretize the operating_system and target based of the spec - ret = any((self._concretize_operating_system(spec), - self._concretize_target(spec))) + ret = any((self._concretize_platform(spec), + self._concretize_operating_system(spec), + self._concretize_target(spec))) return ret diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index f88475d1c8..d15598405f 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -99,6 +99,7 @@ import itertools import hashlib import base64 +import imp from StringIO import StringIO from operator import attrgetter import yaml @@ -107,6 +108,7 @@ import llnl.util.tty as tty from llnl.util.lang import * from llnl.util.tty.color import * +from llnl.util.filesystem import join_path import spack import spack.architecture @@ -119,6 +121,7 @@ from spack.version import * from spack.util.string import * from spack.util.prefix import Prefix +from spack.util.naming import mod_to_class from spack.virtual import ProviderIndex from spack.build_environment import get_path_from_module, load_module @@ -535,10 +538,24 @@ def _add_flag(self, name, value): Known flags currently include "arch" """ valid_flags = FlagMap.valid_compiler_flags() - if name == 'os' or name == 'operating_system': - self._set_os(value) +# if name == 'arch' or name == 'architecture': +# platform, op_sys, target = value.split('-') +# print platform, op_sys, target, '+++++++' +# self._set_platform(platform) +# self._set_os(op_sys) +# self._set_target(target) + if name == 'platform': + self._set_platform(value) + elif name == 'os' or name == 'operating_system': + if self.architecture.platform: + self._set_os(value) + else: + self.architecture.os_string = value elif name == 'target': - self._set_target(value) + if self.architecture.platform: + self._set_target(value) + else: + self.architecture.target_string = value elif name in valid_flags: assert(self.compiler_flags is not None) self.compiler_flags[name] = value.split() @@ -551,6 +568,39 @@ def _set_compiler(self, compiler): "Spec for '%s' cannot have two compilers." % self.name) self.compiler = compiler + def _set_platform(self, value): + """Called by the parser to set the architecture platform""" + if isinstance(value, basestring): + mod_path = spack.platform_path + mod_string = 'spack.platformss' + names = list_modules(mod_path) + if value in names: + # Create a platform object from the name + mod_name = mod_string + value + path = join_path(mod_path, value) + '.py' + mod = imp.load_source(mod_name, path) + class_name = mod_to_class(value) + if not hasattr(mod, class_name): + tty.die('No class %s defined in %s' % (class_name, mod_name)) + cls = getattr(mod, class_name) + if not inspect.isclass(cls): + tty.die('%s.%s is not a class' % (mod_name, class_name)) + platform = cls() + else: + tty.die("No platform class %s defined." % value) + else: + # The value is a platform + platform = value + + self.architecture.platform = platform + + # Set os and target if we previously got strings for them + if self.architecture.os_string: + self._set_os(self.architecture.os_string) + self.architecture.os_string = None + if self.architecture.target_string: + self._set_target(self.architecture.target_string) + self.architecture.target_string = None def _set_os(self, value): """Called by the parser to set the architecture operating system""" @@ -1016,6 +1066,7 @@ def _expand_virtual_packages(self): changed = True spec.dependencies = DependencyMap() replacement.dependencies = DependencyMap() + replacement.architecture = self.architecture # TODO: could this and the stuff in _dup be cleaned up? def feq(cfield, sfield): @@ -1426,7 +1477,6 @@ def constrain(self, other, deps=True): other.variants[v]) # TODO: Check out the logic here - print self.architecture, other.architecture, "^^^^^^^^^^^^^^^^^^^^^^^" if self.architecture is not None and other.architecture is not None: if self.architecture.platform is not None and other.architecture.platform is not None: if self.architecture.platform != other.architecture.platform: @@ -1831,8 +1881,7 @@ def _cmp_node(self): self.variants, self.architecture, self.compiler, - self.compiler_flags, - self.dag_hash()) + self.compiler_flags) def eq_node(self, other): @@ -1946,7 +1995,7 @@ def write(s, c): if self.variants: write(fmt % str(self.variants), c) elif c == '=': - if self.architecture: + if self.architecture and str(self.architecture): write(fmt % (' arch' + c + str(self.architecture)), c) elif c == '#': out.write('-' + fmt % (self.dag_hash(7))) @@ -2004,7 +2053,7 @@ def write(s, c): if self.variants: write(fmt % str(self.variants), '+') elif named_str == 'ARCHITECTURE': - if self.architecture: + if self.architecture and str(self.architecture): write(fmt % str(self.architecture), ' arch=') elif named_str == 'SHA1': if self.dependencies: @@ -2054,13 +2103,13 @@ def __cmp__(self, other): self.variants, other.variants) #Target - if self.target != other.target: - return spack.pkgsort.target_compare(pkgname, - self.target, other.target) + if self.architecture != other.architecture: + return spack.pkgsort.architecture_compare(pkgname, + self.architecture, other.architecture) #Dependency is not configurable - if self.dep_hash() != other.dep_hash(): - return -1 if self.dep_hash() < other.dep_hash() else 1 + if self.dependencies != other.dependencies: + return -1 if self.dependencies < other.dependencies else 1 #Equal specs return 0 @@ -2181,6 +2230,11 @@ def do_parse(self): raise SpecParseError(e) + # If the spec has an os or a target and no platform, give it the default platform + for spec in specs: + for s in spec.traverse(): + if s.architecture.os_string or s.architecture.target_string: + s._set_platform(spack.architecture.sys_type()) return specs @@ -2401,7 +2455,6 @@ class SpecError(spack.error.SpackError): def __init__(self, message): super(SpecError, self).__init__(message) - class SpecParseError(SpecError): """Wrapper for ParseError for when we're parsing specs.""" def __init__(self, parse_error): diff --git a/lib/spack/spack/test/architecture.py b/lib/spack/spack/test/architecture.py index 2927e468a0..f5b1068435 100644 --- a/lib/spack/spack/test/architecture.py +++ b/lib/spack/spack/test/architecture.py @@ -26,12 +26,14 @@ def tearDown(self): def test_dict_functions_for_architecture(self): arch = Arch() + arch.platform = spack.architecture.sys_type() arch.platform_os = arch.platform.operating_system('default_os') arch.target = arch.platform.target('default_target') d = arch.to_dict() new_arch = spack.architecture.arch_from_dict(d) + self.assertEqual(arch, new_arch) self.assertTrue( isinstance(arch, Arch) ) diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index 49281b9a0c..963481054e 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -55,8 +55,8 @@ def check_spec(self, abstract, concrete): if abstract.compiler and abstract.compiler.concrete: self.assertEqual(abstract.compiler, concrete.compiler) - if abstract.architecture and abstract.architecture.target.concrete: - self.assertEqual(abstract.target, concrete.target) + if abstract.architecture and abstract.architecture.concrete: + self.assertEqual(abstract.architecture, concrete.architecture) def check_concretize(self, abstract_spec): diff --git a/lib/spack/spack/test/spec_semantics.py b/lib/spack/spack/test/spec_semantics.py index 9bd32a3d10..45c89100d4 100644 --- a/lib/spack/spack/test/spec_semantics.py +++ b/lib/spack/spack/test/spec_semantics.py @@ -383,14 +383,14 @@ def test_constrain_compiler(self): def test_invalid_constraint(self): -# self.check_invalid_constraint('libelf@0:2.0', 'libelf@2.1:3') -# self.check_invalid_constraint('libelf@0:2.5%gcc@4.8:4.9', 'libelf@2.1:3%gcc@4.5:4.7') + self.check_invalid_constraint('libelf@0:2.0', 'libelf@2.1:3') + self.check_invalid_constraint('libelf@0:2.5%gcc@4.8:4.9', 'libelf@2.1:3%gcc@4.5:4.7') -# self.check_invalid_constraint('libelf+debug', 'libelf~debug') -# self.check_invalid_constraint('libelf+debug~foo', 'libelf+debug+foo') -# self.check_invalid_constraint('libelf debug=2', 'libelf debug=1') + self.check_invalid_constraint('libelf+debug', 'libelf~debug') + self.check_invalid_constraint('libelf+debug~foo', 'libelf+debug+foo') + self.check_invalid_constraint('libelf debug=2', 'libelf debug=1') -# self.check_invalid_constraint('libelf cppflags="-O3"', 'libelf cppflags="-O2"') + self.check_invalid_constraint('libelf cppflags="-O3"', 'libelf cppflags="-O2"') platform = spack.architecture.sys_type() if len(platform.operating_sys.keys()) > 1 or len(platform.targets.keys()) > 1: os1 = platform.operating_sys.keys()[0] @@ -439,9 +439,9 @@ def test_constrain_dependency_changed(self): self.check_constrain_changed('libelf^foo%gcc', 'libelf^foo%gcc@4.5') self.check_constrain_changed('libelf^foo', 'libelf^foo+debug') self.check_constrain_changed('libelf^foo', 'libelf^foo~debug') + platform = spack.architecture.sys_type() default_target = platform.target('default_target').name - print default_target self.check_constrain_changed('libelf^foo', 'libelf^foo target='+default_target) diff --git a/lib/spack/spack/test/spec_syntax.py b/lib/spack/spack/test/spec_syntax.py index ae19de177b..4a534d7b5c 100644 --- a/lib/spack/spack/test/spec_syntax.py +++ b/lib/spack/spack/test/spec_syntax.py @@ -71,12 +71,6 @@ def check_parse(self, expected, spec=None, remove_arch=True): spec = expected output = spack.spec.parse(spec) - # Remove architectures that get added by parser. - if remove_arch: - for spec in output: - for s in spec.traverse(): - s.architecture = None - parsed = (" ".join(str(spec) for spec in output)) self.assertEqual(expected, parsed) From 1da6bbd14667b50f8c970022614baf39e059a779 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Mon, 23 May 2016 17:25:54 -0700 Subject: [PATCH 257/481] Fixed a couple more bugs --- lib/spack/spack/build_environment.py | 3 ++- lib/spack/spack/concretize.py | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 4f4ca26cbc..073f73a91a 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -366,7 +366,8 @@ def get_rpaths(pkg): if os.path.isdir(d.prefix.lib64)) # Second module is our compiler mod name. We use that to get rpaths from # module show output. - rpaths.append(get_path_from_module(pkg.compiler.modules[1])) + if pkg.compiler.modules and len(pkg.compiler.modules > 1): + rpaths.append(get_path_from_module(pkg.compiler.modules[1])) return rpaths diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 7ea8ea6928..7a24c1135e 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -210,7 +210,6 @@ def _concretize_target(self, spec): spec.architecture.target = spec.root.architecture.target else: spec.architecture.target = spec.architecture.platform.target('default_target') - print spec.architecture, spec.architecture.platform, spec.architecture.platform_os, spec.architecture.target return True #changed def _concretize_platform(self, spec): From 513aae5ef8d610644e8f2d2c855e92a080573f5b Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Mon, 23 May 2016 17:39:06 -0700 Subject: [PATCH 258/481] fixed bug where earlier test was blowing away environment, causing an error on which('modulecmd') --- lib/spack/spack/test/environment.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/test/environment.py b/lib/spack/spack/test/environment.py index ded1539e18..a0d959db2f 100644 --- a/lib/spack/spack/test/environment.py +++ b/lib/spack/spack/test/environment.py @@ -24,17 +24,20 @@ ############################################################################## import unittest import os +import copy from spack.environment import EnvironmentModifications class EnvironmentTest(unittest.TestCase): def setUp(self): - os.environ.clear() os.environ['UNSET_ME'] = 'foo' os.environ['EMPTY_PATH_LIST'] = '' os.environ['PATH_LIST'] = '/path/second:/path/third' os.environ['REMOVE_PATH_LIST'] = '/a/b:/duplicate:/a/c:/remove/this:/a/d:/duplicate/:/f/g' + def tearDown(self): + pass + def test_set(self): env = EnvironmentModifications() env.set('A', 'dummy value') From 3a68dd2011488acdff4e3c8cb4690718e87623e6 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 25 May 2016 12:22:16 -0700 Subject: [PATCH 259/481] (hopefully) final fixes of the merge --- lib/spack/spack/architecture.py | 13 +++++++++++-- lib/spack/spack/spec.py | 9 +++++++++ lib/spack/spack/test/modules.py | 19 +++++++++++-------- lib/spack/spack/test/spec_dag.py | 4 ++-- lib/spack/spack/test/spec_semantics.py | 17 +++++------------ 5 files changed, 38 insertions(+), 24 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 05dc969ebe..e9dd489e54 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -233,8 +233,17 @@ def __str__(self): return self.name def _cmp_key(self): - return (self.name, (_cmp_key(t) for t in self.targets.values()), - (_cmp_key(o) for o in self.operating_sys.values())) + t_keys = ''.join(str(t._cmp_key()) for t in sorted(self.targets.values())) + o_keys = ''.join(str(o._cmp_key()) for o in sorted(self.operating_sys.values())) + return (self.name, + self.default, + self.front_end, + self.back_end, + self.default_os, + self.front_os, + self.back_os, + t_keys, + o_keys) @key_ordering class OperatingSystem(object): diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index d15598405f..5d82af4af6 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1649,9 +1649,18 @@ def satisfies(self, other, deps=True, strict=False): # TODO: Need to make sure that comparisons can be made via classes if self.architecture and other.architecture: + print self.architecture, other.architecture if ((self.architecture.platform and other.architecture.platform and self.architecture.platform != other.architecture.platform) or (self.architecture.platform_os and other.architecture.platform_os and self.architecture.platform_os != other.architecture.platform_os) or (self.architecture.target and other.architecture.target and self.architecture.target != other.architecture.target)): + d1 = self.architecture.platform.to_dict() + d2 = other.architecture.platform.to_dict() + print d1 + print d2 + print d1==d2 + print self.architecture.platform == other.architecture.platform + print self.architecture.platform._cmp_key() + print other.architecture.platform._cmp_key() return False elif strict and ((other.architecture and not self.architecture) or (other.architecture.platform and not self.architecture.platform) or diff --git a/lib/spack/spack/test/modules.py b/lib/spack/spack/test/modules.py index c73badf8f2..582e067860 100644 --- a/lib/spack/spack/test/modules.py +++ b/lib/spack/spack/test/modules.py @@ -73,7 +73,7 @@ def mock_open(filename, mode): 'all': { 'filter': {'environment_blacklist': ['CMAKE_PREFIX_PATH']} }, - 'arch=x86-linux': { + 'platform=test target=x86_64': { 'environment': {'set': {'FOO': 'foo'}, 'unset': ['BAR']} } @@ -116,6 +116,7 @@ def tearDown(self): def get_modulefile_content(self, spec): spec.concretize() + print spec, '&&&&&' generator = spack.modules.TclModule(spec) generator.write() content = FILE_REGISTRY[generator.file_name].split('\n') @@ -123,27 +124,28 @@ def get_modulefile_content(self, spec): def test_simple_case(self): spack.modules.CONFIGURATION = configuration_autoload_direct - spec = spack.spec.Spec('mpich@3.0.4 arch=x86-linux') + spec = spack.spec.Spec('mpich@3.0.4') content = self.get_modulefile_content(spec) self.assertTrue('module-whatis "mpich @3.0.4"' in content) def test_autoload(self): spack.modules.CONFIGURATION = configuration_autoload_direct - spec = spack.spec.Spec('mpileaks arch=x86-linux') + spec = spack.spec.Spec('mpileaks') content = self.get_modulefile_content(spec) self.assertEqual(len([x for x in content if 'is-loaded' in x]), 2) self.assertEqual(len([x for x in content if 'module load ' in x]), 2) spack.modules.CONFIGURATION = configuration_autoload_all - spec = spack.spec.Spec('mpileaks arch=x86-linux') + spec = spack.spec.Spec('mpileaks') content = self.get_modulefile_content(spec) self.assertEqual(len([x for x in content if 'is-loaded' in x]), 5) self.assertEqual(len([x for x in content if 'module load ' in x]), 5) def test_alter_environment(self): spack.modules.CONFIGURATION = configuration_alter_environment - spec = spack.spec.Spec('mpileaks arch=x86-linux') + spec = spack.spec.Spec('mpileaks platform=test target=x86_64') content = self.get_modulefile_content(spec) + print content self.assertEqual( len([x for x in content @@ -152,8 +154,9 @@ def test_alter_environment(self): len([x for x in content if 'setenv FOO "foo"' in x]), 1) self.assertEqual(len([x for x in content if 'unsetenv BAR' in x]), 1) - spec = spack.spec.Spec('libdwarf arch=x64-linux') + spec = spack.spec.Spec('libdwarf %clang platform=test target=x86_32') content = self.get_modulefile_content(spec) + print content self.assertEqual( len([x for x in content @@ -164,14 +167,14 @@ def test_alter_environment(self): def test_blacklist(self): spack.modules.CONFIGURATION = configuration_blacklist - spec = spack.spec.Spec('mpileaks arch=x86-linux') + spec = spack.spec.Spec('mpileaks') content = self.get_modulefile_content(spec) self.assertEqual(len([x for x in content if 'is-loaded' in x]), 1) self.assertEqual(len([x for x in content if 'module load ' in x]), 1) def test_conflicts(self): spack.modules.CONFIGURATION = configuration_conflicts - spec = spack.spec.Spec('mpileaks arch=x86-linux') + spec = spack.spec.Spec('mpileaks') content = self.get_modulefile_content(spec) self.assertEqual( len([x for x in content if x.startswith('conflict')]), 2) diff --git a/lib/spack/spack/test/spec_dag.py b/lib/spack/spack/test/spec_dag.py index 99a986a528..712f07ac4d 100644 --- a/lib/spack/spack/test/spec_dag.py +++ b/lib/spack/spack/test/spec_dag.py @@ -244,8 +244,8 @@ def test_unsatisfiable_compiler_version(self): def test_unsatisfiable_architecture(self): platform = spack.architecture.sys_type() - self.set_pkg_dep('mpileaks', 'mpich target=%s' % platform.target('default_target')) - spec = Spec('mpileaks ^mpich target=sles_10_ppc64 ^callpath ^dyninst ^libelf ^libdwarf') + self.set_pkg_dep('mpileaks', 'mpich platform=test target=be') + spec = Spec('mpileaks ^mpich platform=test target=fe ^callpath ^dyninst ^libelf ^libdwarf') self.assertRaises(spack.spec.UnsatisfiableArchitectureSpecError, spec.normalize) diff --git a/lib/spack/spack/test/spec_semantics.py b/lib/spack/spack/test/spec_semantics.py index 45c89100d4..aa99ad71ed 100644 --- a/lib/spack/spack/test/spec_semantics.py +++ b/lib/spack/spack/test/spec_semantics.py @@ -141,10 +141,9 @@ def test_satisfies_compiler_version(self): def test_satisfies_architecture(self): platform = spack.architecture.sys_type() - if platform.name == 'crayxc': - self.check_satisfies('foo target=frontend os=frontend', 'target=frontend os=frontend') - self.check_satisfies('foo target=backend os=backend', 'target=backend', 'os=backend') - self.check_satisfies('foo target=default_target os=default_os','target=default_target os=default_os') + self.check_satisfies('foo platform=test target=frontend os=frontend', 'platform=test target=frontend os=frontend') + self.check_satisfies('foo platform=test target=backend os=backend', 'platform=test target=backend', 'platform=test os=backend') + self.check_satisfies('foo platform=test target=default_target os=default_os','platform=test target=default_target os=default_os') #ifdef NEW @@ -391,14 +390,8 @@ def test_invalid_constraint(self): self.check_invalid_constraint('libelf debug=2', 'libelf debug=1') self.check_invalid_constraint('libelf cppflags="-O3"', 'libelf cppflags="-O2"') - platform = spack.architecture.sys_type() - if len(platform.operating_sys.keys()) > 1 or len(platform.targets.keys()) > 1: - os1 = platform.operating_sys.keys()[0] - os2 = platform.operating_sys.keys()[-1] - target1 = platform.targets.keys()[0] - target2 = platform.targets.keys()[-1] - self.check_invalid_constraint('libelf target=%s os=%s' % (target1, os1), - 'libelf target=%s os=%s' % (target2, os2)) + self.check_invalid_constraint('libelf platform=test target=be os=be', + 'libelf target=fe os=fe') def test_constrain_changed(self): self.check_constrain_changed('libelf', '@1.0') From 0f40174723612c60ca7bdc0ed0ce1e96daa643bd Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 25 May 2016 12:28:59 -0700 Subject: [PATCH 260/481] added test platform --- lib/spack/spack/platforms/test.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 lib/spack/spack/platforms/test.py diff --git a/lib/spack/spack/platforms/test.py b/lib/spack/spack/platforms/test.py new file mode 100644 index 0000000000..3a4f9d5f2d --- /dev/null +++ b/lib/spack/spack/platforms/test.py @@ -0,0 +1,28 @@ +import subprocess +from spack.architecture import Platform, Target +from spack.operating_systems.linux_distro import LinuxDistro +from spack.operating_systems.cnl import Cnl + + +class Test(Platform): + priority = 1000000 + front_end = 'x86_32' + back_end = 'x86_64' + default = 'x86_64' + + back_os = 'CNL' + default_os = 'CNL' + + def __init__(self): + super(Test, self).__init__('test') + self.add_target(self.default, Target(self.default)) + self.add_target(self.front_end, Target(self.front_end)) + + self.add_operating_system(self.default_os, Cnl()) + linux_dist = LinuxDistro() + self.front_os = linux_dist.name + self.add_operating_system(self.front_os, linux_dist) + + @classmethod + def detect(self): + return True From 8f99334f11edd4b56f3e9bc88d2f8666cf0a37a1 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 25 May 2016 16:06:14 -0700 Subject: [PATCH 261/481] Fixed test config to work on yosemite --- lib/spack/spack/architecture.py | 38 +++++++++++----------- lib/spack/spack/test/mock_packages_test.py | 24 +++++++++++++- 2 files changed, 42 insertions(+), 20 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index e9dd489e54..5d16a2f5f5 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -22,26 +22,26 @@ # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -""" -This module contains all the elements that are required to create an -architecture object. These include, the target processor, the operating system, -and the architecture platform (i.e. cray, darwin, linux, bgq, etc) classes. +""" +This module contains all the elements that are required to create an +architecture object. These include, the target processor, the operating system, +and the architecture platform (i.e. cray, darwin, linux, bgq, etc) classes. -On a multiple architecture machine, the architecture spec field can be set to -build a package against any target and operating system that is present on the +On a multiple architecture machine, the architecture spec field can be set to +build a package against any target and operating system that is present on the platform. On Cray platforms or any other architecture that has different front and back end environments, the operating system will determine the method of compiler -detection. +detection. -There are two different types of compiler detection: +There are two different types of compiler detection: 1. Through the $PATH env variable (front-end detection) 2. Through the tcl module system. (back-end detection) Depending on which operating system is specified, the compiler will be detected -using one of those methods. +using one of those methods. For platforms such as linux and darwin, the operating system is autodetected and -the target is set to be x86_64. +the target is set to be x86_64. The command line syntax for specifying an architecture is as follows: @@ -52,27 +52,27 @@ set in the 'platforms/' directory which contains the different subclasses for platforms. If the machine has multiple architectures, the user can also enter front-end, or fe or back-end or be. These settings will concretize -to their respective front-end and back-end targets and operating systems. +to their respective front-end and back-end targets and operating systems. Additional platforms can be added by creating a subclass of Platform -and adding it inside the platform directory. +and adding it inside the platform directory. Platforms are an abstract class that are extended by subclasses. If the user wants to add a new type of platform (such as cray_xe), they can create a subclass -and set all the class attributes such as priority, front_target ,back_target, +and set all the class attributes such as priority, front_target ,back_target, front_os, back_os. Platforms also contain a priority class attribute. A lower number signifies higher priority. These numbers are arbitrarily set and can be changed though often there isn't much need unless a new platform is added and -the user wants that to be detected first. +the user wants that to be detected first. Targets are created inside the platform subclasses. Most architecture (like linux, and darwin) will have only one target (x86_64) but in the case of Cray machines, there is both a frontend and backend processor. The user can specify which targets are present on front-end and back-end architecture -Depending on the platform, operating systems are either auto-detected or are +Depending on the platform, operating systems are either auto-detected or are set. The user can set the front-end and back-end operating setting by the class attributes front_os and back_os. The operating system as described earlier, will -be responsible for compiler detection. +be responsible for compiler detection. """ import os from collections import namedtuple @@ -216,7 +216,7 @@ def operating_system(self, name): return self.operating_sys.get(name, None) - + @classmethod def detect(self): """ Subclass is responsible for implementing this method. @@ -235,7 +235,7 @@ def __str__(self): def _cmp_key(self): t_keys = ''.join(str(t._cmp_key()) for t in sorted(self.targets.values())) o_keys = ''.join(str(o._cmp_key()) for o in sorted(self.operating_sys.values())) - return (self.name, + return (self.name, self.default, self.front_end, self.back_end, @@ -385,7 +385,7 @@ def concrete(self): def __str__(self): if self.platform or self.platform_os or self.target: if self.platform.name == 'darwin': - os_name = self.platform_os.name + os_name = self.platform_os.name if self.platform_os else "None" else: os_name = str(self.platform_os) diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index 79dbdb89a6..0cdf01f1b9 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -69,6 +69,17 @@ f77: None fc: None modules: 'None' + clang3.3OSX: + spec: clang@3.3 + operating_system: + name: yosemite + version: '10.10' + paths: + cc: /path/to/clang + cxx: /path/to/clang++ + f77: None + fc: None + modules: 'None' gcc4.5.0CNL: paths: cc: /path/to/gcc @@ -98,10 +109,21 @@ f77: /path/to/gfortran fc: /path/to/gfortran operating_system: - name: RHL + name: redhat version: '6.7' spec: gcc@4.5.0 modules: 'None' + gcc4.5.0OSX: + paths: + cc: /path/to/gcc + cxx: /path/to/g++ + f77: /path/to/gfortran + fc: /path/to/gfortran + operating_system: + name: yosemite + version: '10.10' + spec: gcc@4.5.0 + modules: 'None' """ mock_packages_config = """\ From c30fe932d96b4a6fbe8845a2f6bdbfb7567ab1d2 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 25 May 2016 16:34:45 -0700 Subject: [PATCH 262/481] fixed mock config to use truncated versions of linux distro versions --- lib/spack/spack/test/mock_packages_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index 0cdf01f1b9..23b1cf52cc 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -62,7 +62,7 @@ spec: clang@3.3 operating_system: name: redhat - version: '6.7' + version: '6' paths: cc: /path/to/clang cxx: /path/to/clang++ @@ -110,7 +110,7 @@ fc: /path/to/gfortran operating_system: name: redhat - version: '6.7' + version: '6' spec: gcc@4.5.0 modules: 'None' gcc4.5.0OSX: From 3b675d8b700369a5b5282e56ecaf6c559e592551 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 26 May 2016 11:09:14 -0700 Subject: [PATCH 263/481] Changed comparison operator so that we don't get caught in an infinite loop --- lib/spack/spack/concretize.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 7a24c1135e..e30cf329f1 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -330,6 +330,8 @@ def concretize_compiler_flags(self, spec): compiler is used, defaulting to no compiler flags in the spec. Default specs set at the compiler level will still be added later. """ + + if not spec.architecture.platform_os: #Although this usually means changed, this means awaiting other changes return True @@ -340,9 +342,9 @@ def concretize_compiler_flags(self, spec): nearest = next(p for p in spec.traverse(direction='parents') if ((p.compiler == spec.compiler and p is not spec) and flag in p.compiler_flags)) - if ((not flag in spec.compiler_flags) or - sorted(spec.compiler_flags[flag]) != sorted(nearest.compiler_flags[flag])): - if flag in spec.compiler_flags: + if not flag in spec.compiler_flags or \ + not (sorted(spec.compiler_flags[flag]) >= sorted(nearest.compiler_flags[flag])): + if flag in spec.compiler_flag: spec.compiler_flags[flag] = list(set(spec.compiler_flags[flag]) | set(nearest.compiler_flags[flag])) else: From b968603a57a4085b05318c295443d370f4b4c270 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Thu, 26 May 2016 11:14:05 -0700 Subject: [PATCH 264/481] made yaml format backwards compatible --- lib/spack/spack/architecture.py | 34 +++++++++++-------- lib/spack/spack/operating_systems/pre_v1.py | 17 ++++++++++ .../spack/platforms/spack_compatibility.py | 23 +++++++++++++ 3 files changed, 60 insertions(+), 14 deletions(-) create mode 100644 lib/spack/spack/operating_systems/pre_v1.py create mode 100644 lib/spack/spack/platforms/spack_compatibility.py diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 5d16a2f5f5..86733ba418 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -90,7 +90,7 @@ from spack.util.environment import get_path from spack.util.multiproc import parmap import spack.error as serr - +import spack.platforms class InvalidSysTypeError(serr.SpackError): def __init__(self, sys_type): @@ -462,21 +462,27 @@ def arch_from_dict(d): """ arch = Arch() - if d is None: - return None - platform_dict = d['platform'] - os_dict = d['platform_os'] - target_dict = d['target'] + if isinstance(d, basestring): + # We have an old spec using a string for the architecture + arch.platform = spack.platforms.spack_compatibility.SpackCompatibility() + arch.platform_os = arch.platform.operating_system('default') + arch.target = Target(d) - platform = _platform_from_dict(platform_dict) if platform_dict else None - target = _target_from_dict(target_dict) if os_dict else None - platform_os = _operating_system_from_dict(os_dict) if os_dict else None - arch.platform = platform - arch.target = target - arch.platform_os = platform_os + arch.os_string = None + arch.target_string = None + else: + if d is None: + return None + platform_dict = d['platform'] + os_dict = d['platform_os'] + target_dict = d['target'] - arch.os_string = None - arch.target_string = None + arch.platform = _platform_from_dict(platform_dict) if platform_dict else None + arch.target = _target_from_dict(target_dict) if os_dict else None + arch.platform_os = _operating_system_from_dict(os_dict) if os_dict else None + + arch.os_string = None + arch.target_string = None return arch diff --git a/lib/spack/spack/operating_systems/pre_v1.py b/lib/spack/spack/operating_systems/pre_v1.py new file mode 100644 index 0000000000..71ee557ac1 --- /dev/null +++ b/lib/spack/spack/operating_systems/pre_v1.py @@ -0,0 +1,17 @@ +import re +import os + +from spack.architecture import OperatingSystem + + +class PreV1(OperatingSystem): + """ Compute Node Linux (CNL) is the operating system used for the Cray XC + series super computers. It is a very stripped down version of GNU/Linux. + Any compilers found through this operating system will be used with + modules. If updated, user must make sure that version and name are + updated to indicate that OS has been upgraded (or downgraded) + """ + def __init__(self): + name = 'PreVersion1.0' + version = '1.0' + super(PreV1, self).__init__(name, version) diff --git a/lib/spack/spack/platforms/spack_compatibility.py b/lib/spack/spack/platforms/spack_compatibility.py new file mode 100644 index 0000000000..86cbfb3044 --- /dev/null +++ b/lib/spack/spack/platforms/spack_compatibility.py @@ -0,0 +1,23 @@ +import subprocess +from spack.architecture import Platform +from spack.operating_systems.pre_v1 + +class SpackCompatibility(Platform): + priority = 9999 + + # We don't use the normal target getters for this platform + # Instead, targets are added directly when parsing the yaml + + # OS is the spack backwards compatibility os. + front_os = 'PreVersion1.0' + back_os = 'PreVersion1.0' + default_os = 'PreVersion1.0' + + def __init__(self): + super(SpackCompatibility, self).__init__('spack_compatibility') + sc_os = spack.operating_systems.pre_v1.PreV1() + self.add_operating_system(sc_os.name, sc_os) + + @classmethod + def detect(self): + return True From c0661744cccffc6ca11a82cea840e7be63736d37 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Thu, 26 May 2016 11:38:40 -0700 Subject: [PATCH 265/481] fixed compatibility work --- lib/spack/spack/architecture.py | 5 ++-- lib/spack/spack/operating_systems/pre_v1.py | 17 -------------- .../spack/platforms/spack_compatibility.py | 23 ------------------- 3 files changed, 2 insertions(+), 43 deletions(-) delete mode 100644 lib/spack/spack/operating_systems/pre_v1.py delete mode 100644 lib/spack/spack/platforms/spack_compatibility.py diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 86733ba418..a0ef4f14da 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -90,7 +90,6 @@ from spack.util.environment import get_path from spack.util.multiproc import parmap import spack.error as serr -import spack.platforms class InvalidSysTypeError(serr.SpackError): def __init__(self, sys_type): @@ -464,8 +463,8 @@ def arch_from_dict(d): if isinstance(d, basestring): # We have an old spec using a string for the architecture - arch.platform = spack.platforms.spack_compatibility.SpackCompatibility() - arch.platform_os = arch.platform.operating_system('default') + arch.platform = Platform('spack_compatibility') + arch.platform_os = OperatingSystem('pre_version', '1.0') arch.target = Target(d) arch.os_string = None diff --git a/lib/spack/spack/operating_systems/pre_v1.py b/lib/spack/spack/operating_systems/pre_v1.py deleted file mode 100644 index 71ee557ac1..0000000000 --- a/lib/spack/spack/operating_systems/pre_v1.py +++ /dev/null @@ -1,17 +0,0 @@ -import re -import os - -from spack.architecture import OperatingSystem - - -class PreV1(OperatingSystem): - """ Compute Node Linux (CNL) is the operating system used for the Cray XC - series super computers. It is a very stripped down version of GNU/Linux. - Any compilers found through this operating system will be used with - modules. If updated, user must make sure that version and name are - updated to indicate that OS has been upgraded (or downgraded) - """ - def __init__(self): - name = 'PreVersion1.0' - version = '1.0' - super(PreV1, self).__init__(name, version) diff --git a/lib/spack/spack/platforms/spack_compatibility.py b/lib/spack/spack/platforms/spack_compatibility.py deleted file mode 100644 index 86cbfb3044..0000000000 --- a/lib/spack/spack/platforms/spack_compatibility.py +++ /dev/null @@ -1,23 +0,0 @@ -import subprocess -from spack.architecture import Platform -from spack.operating_systems.pre_v1 - -class SpackCompatibility(Platform): - priority = 9999 - - # We don't use the normal target getters for this platform - # Instead, targets are added directly when parsing the yaml - - # OS is the spack backwards compatibility os. - front_os = 'PreVersion1.0' - back_os = 'PreVersion1.0' - default_os = 'PreVersion1.0' - - def __init__(self): - super(SpackCompatibility, self).__init__('spack_compatibility') - sc_os = spack.operating_systems.pre_v1.PreV1() - self.add_operating_system(sc_os.name, sc_os) - - @classmethod - def detect(self): - return True From 88bec814eb45ff4997229e092f234d8ce7c76664 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Thu, 26 May 2016 12:10:41 -0700 Subject: [PATCH 266/481] Allow 'arch=' syntax for specs to allow copying from output --- lib/spack/spack/concretize.py | 2 +- lib/spack/spack/platforms/cray_xc.py | 12 ++++++------ lib/spack/spack/platforms/linux.py | 8 ++++---- lib/spack/spack/spec.py | 18 +++++++++++------- lib/spack/spack/test/spec_semantics.py | 1 - 5 files changed, 22 insertions(+), 19 deletions(-) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index e30cf329f1..5d9715feed 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -344,7 +344,7 @@ def concretize_compiler_flags(self, spec): and flag in p.compiler_flags)) if not flag in spec.compiler_flags or \ not (sorted(spec.compiler_flags[flag]) >= sorted(nearest.compiler_flags[flag])): - if flag in spec.compiler_flag: + if flag in spec.compiler_flags: spec.compiler_flags[flag] = list(set(spec.compiler_flags[flag]) | set(nearest.compiler_flags[flag])) else: diff --git a/lib/spack/spack/platforms/cray_xc.py b/lib/spack/spack/platforms/cray_xc.py index 4843a47c62..e710303e23 100644 --- a/lib/spack/spack/platforms/cray_xc.py +++ b/lib/spack/spack/platforms/cray_xc.py @@ -9,16 +9,16 @@ class CrayXc(Platform): back_end = 'ivybridge' default = 'ivybridge' - front_os = "SuSE" - back_os = "CNL" - default_os = "CNL" + front_os = "SuSE11" + back_os = "CNL10" + default_os = "CNL10" def __init__(self): ''' Since cori doesn't have ivybridge as a front end it's better if we use CRAY_CPU_TARGET as the default. This will ensure that if we're on a XC-40 or XC-30 then we can detect the target ''' - super(CrayXc, self).__init__('crayxc') + super(CrayXc, self).__init__('cray_xc') # Handle the default here so we can check for a key error if 'CRAY_CPU_TARGET' in os.environ: @@ -37,8 +37,8 @@ def __init__(self): self.add_target('haswell', Target('haswell','craype-haswell')) - self.add_operating_system('SuSE', LinuxDistro()) - self.add_operating_system('CNL', Cnl()) + self.add_operating_system('SuSE11', LinuxDistro()) + self.add_operating_system('CNL10', Cnl()) @classmethod def detect(self): diff --git a/lib/spack/spack/platforms/linux.py b/lib/spack/spack/platforms/linux.py index 18050ac79e..ebe018812a 100644 --- a/lib/spack/spack/platforms/linux.py +++ b/lib/spack/spack/platforms/linux.py @@ -12,10 +12,10 @@ def __init__(self): super(Linux, self).__init__('linux') self.add_target(self.default, Target(self.default)) linux_dist = LinuxDistro() - self.default_os = linux_dist.name - self.front_os = linux_dist.name - self.back_os = linux_dist.name - self.add_operating_system(linux_dist.name, linux_dist) + self.default_os = str(linux_dist) + self.front_os = self.default_os + self.back_os = self.default._os + self.add_operating_system(str(linux_dist), linux_dist) @classmethod def detect(self): diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 5d82af4af6..58d76fd7a9 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -538,13 +538,17 @@ def _add_flag(self, name, value): Known flags currently include "arch" """ valid_flags = FlagMap.valid_compiler_flags() -# if name == 'arch' or name == 'architecture': -# platform, op_sys, target = value.split('-') -# print platform, op_sys, target, '+++++++' -# self._set_platform(platform) -# self._set_os(op_sys) -# self._set_target(target) - if name == 'platform': + if name == 'arch' or name == 'architecture': + platform, op_sys, target = value.split('-') + assert(self.architecture.platform is None) + assert(self.architecture.platform_os is None) + assert(self.architecture.target is None) + assert(self.architecture.os_string is None) + assert(self.architecture.target_string is None) + self._set_platform(platform) + self._set_os(op_sys) + self._set_target(target) + elif name == 'platform': self._set_platform(value) elif name == 'os' or name == 'operating_system': if self.architecture.platform: diff --git a/lib/spack/spack/test/spec_semantics.py b/lib/spack/spack/test/spec_semantics.py index aa99ad71ed..0b2111bfe8 100644 --- a/lib/spack/spack/test/spec_semantics.py +++ b/lib/spack/spack/test/spec_semantics.py @@ -402,7 +402,6 @@ def test_constrain_changed(self): self.check_constrain_changed('libelf', '~debug') self.check_constrain_changed('libelf', 'debug=2') self.check_constrain_changed('libelf', 'cppflags="-O3"') - self.check_constrain_changed('libelf', ' arch=bgqos_0') platform = spack.architecture.sys_type() self.check_constrain_changed('libelf', 'target='+platform.target('default_target').name) From 189e8b3f349c8102772fa995fd8ee79df3c242c0 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Thu, 26 May 2016 13:17:01 -0700 Subject: [PATCH 267/481] fixed compiler config init logic --- lib/spack/spack/compilers/__init__.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 8fe7a17116..ea79f655b0 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -102,15 +102,18 @@ def init_compiler_config(): # configured. Avoid updating automatically if there ARE site # compilers configured but no user ones. # if (isinstance(arch, basestring) or arch == my_arch) and arch not in config: - if scope is None: - # We know no compilers were configured in any scope. - init_compiler_config() - elif scope == 'user': - # Check the site config and update the user config if - # nothing is configured at the site level. - site_config = spack.config.get_config('compilers', scope='site') - if not site_config: + if not config: + if scope is None: + # We know no compilers were configured in any scope. init_compiler_config() + config = spack.config.get_config('compilers', scope=scope) + elif scope == 'user': + # Check the site config and update the user config if + # nothing is configured at the site level. + site_config = spack.config.get_config('compilers', scope='site') + if not site_config: + init_compiler_config() + config = spack.config.get_config('compilers', scope=scope) return config From 9c1da4fa4cdfe32d8a7a0026f457066b0cac7e8e Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 26 May 2016 13:58:17 -0700 Subject: [PATCH 268/481] Corrected parantheses wrapping --- lib/spack/spack/build_environment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 073f73a91a..4483b9d132 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -366,7 +366,7 @@ def get_rpaths(pkg): if os.path.isdir(d.prefix.lib64)) # Second module is our compiler mod name. We use that to get rpaths from # module show output. - if pkg.compiler.modules and len(pkg.compiler.modules > 1): + if pkg.compiler.modules and len(pkg.compiler.modules) > 1: rpaths.append(get_path_from_module(pkg.compiler.modules[1])) return rpaths From a2f90453f4d1ac1372f717911bd08e33d801ff31 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Thu, 26 May 2016 20:30:05 -0700 Subject: [PATCH 269/481] Fixed backwards compatibility for compilers.yaml config file --- bin/spack | 3 ++ lib/spack/spack/yaml_version_check.py | 55 +++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 lib/spack/spack/yaml_version_check.py diff --git a/bin/spack b/bin/spack index 3544feb10a..9b1276a866 100755 --- a/bin/spack +++ b/bin/spack @@ -138,6 +138,9 @@ def main(): import spack.util.debug as debug debug.register_interrupt_handler() + from spack.yaml_version_check import check_yaml_versions + check_yaml_versions() + spack.spack_working_dir = working_dir if args.mock: from spack.repository import RepoPath diff --git a/lib/spack/spack/yaml_version_check.py b/lib/spack/spack/yaml_version_check.py new file mode 100644 index 0000000000..7e7bae4edf --- /dev/null +++ b/lib/spack/spack/yaml_version_check.py @@ -0,0 +1,55 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +"""Yaml Version Check is a module for ensuring that config file +formats are compatible with the current version of Spack.""" +import os.path +import os +import llnl.util.tty as tty +import spack.util.spack_yaml as syaml +import spack.config + + +def check_yaml_versions(): + check_compiler_yaml_version() + +def check_compiler_yaml_version(): + config_scopes = spack.config.config_scopes + for scope in config_scopes.values(): + file_name = os.path.join(scope.path, 'compilers.yaml') + data = None + if os.path.isfile(file_name): + with open(file_name) as f: + data = syaml.load(f) + + if data: + compilers = data['compilers'].items() + if len(compilers) > 0: + if 'operating_system' not in compilers[0][1]: + new_file = os.path.join(scope.path, '_old_compilers.yaml') + tty.warn('%s in out of date compilers format. ' + 'Moved to %s. Spack automatically generate ' + 'a compilers config file ' + % (file_name, new_file)) + os.rename(file_name, new_file) From dd5a6914f161f2237a90df856126a905e305d80f Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Fri, 27 May 2016 12:56:43 -0500 Subject: [PATCH 270/481] Undo changes to flex --- var/spack/repos/builtin/packages/flex/package.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/var/spack/repos/builtin/packages/flex/package.py b/var/spack/repos/builtin/packages/flex/package.py index f41af061a5..926651010f 100644 --- a/var/spack/repos/builtin/packages/flex/package.py +++ b/var/spack/repos/builtin/packages/flex/package.py @@ -24,7 +24,6 @@ ############################################################################## from spack import * - class Flex(Package): """Flex is a tool for generating scanners.""" @@ -34,8 +33,6 @@ class Flex(Package): version('2.6.0', '5724bcffed4ebe39e9b55a9be80859ec') version('2.5.39', 'e133e9ead8ec0a58d81166b461244fde') - depends_on('bison') - def install(self, spec, prefix): configure("--prefix=%s" % prefix) From 023504ed645fecc7e7c12948dac890b65124c880 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 27 May 2016 12:26:59 -0700 Subject: [PATCH 271/481] Fixed the external modules bug --- lib/spack/spack/spec.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 58d76fd7a9..bbc1abfa9e 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1065,7 +1065,7 @@ def _expand_virtual_packages(self): continue # If replacement is external then trim the dependencies - if replacement.external: + if replacement.external or replacement.external_module: if (spec.dependencies): changed = True spec.dependencies = DependencyMap() @@ -1082,7 +1082,8 @@ def feq(cfield, sfield): feq(replacement.architecture, spec.architecture) and feq(replacement.dependencies, spec.dependencies) and feq(replacement.variants, spec.variants) and - feq(replacement.external, spec.external)): + feq(replacement.external, spec.external) and + feq(replacement.external_module, spec.external_module)): continue # Refine this spec to the candidate. This uses @@ -1653,18 +1654,9 @@ def satisfies(self, other, deps=True, strict=False): # TODO: Need to make sure that comparisons can be made via classes if self.architecture and other.architecture: - print self.architecture, other.architecture if ((self.architecture.platform and other.architecture.platform and self.architecture.platform != other.architecture.platform) or (self.architecture.platform_os and other.architecture.platform_os and self.architecture.platform_os != other.architecture.platform_os) or (self.architecture.target and other.architecture.target and self.architecture.target != other.architecture.target)): - d1 = self.architecture.platform.to_dict() - d2 = other.architecture.platform.to_dict() - print d1 - print d2 - print d1==d2 - print self.architecture.platform == other.architecture.platform - print self.architecture.platform._cmp_key() - print other.architecture.platform._cmp_key() return False elif strict and ((other.architecture and not self.architecture) or (other.architecture.platform and not self.architecture.platform) or From f49644cdea3b00b057f33bf0c609e3a83eb3cd65 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 27 May 2016 12:37:58 -0700 Subject: [PATCH 272/481] Fixed typo in linux platform --- lib/spack/spack/platforms/linux.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/platforms/linux.py b/lib/spack/spack/platforms/linux.py index ebe018812a..4d8adac384 100644 --- a/lib/spack/spack/platforms/linux.py +++ b/lib/spack/spack/platforms/linux.py @@ -14,7 +14,7 @@ def __init__(self): linux_dist = LinuxDistro() self.default_os = str(linux_dist) self.front_os = self.default_os - self.back_os = self.default._os + self.back_os = self.default_os self.add_operating_system(str(linux_dist), linux_dist) @classmethod From becec8ac7e17a9d56f1d5b0640c1f56be8f64a7f Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Sat, 28 May 2016 10:12:53 -0700 Subject: [PATCH 273/481] Added external module check in normalize to avoid infinite loop --- lib/spack/spack/spec.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index a505b3c12e..f8bad3ff03 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1002,11 +1002,12 @@ def _expand_virtual_packages(self): """ # Make an index of stuff this spec already provides self_index = ProviderIndex(self.traverse(), restrict=True) - changed = False done = False + while not done: done = True + for spec in list(self.traverse()): replacement = None if spec.virtual: @@ -1329,7 +1330,7 @@ def _normalize_helper(self, visited, spec_deps, provider_index): # if we descend into a virtual spec, there's nothing more # to normalize. Concretize will finish resolving it later. - if self.virtual or self.external: + if self.virtual or self.external or self.external_module: return False # Combine constraints from package deps with constraints from @@ -1376,7 +1377,6 @@ def normalize(self, force=False): # Ensure first that all packages & compilers in the DAG exist. self.validate_names() - # Get all the dependencies into one DependencyMap spec_deps = self.flat_dependencies(copy=False) From 98a4a9f2cc67c141deb7903f577c60c21912667c Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Sat, 28 May 2016 10:13:22 -0700 Subject: [PATCH 274/481] Adding test for external modules --- lib/spack/spack/test/concretize.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index 963481054e..77ff026c7a 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -253,6 +253,13 @@ def test_external_package(self): self.assertTrue(spec['externaltool'].compiler.satisfies('gcc')) + def test_external_package_module(self): + spec = Spec('externalmodule') + spec.concretize() + self.assertEqual(spec['externalmodule'].external_module, 'external-module') + self.assertFalse('externalprereq' in spec) + self.assertTrue(spec['externalmodule'].compiler.satisfies('gcc')) + def test_nobuild_package(self): got_error = False spec = Spec('externaltool%clang') From f0a9c45207eabc81b5f50b68f6dc02f807f1a221 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Sat, 28 May 2016 10:13:57 -0700 Subject: [PATCH 275/481] Adding external to mock config --- lib/spack/spack/test/mock_packages_test.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index 23b1cf52cc..5561027aac 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -137,6 +137,10 @@ paths: externalvirtual@2.0%clang@3.3: /path/to/external_virtual_clang externalvirtual@1.0%gcc@4.5.0: /path/to/external_virtual_gcc + externalmodule: + buildable: False + modules: + externalmodule@1.0%gcc@4.5.0: external-module """ class MockPackagesTest(unittest.TestCase): From 33e1dcc4763024db87e709b3db6ff4ee12d43dc5 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Sat, 28 May 2016 10:14:40 -0700 Subject: [PATCH 276/481] Adding mock package file for external module --- .../packages/externalmodule/package.py | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 var/spack/repos/builtin.mock/packages/externalmodule/package.py diff --git a/var/spack/repos/builtin.mock/packages/externalmodule/package.py b/var/spack/repos/builtin.mock/packages/externalmodule/package.py new file mode 100644 index 0000000000..f7b0da3fd9 --- /dev/null +++ b/var/spack/repos/builtin.mock/packages/externalmodule/package.py @@ -0,0 +1,37 @@ + +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class Externalmodule(Package): + homepage = "http://somewhere.com" + url = "http://somewhere.com/module-1.0.tar.gz" + + version('1.0', '1234567890abcdef1234567890abcdef') + + depends_on('externalprereq') + + def install(self, spec, prefix): + pass From 69585cb6e3b391d2771387830067d893f2e7ec32 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Sat, 28 May 2016 12:39:43 -0700 Subject: [PATCH 277/481] Added elcap compilers --- lib/spack/spack/test/mock_packages_test.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index 5561027aac..f02bfb7a07 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -124,6 +124,28 @@ version: '10.10' spec: gcc@4.5.0 modules: 'None' + gcc4.5.0ELCAP: + paths: + cc: /path/to/gcc + cxx: /path/to/g++ + f77: /path/to/gfortran + fc: /path/to/gfortran + operating_system: + name: elcapitan + version: '10.11' + spec: gcc@4.5.0 + modules: 'None' + clang3.3ELCAP: + spec: clang@3.3 + operating_system: + name: elcapitan + version: '10.11' + paths: + cc: /path/to/clang + cxx: /path/to/clang++ + f77: None + fc: None + modules: 'None' """ mock_packages_config = """\ From 22ca72e7b9870dddfd66760658a9221f4f24ffe3 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Sat, 28 May 2016 12:40:29 -0700 Subject: [PATCH 278/481] Added conditional skipIf to test_external_modules, darwin machines don't have tcl modules so skip this test and let user know why --- lib/spack/spack/test/concretize.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index 77ff026c7a..cf2daa4be2 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -23,6 +23,7 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import spack +import spack.architecture from spack.spec import Spec, CompilerSpec from spack.version import ver from spack.concretize import find_spec @@ -252,7 +253,8 @@ def test_external_package(self): self.assertFalse('externalprereq' in spec) self.assertTrue(spec['externaltool'].compiler.satisfies('gcc')) - + @unittest.skipIf(spack.architecture.sys_type().name == 'darwin', + "No tcl modules on darwin machines") def test_external_package_module(self): spec = Spec('externalmodule') spec.concretize() @@ -269,7 +271,7 @@ def test_nobuild_package(self): got_error = True self.assertTrue(got_error) - + def test_external_and_virtual(self): spec = Spec('externaltest') spec.concretize() From d7e84fe07c748206afa25e8e45a2ebfac71c7cdf Mon Sep 17 00:00:00 2001 From: Ali Dorostkar Date: Mon, 30 May 2016 14:03:27 +0200 Subject: [PATCH 279/481] dealii, test for specific version MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Specified that test case “step-40” is only to be run when using dealii version 8.4.0 or later --- .../repos/builtin/packages/dealii/package.py | 93 ++++++++++--------- 1 file changed, 47 insertions(+), 46 deletions(-) diff --git a/var/spack/repos/builtin/packages/dealii/package.py b/var/spack/repos/builtin/packages/dealii/package.py index 9c09b205a8..c871059075 100644 --- a/var/spack/repos/builtin/packages/dealii/package.py +++ b/var/spack/repos/builtin/packages/dealii/package.py @@ -228,54 +228,55 @@ def install(self, spec, prefix): # take step-40 which can use both PETSc and Trilinos # FIXME: switch step-40 to MPI run - with working_dir('examples/step-40'): - print('=====================================') - print('========== Step-40 PETSc ============') - print('=====================================') - # list the number of cycles to speed up - filter_file(r'(const unsigned int n_cycles = 8;)', - ('const unsigned int n_cycles = 2;'), 'step-40.cc') - cmake('.') - if '^petsc' in spec: - make('release') - make('run', parallel=False) + if spec.satisfies('@8.4.0:'): + with working_dir('examples/step-40'): + print('=====================================') + print('========== Step-40 PETSc ============') + print('=====================================') + # list the number of cycles to speed up + filter_file(r'(const unsigned int n_cycles = 8;)', + ('const unsigned int n_cycles = 2;'), 'step-40.cc') + cmake('.') + if '^petsc' in spec: + make('release') + make('run', parallel=False) + + print('=====================================') + print('========= Step-40 Trilinos ==========') + print('=====================================') + # change Linear Algebra to Trilinos + filter_file(r'(\/\/ #define FORCE_USE_OF_TRILINOS.*)', + ('#define FORCE_USE_OF_TRILINOS'), 'step-40.cc') + if '^trilinos+hypre' in spec: + make('release') + make('run', parallel=False) - print('=====================================') - print('========= Step-40 Trilinos ==========') - print('=====================================') - # change Linear Algebra to Trilinos - filter_file(r'(\/\/ #define FORCE_USE_OF_TRILINOS.*)', - ('#define FORCE_USE_OF_TRILINOS'), 'step-40.cc') - if '^trilinos+hypre' in spec: - make('release') - make('run', parallel=False) + print('=====================================') + print('=== Step-40 Trilinos SuperluDist ====') + print('=====================================') + # change to direct solvers + filter_file(r'(LA::SolverCG solver\(solver_control\);)', ('TrilinosWrappers::SolverDirect::AdditionalData data(false,"Amesos_Superludist"); TrilinosWrappers::SolverDirect solver(solver_control,data);'), 'step-40.cc') # NOQA: ignore=E501 + filter_file(r'(LA::MPI::PreconditionAMG preconditioner;)', + (''), 'step-40.cc') + filter_file(r'(LA::MPI::PreconditionAMG::AdditionalData data;)', + (''), 'step-40.cc') + filter_file(r'(preconditioner.initialize\(system_matrix, data\);)', + (''), 'step-40.cc') + filter_file(r'(solver\.solve \(system_matrix, completely_distributed_solution, system_rhs,)', ('solver.solve (system_matrix, completely_distributed_solution, system_rhs);'), 'step-40.cc') # NOQA: ignore=E501 + filter_file(r'(preconditioner\);)', (''), 'step-40.cc') + if '^trilinos+superlu-dist' in spec: + make('release') + make('run', paralle=False) - print('=====================================') - print('=== Step-40 Trilinos SuperluDist ====') - print('=====================================') - # change to direct solvers - filter_file(r'(LA::SolverCG solver\(solver_control\);)', ('TrilinosWrappers::SolverDirect::AdditionalData data(false,"Amesos_Superludist"); TrilinosWrappers::SolverDirect solver(solver_control,data);'), 'step-40.cc') # NOQA: ignore=E501 - filter_file(r'(LA::MPI::PreconditionAMG preconditioner;)', - (''), 'step-40.cc') - filter_file(r'(LA::MPI::PreconditionAMG::AdditionalData data;)', - (''), 'step-40.cc') - filter_file(r'(preconditioner.initialize\(system_matrix, data\);)', - (''), 'step-40.cc') - filter_file(r'(solver\.solve \(system_matrix, completely_distributed_solution, system_rhs,)', ('solver.solve (system_matrix, completely_distributed_solution, system_rhs);'), 'step-40.cc') # NOQA: ignore=E501 - filter_file(r'(preconditioner\);)', (''), 'step-40.cc') - if '^trilinos+superlu-dist' in spec: - make('release') - make('run', paralle=False) - - print('=====================================') - print('====== Step-40 Trilinos MUMPS =======') - print('=====================================') - # switch to Mumps - filter_file(r'(Amesos_Superludist)', - ('Amesos_Mumps'), 'step-40.cc') - if '^trilinos+mumps' in spec: - make('release') - make('run', parallel=False) + print('=====================================') + print('====== Step-40 Trilinos MUMPS =======') + print('=====================================') + # switch to Mumps + filter_file(r'(Amesos_Superludist)', + ('Amesos_Mumps'), 'step-40.cc') + if '^trilinos+mumps' in spec: + make('release') + make('run', parallel=False) print('=====================================') print('============ Step-36 ================') From 4f591a6fe3f2fc1477d5b740d1eb9e56a37d56b3 Mon Sep 17 00:00:00 2001 From: Ali Dorostkar Date: Mon, 30 May 2016 16:42:24 +0200 Subject: [PATCH 280/481] Step-40 test for 8.3.0 and earlier Integrated part of step-40 to be tested for earlier versions of dealii --- .../repos/builtin/packages/dealii/package.py | 44 ++++++++++--------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/var/spack/repos/builtin/packages/dealii/package.py b/var/spack/repos/builtin/packages/dealii/package.py index c871059075..e83347409e 100644 --- a/var/spack/repos/builtin/packages/dealii/package.py +++ b/var/spack/repos/builtin/packages/dealii/package.py @@ -228,29 +228,33 @@ def install(self, spec, prefix): # take step-40 which can use both PETSc and Trilinos # FIXME: switch step-40 to MPI run - if spec.satisfies('@8.4.0:'): - with working_dir('examples/step-40'): - print('=====================================') - print('========== Step-40 PETSc ============') - print('=====================================') - # list the number of cycles to speed up - filter_file(r'(const unsigned int n_cycles = 8;)', - ('const unsigned int n_cycles = 2;'), 'step-40.cc') - cmake('.') - if '^petsc' in spec: - make('release') - make('run', parallel=False) - - print('=====================================') - print('========= Step-40 Trilinos ==========') - print('=====================================') - # change Linear Algebra to Trilinos + with working_dir('examples/step-40'): + print('=====================================') + print('========== Step-40 PETSc ============') + print('=====================================') + # list the number of cycles to speed up + filter_file(r'(const unsigned int n_cycles = 8;)', + ('const unsigned int n_cycles = 2;'), 'step-40.cc') + cmake('.') + if '^petsc' in spec: + make('release') + make('run', parallel=False) + + print('=====================================') + print('========= Step-40 Trilinos ==========') + print('=====================================') + # change Linear Algebra to Trilinos + if spec.satisfies('@8.4.0:'): filter_file(r'(\/\/ #define FORCE_USE_OF_TRILINOS.*)', ('#define FORCE_USE_OF_TRILINOS'), 'step-40.cc') - if '^trilinos+hypre' in spec: - make('release') - make('run', parallel=False) + else: + filter_file(r'(#define USE_PETSC_LA.*)', + ('// #define USE_PETSC_LA'), 'step-40.cc') + if '^trilinos+hypre' in spec: + make('release') + make('run', parallel=False) + if spec.satisfies('@8.4.0:'): print('=====================================') print('=== Step-40 Trilinos SuperluDist ====') print('=====================================') From c4d5bd13c7596577bef119dcac2b83e213a92ce5 Mon Sep 17 00:00:00 2001 From: Ali Dorostkar Date: Mon, 30 May 2016 18:11:34 +0200 Subject: [PATCH 281/481] dealii, Added comment Added comment for why step-40 is done differently for dealii 8.4.0 and later --- var/spack/repos/builtin/packages/dealii/package.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/var/spack/repos/builtin/packages/dealii/package.py b/var/spack/repos/builtin/packages/dealii/package.py index e83347409e..2b148da074 100644 --- a/var/spack/repos/builtin/packages/dealii/package.py +++ b/var/spack/repos/builtin/packages/dealii/package.py @@ -244,6 +244,8 @@ def install(self, spec, prefix): print('========= Step-40 Trilinos ==========') print('=====================================') # change Linear Algebra to Trilinos + # The below filter_file should be different for versions + # before and after 8.4.0 if spec.satisfies('@8.4.0:'): filter_file(r'(\/\/ #define FORCE_USE_OF_TRILINOS.*)', ('#define FORCE_USE_OF_TRILINOS'), 'step-40.cc') @@ -254,6 +256,8 @@ def install(self, spec, prefix): make('release') make('run', parallel=False) + # the rest of the tests on step 40 only works for + # dealii version 8.4.0 and after if spec.satisfies('@8.4.0:'): print('=====================================') print('=== Step-40 Trilinos SuperluDist ====') From f96c97902452761cbf2bff64d3efa1240df70771 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 30 May 2016 13:05:49 -0700 Subject: [PATCH 282/481] Added a conditional to check if package is external --- lib/spack/spack/concretize.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 5d9715feed..58811c3419 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -84,7 +84,8 @@ def _valid_virtuals_and_externals(self, spec): raise NoBuildError(spec) def cmp_externals(a, b): - if a.name != b.name: + if a.name != b.name and (not a.external or a.external_module and + not b.external and b.external_module): # We're choosing between different providers, so # maintain order from provider sort return candidates.index(a) - candidates.index(b) From 08a9d80ebcd239790d28904787ea5f6a9bf5fa5d Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 30 May 2016 13:07:20 -0700 Subject: [PATCH 283/481] Changed platform to py_platform to avoid name collisions. The naming collisions resulted in an error on linux machines --- lib/spack/spack/test/architecture.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/test/architecture.py b/lib/spack/spack/test/architecture.py index f5b1068435..3d08269dee 100644 --- a/lib/spack/spack/test/architecture.py +++ b/lib/spack/spack/test/architecture.py @@ -3,7 +3,7 @@ """ import unittest import os -import platform +import platform as py_platform import spack from spack.architecture import * from spack.spec import * @@ -61,9 +61,9 @@ def test_sys_type(self): my_platform_class = CrayXc() elif os.path.exists('/bgsys'): my_platform_class = Bgq() - elif 'Linux' in platform.system(): + elif 'Linux' in py_platform.system(): my_platform_class = Linux() - elif 'Darwin' in platform.system(): + elif 'Darwin' in py_platform.system(): my_platform_class = Darwin() self.assertEqual(str(output_platform_class), str(my_platform_class)) From 716991775d6f6541e2acf41d51ff528e4c3412c7 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 30 May 2016 13:09:41 -0700 Subject: [PATCH 284/481] Added extra check on linux machines, since most linux machines don't have module system --- lib/spack/spack/test/concretize.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index cf2daa4be2..67b20b4084 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -253,8 +253,9 @@ def test_external_package(self): self.assertFalse('externalprereq' in spec) self.assertTrue(spec['externaltool'].compiler.satisfies('gcc')) - @unittest.skipIf(spack.architecture.sys_type().name == 'darwin', - "No tcl modules on darwin machines") + @unittest.skipIf(spack.architecture.sys_type().name == 'darwin' or + spack.architecture.sys_type().name == 'linux', + "No tcl modules on darwin/linux machines") def test_external_package_module(self): spec = Spec('externalmodule') spec.concretize() From b14ba3125033dc93afb9662ba87c2ca0acbbbde4 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 30 May 2016 13:22:08 -0700 Subject: [PATCH 285/481] Quick fix for mock compilers.yaml to work on linux machines. --- lib/spack/spack/test/mock_packages_test.py | 35 +++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index f02bfb7a07..561f319b16 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -34,8 +34,41 @@ from spack.repository import RepoPath from spack.spec import Spec +platform = spack.architecture.sys_type() + +linux_os_name = 'debian' +linux_os_version = '6' + +if platform.name == 'linux': + global linux_os_name, linux_os_version + linux_os = platform.operating_system("default_os") + linux_os_name = linux_os.name + linux_os_version = linux_os.version + mock_compiler_config = """\ compilers: + clang3.3GENLINUX: + spec: clang@3.3 + operating_system: + name: {0} + version: '{1}' + paths: + cc: /path/to/clang + cxx: /path/to/clang++ + f77: None + fc: None + modules: 'None' + gcc4.5GENLINUX: + spec: gcc@4.5.0 + operating_system: + name: {0} + version: '{1}' + paths: + cc: /path/to/gcc + cxx: /path/to/g++ + f77: None + fc: None + modules: 'None' clang3.3CNL: spec: clang@3.3 operating_system: @@ -146,7 +179,7 @@ f77: None fc: None modules: 'None' -""" +""".format(linux_os_name, linux_os_version) mock_packages_config = """\ packages: From 24ee32d7b0cb209f8a17f6df7e8715fa508d0f5d Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 30 May 2016 22:02:22 -0700 Subject: [PATCH 286/481] More flexible reading of specs from DB, formatting. --- lib/spack/spack/database.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py index e768ddf5fe..38bb7541e0 100644 --- a/lib/spack/spack/database.py +++ b/lib/spack/spack/database.py @@ -214,9 +214,10 @@ def _read_spec_from_yaml(self, hash_key, installs, parent_key=None): # Add dependencies from other records in the install DB to # form a full spec. - for dep_hash in spec_dict[spec.name]['dependencies'].values(): - child = self._read_spec_from_yaml(dep_hash, installs, hash_key) - spec._add_dependency(child) + if 'dependencies' in spec_dict[spec.name]: + for dep_hash in spec_dict[spec.name]['dependencies'].values(): + child = self._read_spec_from_yaml(dep_hash, installs, hash_key) + spec._add_dependency(child) # Specs from the database need to be marked concrete because # they represent actual installations. @@ -289,7 +290,8 @@ def check(cond, msg): except Exception as e: tty.warn("Invalid database reecord:", "file: %s" % self._index_path, - "hash: %s" % hash_key, "cause: %s" % str(e)) + "hash: %s" % hash_key, + "cause: %s: %s" % (type(e).__name__, str(e))) raise self._data = data From e82fb5b58f4ccdf7a4bfcda0331b4649dec3de75 Mon Sep 17 00:00:00 2001 From: Ali Dorostkar Date: Tue, 31 May 2016 08:44:46 +0200 Subject: [PATCH 287/481] Fixed spacing Replaced tabs with spaces --- var/spack/repos/builtin/packages/dealii/package.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/var/spack/repos/builtin/packages/dealii/package.py b/var/spack/repos/builtin/packages/dealii/package.py index 2b148da074..35658c75a0 100644 --- a/var/spack/repos/builtin/packages/dealii/package.py +++ b/var/spack/repos/builtin/packages/dealii/package.py @@ -244,8 +244,8 @@ def install(self, spec, prefix): print('========= Step-40 Trilinos ==========') print('=====================================') # change Linear Algebra to Trilinos - # The below filter_file should be different for versions - # before and after 8.4.0 + # The below filter_file should be different for versions + # before and after 8.4.0 if spec.satisfies('@8.4.0:'): filter_file(r'(\/\/ #define FORCE_USE_OF_TRILINOS.*)', ('#define FORCE_USE_OF_TRILINOS'), 'step-40.cc') @@ -256,8 +256,8 @@ def install(self, spec, prefix): make('release') make('run', parallel=False) - # the rest of the tests on step 40 only works for - # dealii version 8.4.0 and after + # the rest of the tests on step 40 only works for + # dealii version 8.4.0 and after if spec.satisfies('@8.4.0:'): print('=====================================') print('=== Step-40 Trilinos SuperluDist ====') From 6f96484f83dff008ef8712c63202ee140000ff22 Mon Sep 17 00:00:00 2001 From: Ali Dorostkar Date: Tue, 31 May 2016 09:58:29 +0200 Subject: [PATCH 288/481] Spacing correction More spacing correction --- var/spack/repos/builtin/packages/dealii/package.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/var/spack/repos/builtin/packages/dealii/package.py b/var/spack/repos/builtin/packages/dealii/package.py index 35658c75a0..54b6426d36 100644 --- a/var/spack/repos/builtin/packages/dealii/package.py +++ b/var/spack/repos/builtin/packages/dealii/package.py @@ -244,8 +244,8 @@ def install(self, spec, prefix): print('========= Step-40 Trilinos ==========') print('=====================================') # change Linear Algebra to Trilinos - # The below filter_file should be different for versions - # before and after 8.4.0 + # The below filter_file should be different for versions + # before and after 8.4.0 if spec.satisfies('@8.4.0:'): filter_file(r'(\/\/ #define FORCE_USE_OF_TRILINOS.*)', ('#define FORCE_USE_OF_TRILINOS'), 'step-40.cc') @@ -256,8 +256,8 @@ def install(self, spec, prefix): make('release') make('run', parallel=False) - # the rest of the tests on step 40 only works for - # dealii version 8.4.0 and after + # the rest of the tests on step 40 only works for + # dealii version 8.4.0 and after if spec.satisfies('@8.4.0:'): print('=====================================') print('=== Step-40 Trilinos SuperluDist ====') From 7bdf63a0fa87cce6f9237ffae4a8903311327d54 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 31 May 2016 03:01:05 -0700 Subject: [PATCH 289/481] Fix some bugs in architecture. --- lib/spack/spack/architecture.py | 14 ++++++-------- lib/spack/spack/operating_systems/linux_distro.py | 13 ++++++++++--- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index a0ef4f14da..88cebc0943 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -487,25 +487,23 @@ def arch_from_dict(d): @memoized def all_platforms(): - modules = [] - + classes = [] mod_path = spack.platform_path - mod_string = "spack.platformss" + parent_module = "spack.platforms" for name in list_modules(mod_path): - mod_name = mod_string + name - path = join_path(mod_path, name) + ".py" - mod = imp.load_source(mod_name, path) + mod_name = '%s.%s' % (parent_module, name) class_name = mod_to_class(name) + mod = __import__(mod_name, fromlist=[class_name]) if not hasattr(mod, class_name): tty.die('No class %s defined in %s' % (class_name, mod_name)) cls = getattr(mod, class_name) if not inspect.isclass(cls): tty.die('%s.%s is not a class' % (mod_name, class_name)) - modules.append(cls) + classes.append(cls) - return modules + return classes @memoized def sys_type(): diff --git a/lib/spack/spack/operating_systems/linux_distro.py b/lib/spack/spack/operating_systems/linux_distro.py index d0f24d842f..2e3c72719b 100644 --- a/lib/spack/spack/operating_systems/linux_distro.py +++ b/lib/spack/spack/operating_systems/linux_distro.py @@ -1,3 +1,4 @@ +import re import platform as py_platform from spack.architecture import OperatingSystem @@ -9,7 +10,13 @@ class LinuxDistro(OperatingSystem): platform.dist() """ def __init__(self): - name = py_platform.dist()[0] - version = py_platform.dist()[1].split(".")[0] # Grabs major version from tuple + distname, version, _ = py_platform.linux_distribution( + full_distribution_name=False) - super(LinuxDistro, self).__init__(name, version) + # Grabs major version from tuple on redhat; on other platforms + # grab the first legal identifier in the version field. On + # debian you get things like 'wheezy/sid'; sid means unstable. + # We just record 'wheezy' and don't get quite so detailed. + version = re.split(r'[^\w-]', version)[0] + + super(LinuxDistro, self).__init__(distname, version) From 564e7f5e5bba55d3f4f96916c054d3c3d01a35cf Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 31 May 2016 09:32:24 -0500 Subject: [PATCH 290/481] Don't build in parallel --- var/spack/repos/builtin/packages/hpl/package.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/var/spack/repos/builtin/packages/hpl/package.py b/var/spack/repos/builtin/packages/hpl/package.py index 3d6e996323..efd5c8bb1d 100644 --- a/var/spack/repos/builtin/packages/hpl/package.py +++ b/var/spack/repos/builtin/packages/hpl/package.py @@ -43,6 +43,8 @@ class Hpl(Package): depends_on('mpi@1.1:') depends_on('blas') + parallel = False + def configure(self, spec, arch): # List of configuration options # Order is important From c03c689cb225a3191a16d65d57ad69a70914f238 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Tue, 31 May 2016 12:18:17 -0400 Subject: [PATCH 291/481] Update HDF5 to version 1.10.0-patch1 --- var/spack/repos/builtin/packages/hdf5/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/hdf5/package.py b/var/spack/repos/builtin/packages/hdf5/package.py index 21137ef356..e46f432be5 100644 --- a/var/spack/repos/builtin/packages/hdf5/package.py +++ b/var/spack/repos/builtin/packages/hdf5/package.py @@ -38,6 +38,7 @@ class Hdf5(Package): list_url = "http://www.hdfgroup.org/ftp/HDF5/releases" list_depth = 3 + version('1.10.0-patch1', '9180ff0ef8dc2ef3f61bd37a7404f295') version('1.10.0', 'bdc935337ee8282579cd6bc4270ad199') version('1.8.16', 'b8ed9a36ae142317f88b0c7ef4b9c618', preferred=True) version('1.8.15', '03cccb5b33dbe975fdcd8ae9dc021f24') From 817f43f92766bb37877888043eba6cecd9491f2e Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Tue, 31 May 2016 13:39:25 -0700 Subject: [PATCH 292/481] Fixed config format to allow default compiler flags --- lib/spack/spack/config.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index ec37bd290c..88544aa7bb 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -167,6 +167,18 @@ {'type' : 'null' }]}, 'fc': { 'anyOf': [ {'type' : 'string' }, {'type' : 'null' }]}, + 'fflags': { 'anyOf': [ {'type' : 'string' }, + {'type' : 'null' }]}, + 'cppflags': { 'anyOf': [ {'type' : 'string' }, + {'type' : 'null' }]}, + 'cflags': { 'anyOf': [ {'type' : 'string' }, + {'type' : 'null' }]}, + 'cxxflags': { 'anyOf': [ {'type' : 'string' }, + {'type' : 'null' }]}, + 'ldflags': { 'anyOf': [ {'type' : 'string' }, + {'type' : 'null' }]}, + 'ldlibs': { 'anyOf': [ {'type' : 'string' }, + {'type' : 'null' }]}, },},},},},},},}, 'mirrors': { From cb673765c27049c965e76935e193f86660ffaf02 Mon Sep 17 00:00:00 2001 From: Nicolas Richart Date: Wed, 1 Jun 2016 03:02:48 +0200 Subject: [PATCH 293/481] Adding a version to tetgen --- var/spack/repos/builtin/packages/tetgen/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/tetgen/package.py b/var/spack/repos/builtin/packages/tetgen/package.py index 5e87ed7fba..1c536197ce 100644 --- a/var/spack/repos/builtin/packages/tetgen/package.py +++ b/var/spack/repos/builtin/packages/tetgen/package.py @@ -34,6 +34,7 @@ class Tetgen(Package): url = "http://www.tetgen.org/files/tetgen1.4.3.tar.gz" version('1.4.3', 'd6a4bcdde2ac804f7ec66c29dcb63c18') + version('1.5.0', '3b9fd9cdec121e52527b0308f7aad5c1', url='http://www.tetgen.org/1.5/src/tetgen1.5.0.tar.gz') # TODO: Make this a build dependency once build dependencies are supported # (see: https://github.com/LLNL/spack/pull/378). From 58052e6632a91a5d4549ba811ae0405942ccc0b3 Mon Sep 17 00:00:00 2001 From: Nicolas Richart Date: Tue, 31 May 2016 22:13:11 +0200 Subject: [PATCH 294/481] Adding package fenics --- .../repos/builtin/packages/fenics/package.py | 175 ++++++++++++++++++ .../repos/builtin/packages/py-ply/package.py | 38 ++++ 2 files changed, 213 insertions(+) create mode 100644 var/spack/repos/builtin/packages/fenics/package.py create mode 100644 var/spack/repos/builtin/packages/py-ply/package.py diff --git a/var/spack/repos/builtin/packages/fenics/package.py b/var/spack/repos/builtin/packages/fenics/package.py new file mode 100644 index 0000000000..72282994f5 --- /dev/null +++ b/var/spack/repos/builtin/packages/fenics/package.py @@ -0,0 +1,175 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class Fenics(Package): + """FEniCS is organized as a collection of interoperable components + that together form the FEniCS Project. These components include + the problem-solving environment DOLFIN, the form compiler FFC, the + finite element tabulator FIAT, the just-in-time compiler Instant, + the code generation interface UFC, the form language UFL and a + range of additional components.""" + + homepage = "http://fenicsproject.org/" + url = "https://bitbucket.org/fenics-project/dolfin/downloads/dolfin-1.6.0.tar.gz" + + base_url = "https://bitbucket.org/fenics-project/{pkg}/downloads/{pkg}-{version}.tar.gz" # NOQA: ignore E501 + + variant('hdf5', default=True, description='Compile with HDF5') + variant('parmetis', default=True, description='Compile with ParMETIS') + variant('scotch', default=True, description='Compile with Scotch') + variant('petsc', default=True, description='Compile with PETSc') + variant('slepc', default=True, description='Compile with SLEPc') + variant('trilinos', default=True, description='Compile with Trilinos') + variant('suite-sparse', default=True, description='Compile with SuiteSparse solvers') + variant('vtk', default=False, description='Compile with VTK') + variant('qt', default=False, description='Compile with QT') + variant('mpi', default=True, description='Enables the distributed memory support') + variant('openmp', default=True, description='Enables the shared memory support') + variant('shared', default=True, description='Enables the build of shared libraries') + variant('debug', default=False, description='Builds a debug version of the libraries') + + # not part of spack list for now + # variant('petsc4py', default=True, description='Uses PETSc4py') + # variant('slepc4py', default=True, description='Uses SLEPc4py') + # variant('pastix', default=True, description='Compile with Pastix') + + extends('python') + + depends_on('py-numpy') + depends_on('py-ply') + depends_on('py-six') + depends_on('py-sphinx@1.0.1:', when='+doc') + depends_on('eigen@3.2.0:') + depends_on('boost') + depends_on('mpi', when='+mpi') + depends_on('hdf5', when='+hdf5') + depends_on('parmetis@4.0.2:^metis+real64', when='+parmetis') + depends_on('scotch~metis', when='+scotch~mpi') + depends_on('scotch+mpi~metis', when='+scotch+mpi') + depends_on('petsc@3.4:', when='+petsc') + depends_on('slepc@3.4:', when='+slepc') + depends_on('trilinos', when='+trilinos') + depends_on('vtk', when='+vtk') + depends_on('suite-sparse', when='+suite-sparse') + depends_on('qt', when='+qt') + + # This are the build dependencies + depends_on('py-setuptools') + depends_on('cmake@2.8.12:') + depends_on('swig@3.0.3:') + + releases = [ + { + 'version': '1.6.0', + 'md5': '35cb4baf7ab4152a40fb7310b34d5800', + 'resources': { + 'ffc': '358faa3e9da62a1b1a717070217b793e', + 'fiat': 'f4509d05c911fd93cea8d288a78a6c6f', + 'instant': '5f2522eb032a5bebbad6597b6fe0732a', + 'ufl': 'c40c5f04eaa847377ab2323122284016', + } + }, + { + 'version': '1.5.0', + 'md5': '9b589a3534299a5e6d22c13c5eb30bb8', + 'resources': { + 'ffc': '343f6d30e7e77d329a400fd8e73e0b63', + 'fiat': 'da3fa4dd8177bb251e7f68ec9c7cf6c5', + 'instant': 'b744023ded27ee9df4a8d8c6698c0d58', + 'ufl': '130d7829cf5a4bd5b52bf6d0955116fd', + } + }, + ] + + for release in releases: + version(release['version'], release['md5'], url=base_url.format(pkg='dolfin', version=release['version'])) + for name, md5 in release['resources'].items(): + resource(name=name, + url=base_url.format(pkg=name, **release), + md5=md5, + destination='depends', + when='@{version}'.format(**release), + placement=name) + + def cmake_is_on(self, option): + return 'ON' if option in self.spec else 'OFF' + + def install(self, spec, prefix): + for package in ['ufl', 'ffc', 'fiat', 'instant']: + with working_dir(join_path('depends', package)): + python('setup.py', 'install', '--prefix=%s' % prefix) + + cmake_args.extend([ + '-DCMAKE_BUILD_TYPE:STRING={0}'.format( + 'Debug' if '+debug' in spec else 'RelWithDebInfo'), + '-DBUILD_SHARED_LIBS:BOOL={0}'.format( + self.cmake_is_on('+shared')), + '-DDOLFIN_SKIP_BUILD_TESTS:BOOL=ON', + '-DDOLFIN_ENABLE_OPENMP:BOOL={0}'.format( + self.cmake_is_on('+openmp')), + '-DDOLFIN_ENABLE_CHOLMOD:BOOL={0}'.format( + self.cmake_is_on('suite-sparse')), + '-DDOLFIN_ENABLE_HDF5:BOOL={0}'.format( + self.cmake_is_on('hdf5')), + '-DDOLFIN_ENABLE_MPI:BOOL={0}'.format( + self.cmake_is_on('mpi')), + '-DDOLFIN_ENABLE_PARMETIS:BOOL={0}'.format( + self.cmake_is_on('parmetis')), + '-DDOLFIN_ENABLE_PASTIX:BOOL={0}'.format( + self.cmake_is_on('pastix')), + '-DDOLFIN_ENABLE_PETSC:BOOL={0}'.format( + self.cmake_is_on('petsc')), + '-DDOLFIN_ENABLE_PETSC4PY:BOOL={0}'.format( + self.cmake_is_on('py-petsc4py')), + '-DDOLFIN_ENABLE_PYTHON:BOOL={0}'.format( + self.cmake_is_on('python')), + '-DDOLFIN_ENABLE_QT:BOOL={0}'.format( + self.cmake_is_on('qt')), + '-DDOLFIN_ENABLE_SCOTCH:BOOL={0}'.format( + self.cmake_is_on('scotch')), + '-DDOLFIN_ENABLE_SLEPC:BOOL={0}'.format( + self.cmake_is_on('slepc')), + '-DDOLFIN_ENABLE_SLEPC4PY:BOOL={0}'.format( + self.cmake_is_on('py-slepc4py')), + '-DDOLFIN_ENABLE_SPHINX:BOOL={0}'.format( + self.cmake_is_on('py-sphinx')), + '-DDOLFIN_ENABLE_TRILINOS:BOOL={0}'.format( + self.cmake_is_on('trilinos')), + '-DDOLFIN_ENABLE_UMFPACK:BOOL={0}'.format( + self.cmake_is_on('suite-sparse')), + '-DDOLFIN_ENABLE_VTK:BOOL={0}'.format( + self.cmake_is_on('vtk')), + '-DDOLFIN_ENABLE_ZLIB:BOOL={0}'.format( + self.cmake_is_on('zlib')), + ]) + cmake_args.extend(std_cmake_args) + + with working_dir('build', create=True): + cmake('..', *cmake_args) + + make() + make('install') diff --git a/var/spack/repos/builtin/packages/py-ply/package.py b/var/spack/repos/builtin/packages/py-ply/package.py new file mode 100644 index 0000000000..47cd3b5dc8 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-ply/package.py @@ -0,0 +1,38 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class PyPly(Package): + """PLY is nothing more than a straightforward lex/yacc implementation.""" + homepage = "http://www.dabeaz.com/ply" + url = "http://www.dabeaz.com/ply/ply-3.8.tar.gz" + + version('3.8', '94726411496c52c87c2b9429b12d5c50') + + extends('python') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) From 97bad2c1ac33139f35094727a269001e4815c227 Mon Sep 17 00:00:00 2001 From: Nicolas Richart Date: Wed, 1 Jun 2016 03:25:37 +0200 Subject: [PATCH 295/481] Corrections for flake8 --- var/spack/repos/builtin/packages/tetgen/package.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/var/spack/repos/builtin/packages/tetgen/package.py b/var/spack/repos/builtin/packages/tetgen/package.py index 1c536197ce..c301a5b4e5 100644 --- a/var/spack/repos/builtin/packages/tetgen/package.py +++ b/var/spack/repos/builtin/packages/tetgen/package.py @@ -24,11 +24,13 @@ ############################################################################## from spack import * + class Tetgen(Package): - """TetGen is a program and library that can be used to generate tetrahedral - meshes for given 3D polyhedral domains. TetGen generates exact constrained - Delaunay tetrahedralizations, boundary conforming Delaunay meshes, and - Voronoi paritions.""" + """TetGen is a program and library that can be used to generate + tetrahedral meshes for given 3D polyhedral domains. TetGen + generates exact constrained Delaunay tetrahedralizations, + boundary conforming Delaunay meshes, and Voronoi paritions. + """ homepage = "http://www.tetgen.org" url = "http://www.tetgen.org/files/tetgen1.4.3.tar.gz" From d802144f39366a2cbd67554de5a099ed8ce80598 Mon Sep 17 00:00:00 2001 From: Nicolas Richart Date: Wed, 1 Jun 2016 03:31:29 +0200 Subject: [PATCH 296/481] Introduced a bug by correcting flake8 warnings --- var/spack/repos/builtin/packages/fenics/package.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/fenics/package.py b/var/spack/repos/builtin/packages/fenics/package.py index 72282994f5..0845237656 100644 --- a/var/spack/repos/builtin/packages/fenics/package.py +++ b/var/spack/repos/builtin/packages/fenics/package.py @@ -123,7 +123,7 @@ def install(self, spec, prefix): with working_dir(join_path('depends', package)): python('setup.py', 'install', '--prefix=%s' % prefix) - cmake_args.extend([ + cmake_args = [ '-DCMAKE_BUILD_TYPE:STRING={0}'.format( 'Debug' if '+debug' in spec else 'RelWithDebInfo'), '-DBUILD_SHARED_LIBS:BOOL={0}'.format( @@ -165,7 +165,8 @@ def install(self, spec, prefix): self.cmake_is_on('vtk')), '-DDOLFIN_ENABLE_ZLIB:BOOL={0}'.format( self.cmake_is_on('zlib')), - ]) + ] + cmake_args.extend(std_cmake_args) with working_dir('build', create=True): From 0451c8e55f35c58adef120a986e8782abb3155eb Mon Sep 17 00:00:00 2001 From: Nicolas Richart Date: Wed, 1 Jun 2016 15:58:00 +0200 Subject: [PATCH 297/481] Addapting the mvapich2@2.1: configure options for slurm --- .../builtin/packages/mvapich2/package.py | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/var/spack/repos/builtin/packages/mvapich2/package.py b/var/spack/repos/builtin/packages/mvapich2/package.py index f4997bdfa1..e871d0cd2b 100644 --- a/var/spack/repos/builtin/packages/mvapich2/package.py +++ b/var/spack/repos/builtin/packages/mvapich2/package.py @@ -124,10 +124,19 @@ def set_process_manager(self, spec, configure_args): raise RuntimeError(" %s : 'slurm' cannot be activated together with other process managers" % self.name) process_manager_options = [] + # See: http://slurm.schedmd.com/mpi_guide.html#mvapich2 if self.enabled(Mvapich2.SLURM) in spec: - process_manager_options = [ - "--with-pm=slurm" - ] + if self.version > Version('2.0'): + process_manager_options = [ + "--with-pmi=pmi2", + "--with-pm=slurm" + ] + else: + process_manager_options = [ + "--with-pmi=slurm", + "--with-pm=no" + ] + elif has_slurm_incompatible_variants: pms = [] # The variant name is equal to the process manager name in the configuration options @@ -164,6 +173,11 @@ def set_network_type(self, spec, configure_args): configure_args.extend(network_options) + def setup_environment(self, spack_env, run_env): + if self.enabled(Mvapich2.SLURM) in self.spec and \ + self.version > Version('2.0'): + run_env.set('SLURM_MPI_TYPE', 'pmi2') + def setup_dependent_environment(self, spack_env, run_env, extension_spec): spack_env.set('MPICH_CC', spack_cc) spack_env.set('MPICH_CXX', spack_cxx) @@ -176,7 +190,7 @@ def setup_dependent_package(self, module, dep_spec): self.spec.mpicxx = join_path(self.prefix.bin, 'mpicxx') self.spec.mpifc = join_path(self.prefix.bin, 'mpif90') self.spec.mpif77 = join_path(self.prefix.bin, 'mpif77') - + def install(self, spec, prefix): # we'll set different configure flags depending on our environment configure_args = [ From 3ab8468e732a9ca68e4a846e1ae6496757d565f2 Mon Sep 17 00:00:00 2001 From: Nicolas Richart Date: Wed, 1 Jun 2016 16:43:08 +0200 Subject: [PATCH 298/481] Correcting the flake8 warnings --- .../builtin/packages/mvapich2/package.py | 72 ++++++++++++------- 1 file changed, 47 insertions(+), 25 deletions(-) diff --git a/var/spack/repos/builtin/packages/mvapich2/package.py b/var/spack/repos/builtin/packages/mvapich2/package.py index e871d0cd2b..d1944023d1 100644 --- a/var/spack/repos/builtin/packages/mvapich2/package.py +++ b/var/spack/repos/builtin/packages/mvapich2/package.py @@ -25,6 +25,7 @@ from spack import * import os + class Mvapich2(Package): """MVAPICH2 is an MPI implementation for Infiniband networks.""" homepage = "http://mvapich.cse.ohio-state.edu/" @@ -43,8 +44,9 @@ class Mvapich2(Package): variant('debug', default=False, description='Enables debug information and error messages at run-time') ########## - # TODO : Process managers should be grouped into the same variant, as soon as variant capabilities will be extended - # See https://groups.google.com/forum/#!topic/spack/F8-f8B4_0so + # TODO : Process managers should be grouped into the same variant, + # as soon as variant capabilities will be extended See + # https://groups.google.com/forum/#!topic/spack/F8-f8B4_0so SLURM = 'slurm' HYDRA = 'hydra' GFORKER = 'gforker' @@ -57,7 +59,8 @@ class Mvapich2(Package): ########## ########## - # TODO : Network types should be grouped into the same variant, as soon as variant capabilities will be extended + # TODO : Network types should be grouped into the same variant, as + # soon as variant capabilities will be extended PSM = 'psm' SOCK = 'sock' NEMESISIBTCP = 'nemesisibtcp' @@ -84,8 +87,8 @@ def url_for_version(self, version): @staticmethod def enabled(x): - """ - Given a variant name returns the string that means the variant is enabled + """Given a variant name returns the string that means the variant is + enabled :param x: variant name :return: @@ -93,8 +96,8 @@ def enabled(x): return '+' + x def set_build_type(self, spec, configure_args): - """ - Appends to configure_args the flags that depends only on the build type (i.e. release or debug) + """Appends to configure_args the flags that depends only on the build + type (i.e. release or debug) :param spec: spec :param configure_args: list of current configure arguments @@ -104,7 +107,8 @@ def set_build_type(self, spec, configure_args): "--disable-fast", "--enable-error-checking=runtime", "--enable-error-messages=all", - "--enable-g=dbg", "--enable-debuginfo" # Permits debugging with TotalView + # Permits debugging with TotalView + "--enable-g=dbg", "--enable-debuginfo" ] else: build_type_options = ["--enable-fast=all"] @@ -112,16 +116,22 @@ def set_build_type(self, spec, configure_args): configure_args.extend(build_type_options) def set_process_manager(self, spec, configure_args): - """ - Appends to configure_args the flags that will enable the appropriate process managers + """Appends to configure_args the flags that will enable the + appropriate process managers :param spec: spec :param configure_args: list of current configure arguments """ - # Check that slurm variant is not activated together with other pm variants - has_slurm_incompatible_variants = any(self.enabled(x) in spec for x in Mvapich2.SLURM_INCOMPATIBLE_PMS) - if self.enabled(Mvapich2.SLURM) in spec and has_slurm_incompatible_variants: - raise RuntimeError(" %s : 'slurm' cannot be activated together with other process managers" % self.name) + # Check that slurm variant is not activated together with + # other pm variants + has_slurm_incompatible_variants = \ + any(self.enabled(x) in spec + for x in Mvapich2.SLURM_INCOMPATIBLE_PMS) + + if self.enabled(Mvapich2.SLURM) in spec and \ + has_slurm_incompatible_variants: + raise RuntimeError(" %s : 'slurm' cannot be activated \ + together with other process managers" % self.name) process_manager_options = [] # See: http://slurm.schedmd.com/mpi_guide.html#mvapich2 @@ -139,7 +149,8 @@ def set_process_manager(self, spec, configure_args): elif has_slurm_incompatible_variants: pms = [] - # The variant name is equal to the process manager name in the configuration options + # The variant name is equal to the process manager name in + # the configuration options for x in Mvapich2.SLURM_INCOMPATIBLE_PMS: if self.enabled(x) in spec: pms.append(x) @@ -155,7 +166,9 @@ def set_network_type(self, spec, configure_args): if self.enabled(x) in spec: count += 1 if count > 1: - raise RuntimeError('network variants are mutually exclusive (only one can be selected at a time)') + raise RuntimeError('network variants are mutually exclusive \ + (only one can be selected at a time)') + network_options = [] # From here on I can suppose that only one variant has been selected if self.enabled(Mvapich2.PSM) in spec: @@ -177,7 +190,7 @@ def setup_environment(self, spack_env, run_env): if self.enabled(Mvapich2.SLURM) in self.spec and \ self.version > Version('2.0'): run_env.set('SLURM_MPI_TYPE', 'pmi2') - + def setup_dependent_environment(self, spack_env, run_env, extension_spec): spack_env.set('MPICH_CC', spack_cc) spack_env.set('MPICH_CXX', spack_cxx) @@ -190,9 +203,10 @@ def setup_dependent_package(self, module, dep_spec): self.spec.mpicxx = join_path(self.prefix.bin, 'mpicxx') self.spec.mpifc = join_path(self.prefix.bin, 'mpif90') self.spec.mpif77 = join_path(self.prefix.bin, 'mpif77') - + def install(self, spec, prefix): - # we'll set different configure flags depending on our environment + # we'll set different configure flags depending on our + # environment configure_args = [ "--prefix=%s" % prefix, "--enable-shared", @@ -222,7 +236,6 @@ def install(self, spec, prefix): self.filter_compilers() - def filter_compilers(self): """Run after install to make the MPI compilers use the compilers that Spack built the package with. @@ -242,8 +255,17 @@ def filter_compilers(self): spack_f77 = os.environ['F77'] spack_fc = os.environ['FC'] - kwargs = { 'ignore_absent' : True, 'backup' : False, 'string' : True } - filter_file('CC="%s"' % spack_cc , 'CC="%s"' % self.compiler.cc, mpicc, **kwargs) - filter_file('CXX="%s"'% spack_cxx, 'CXX="%s"' % self.compiler.cxx, mpicxx, **kwargs) - filter_file('F77="%s"'% spack_f77, 'F77="%s"' % self.compiler.f77, mpif77, **kwargs) - filter_file('FC="%s"' % spack_fc , 'FC="%s"' % self.compiler.fc, mpif90, **kwargs) + kwargs = { + 'ignore_absent': True, + 'backup': False, + 'string': True + } + + filter_file('CC="%s"' % spack_cc, + 'CC="%s"' % self.compiler.cc, mpicc, **kwargs) + filter_file('CXX="%s"' % spack_cxx, + 'CXX="%s"' % self.compiler.cxx, mpicxx, **kwargs) + filter_file('F77="%s"' % spack_f77, + 'F77="%s"' % self.compiler.f77, mpif77, **kwargs) + filter_file('FC="%s"' % spack_fc, + 'FC="%s"' % self.compiler.fc, mpif90, **kwargs) From 9d04dccb79c606b3d3ca033b5365d31d9a1b0442 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 2 Jun 2016 01:20:40 -0700 Subject: [PATCH 299/481] Update version to correspond to master. --- lib/spack/spack/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index c7a155597f..8c6e0ba527 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -105,7 +105,7 @@ # Version information from spack.version import Version -spack_version = Version("0.9") +spack_version = Version("0.9.1") # # Executables used by Spack From 24d160e93ed80052ad30f99ce36b34de2872a888 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 3 Jun 2016 15:57:54 -0700 Subject: [PATCH 300/481] Changed how architecture looks like in spec.yaml. Now it's only three strings, platform, os, target in spec.yaml. Also did some flake8 formatting changes --- lib/spack/spack/architecture.py | 210 ++++++++++++++++---------------- 1 file changed, 102 insertions(+), 108 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index a0ef4f14da..ee0d2173c7 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -29,8 +29,9 @@ On a multiple architecture machine, the architecture spec field can be set to build a package against any target and operating system that is present on the -platform. On Cray platforms or any other architecture that has different front and -back end environments, the operating system will determine the method of compiler +platform. On Cray platforms or any other architecture that has different front +and back end environments, the operating system will determine the method of +compiler detection. There are two different types of compiler detection: @@ -40,44 +41,42 @@ Depending on which operating system is specified, the compiler will be detected using one of those methods. -For platforms such as linux and darwin, the operating system is autodetected and -the target is set to be x86_64. +For platforms such as linux and darwin, the operating system is autodetected +and the target is set to be x86_64. The command line syntax for specifying an architecture is as follows: target= os= If the user wishes to use the defaults, either target or os can be left out of -the command line and Spack will concretize using the default. These defaults are -set in the 'platforms/' directory which contains the different subclasses for -platforms. If the machine has multiple architectures, the user can +the command line and Spack will concretize using the default. These defaults +are set in the 'platforms/' directory which contains the different subclasses +for platforms. If the machine has multiple architectures, the user can also enter front-end, or fe or back-end or be. These settings will concretize to their respective front-end and back-end targets and operating systems. Additional platforms can be added by creating a subclass of Platform and adding it inside the platform directory. Platforms are an abstract class that are extended by subclasses. If the user -wants to add a new type of platform (such as cray_xe), they can create a subclass -and set all the class attributes such as priority, front_target ,back_target, -front_os, back_os. Platforms also contain a priority class attribute. A lower -number signifies higher priority. These numbers are arbitrarily set and can be -changed though often there isn't much need unless a new platform is added and -the user wants that to be detected first. +wants to add a new type of platform (such as cray_xe), they can create a +subclass and set all the class attributes such as priority, front_target, +back_target, front_os, back_os. Platforms also contain a priority class +attribute. A lower number signifies higher priority. These numbers are +arbitrarily set and can be changed though often there isn't much need unless a +new platform is added and the user wants that to be detected first. -Targets are created inside the platform subclasses. Most architecture (like linux, -and darwin) will have only one target (x86_64) but in the case of Cray machines, -there is both a frontend and backend processor. The user can specify which targets -are present on front-end and back-end architecture +Targets are created inside the platform subclasses. Most architecture +(like linux, and darwin) will have only one target (x86_64) but in the case of +Cray machines, there is both a frontend and backend processor. The user can +specify which targets are present on front-end and back-end architecture Depending on the platform, operating systems are either auto-detected or are set. The user can set the front-end and back-end operating setting by the class -attributes front_os and back_os. The operating system as described earlier, will -be responsible for compiler detection. +attributes front_os and back_os. The operating system as described earlier, +will be responsible for compiler detection. """ import os -from collections import namedtuple import imp -import platform as py_platform import inspect from llnl.util.lang import memoized, list_modules, key_ordering @@ -91,6 +90,7 @@ from spack.util.multiproc import parmap import spack.error as serr + class InvalidSysTypeError(serr.SpackError): def __init__(self, sys_type): super(InvalidSysTypeError, self).__init__( @@ -128,12 +128,6 @@ def __repr__(self): def __str__(self): return self.name - def to_dict(self): - d = {} - d['name'] = self.name - d['module_name'] = self.module_name - - return d @key_ordering class Platform(object): @@ -142,7 +136,7 @@ class Platform(object): is returned """ - priority = None # Subclass needs to set this number. This controls order in which platform is detected. + priority = None # Subclass sets number. Controls detection order front_end = None back_end = None default = None # The default back end target. On cray ivybridge @@ -156,19 +150,6 @@ def __init__(self, name): self.operating_sys = {} self.name = name - def to_dict(self): - n = {} - n['targets'] = dict((name, target.to_dict()) for (name, target) in self.targets.items()) - n['operating_systems'] = dict((name, os.to_dict()) for (name, os) in self.operating_sys.items()) - n['priority'] = self.priority - n['default_front_end_target'] = self.front_end - n['default_back_end_target'] = self.back_end - n['default_target'] = self.default - n['default_front_end_os'] = self.front_os - n['default_back_end_os'] = self.back_os - n['default_os'] = self.default_os - return {self.name: n} - def add_target(self, name, target): """Used by the platform specific subclass to list available targets. Raises an error if the platform specifies a name @@ -215,7 +196,6 @@ def operating_system(self, name): return self.operating_sys.get(name, None) - @classmethod def detect(self): """ Subclass is responsible for implementing this method. @@ -232,8 +212,10 @@ def __str__(self): return self.name def _cmp_key(self): - t_keys = ''.join(str(t._cmp_key()) for t in sorted(self.targets.values())) - o_keys = ''.join(str(o._cmp_key()) for o in sorted(self.operating_sys.values())) + t_keys = ''.join(str(t._cmp_key()) for t in + sorted(self.targets.values())) + o_keys = ''.join(str(o._cmp_key()) for o in + sorted(self.operating_sys.values())) return (self.name, self.default, self.front_end, @@ -244,6 +226,7 @@ def _cmp_key(self): t_keys, o_keys) + @key_ordering class OperatingSystem(object): """ Operating System will be like a class similar to platform extended @@ -265,7 +248,6 @@ def __repr__(self): def _cmp_key(self): return (self.name, self.version) - def find_compilers(self, *paths): """ Return a list of compilers found in the suppied paths. @@ -293,11 +275,13 @@ def find_compilers(self, *paths): # compiler. We can spawn a bunch of parallel searches to reduce # the overhead of spelunking all these directories. types = spack.compilers.all_compiler_types() - compiler_lists = parmap(lambda cmp_cls: self.find_compiler(cmp_cls, *filtered_path), types) + compiler_lists = parmap(lambda cmp_cls: + self.find_compiler(cmp_cls, *filtered_path), + types) # ensure all the version calls we made are cached in the parent # process, as well. This speeds up Spack a lot. - clist = reduce(lambda x,y: x+y, compiler_lists) + clist = reduce(lambda x, y: x+y, compiler_lists) return clist def find_compiler(self, cmp_cls, *path): @@ -338,7 +322,7 @@ def find_compiler(self, cmp_cls, *path): # prefer the one with more compilers. prev_paths = [prev.cc, prev.cxx, prev.f77, prev.fc] - newcount = len([p for p in paths if p is not None]) + newcount = len([p for p in paths if p is not None]) prevcount = len([p for p in prev_paths if p is not None]) # Don't add if it's not an improvement over prev compiler. @@ -349,14 +333,7 @@ def find_compiler(self, cmp_cls, *path): return list(compilers.values()) - def to_dict(self): - d = {} - d['name'] = self.name - d['version'] = self.version - return d - -#NOTE: Key error caused because Architecture has no comparison method @key_ordering class Arch(object): "Architecture is now a class to help with setting attributes" @@ -376,10 +353,11 @@ def __init__(self, platform=None, platform_os=None, target=None): @property def concrete(self): - return all( (self.platform is not None, isinstance(self.platform, Platform), - self.platform_os is not None, isinstance(self.platform_os, OperatingSystem), - self.target is not None, isinstance(self.target, Target) ) ) - + return all((self.platform is not None, + isinstance(self.platform, Platform), + self.platform_os is not None, + isinstance(self.platform_os, OperatingSystem), + self.target is not None, isinstance(self.target, Target))) def __str__(self): if self.platform or self.platform_os or self.target: @@ -388,71 +366,75 @@ def __str__(self): else: os_name = str(self.platform_os) - return (str(self.platform) +"-"+ + return (str(self.platform) + "-" + os_name + "-" + str(self.target)) else: return '' def _cmp_key(self): - platform = self.platform.name if isinstance(self.platform, Platform) else self.platform - os = self.platform_os.name if isinstance(self.platform_os, OperatingSystem) else self.platform_os - target = self.target.name if isinstance(self.target, Target) else self.target - return (platform, os, target) + if isinstance(self.platform, Platform): + platform = self.platform.name + else: + platform = self.platform + if isinstance(self.platform_os, OperatingSystem): + platform_os = self.platform_os.name + else: + platform_os = self.platform_os + if isinstance(self.target, Target): + target = self.target.name + else: + target = self.target + print (platform, platform_os, target) + return (platform, platform_os, target) def to_dict(self): d = {} - platform = self.platform - platform_os = self.platform_os - target = self.target - - d['platform'] = self.platform.to_dict() if self.platform else None - d['platform_os'] = self.platform_os.to_dict() if self.platform_os else None - d['target'] = self.target.to_dict() if self.target else None + d['platform'] = str(self.platform) if self.platform else None + d['platform_os'] = str(self.platform_os) if self.platform_os else None + d['target'] = str(self.target) if self.target else None return d -def _target_from_dict(target_dict): +def _target_from_dict(target_name, platform=None): """ Creates new instance of target and assigns all the attributes of that target from the dictionary """ - target = Target.__new__(Target) - target.name = target_dict['name'] - target.module_name = target_dict['module_name'] - if 'platform_name' in target_dict: - target.platform_name = target_dict['platform_name'] - return target + if not platform: + platform = sys_type() + return platform.target(target_name) -def _operating_system_from_dict(os_dict): + +def _operating_system_from_dict(os_name, platform=None): """ uses platform's operating system method to grab the constructed operating systems that are valid on the platform. """ -# NOTE: Might need a better way to create operating system objects - operating_system = OperatingSystem.__new__(OperatingSystem) - operating_system.name = os_dict['name'] - operating_system.version = os_dict['version'] - return operating_system + if not platform: + platform = sys_type() + if isinstance(os_name, spack.util.spack_yaml.syaml_dict): + name = os_name['name'] + version = os_name['version'] + return platform.operating_system(name+version) + else: + return platform.operating_system(os_name) -def _platform_from_dict(platform_dict): + +def _platform_from_dict(platform_name): """ Constructs a platform from a dictionary. """ - platform = Platform.__new__(Platform) - name, p_dict = platform_dict.items()[0] - platform.name = name - platform.targets = {} - for name, t_dict in p_dict['targets'].items(): - platform.add_target(name, _target_from_dict(t_dict)) - platform.operating_sys = {} - for name, o_dict in p_dict['operating_systems'].items(): - platform.add_operating_system(name, _operating_system_from_dict(o_dict)) - platform.priority = p_dict['priority'] - platform.front_end = p_dict['default_front_end_target'] - platform.back_end = p_dict['default_back_end_target'] - platform.default = p_dict['default_target'] - platform.front_os = p_dict['default_front_end_os'] - platform.back_os = p_dict['default_back_end_os'] - platform.default_os = p_dict['default_os'] + platform_path = spack.platform_path + mod_string = "spack.platforms" + + for p in list_modules(platform_path): + if platform_name == p: + mod_name = mod_string + platform_name + path = join_path(platform_path, platform_name) + ".py" + mod = imp.load_source(mod_name, path) + platform_class = mod_to_class(platform_name) + cls = getattr(mod, platform_class) + platform = cls() + return platform + return None - return platform def arch_from_dict(d): """ Uses _platform_from_dict, _operating_system_from_dict, _target_from_dict @@ -472,19 +454,30 @@ def arch_from_dict(d): else: if d is None: return None - platform_dict = d['platform'] - os_dict = d['platform_os'] - target_dict = d['target'] + platform_name = d['platform'] + os_name = d['platform_os'] + target_name = d['target'] - arch.platform = _platform_from_dict(platform_dict) if platform_dict else None - arch.target = _target_from_dict(target_dict) if os_dict else None - arch.platform_os = _operating_system_from_dict(os_dict) if os_dict else None + if platform_name: + arch.platform = _platform_from_dict(platform_name) + else: + arch.platform = None + if target_name: + arch.target = _target_from_dict(target_name, arch.platform) + else: + arch.target = None + if os_name: + arch.platform_os = _operating_system_from_dict(os_name, + arch.platform) + else: + arch.platform_os = None arch.os_string = None arch.target_string = None return arch + @memoized def all_platforms(): modules = [] @@ -507,6 +500,7 @@ def all_platforms(): return modules + @memoized def sys_type(): """ Gather a list of all available subclasses of platforms. From 30c204a0a5ca116d2e6ab1ab177e103a5857ca4e Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 3 Jun 2016 16:16:59 -0700 Subject: [PATCH 301/481] Made a merge commit earlier with Todd fixing all_platforms, but changed how architecture looks in spec.yaml. Just three strings. Also made some flake8 formatting changes --- lib/spack/spack/architecture.py | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index e74e72d6cd..818731cbf8 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -384,7 +384,6 @@ def _cmp_key(self): target = self.target.name else: target = self.target - print (platform, platform_os, target) return (platform, platform_os, target) def to_dict(self): @@ -421,19 +420,10 @@ def _operating_system_from_dict(os_name, platform=None): def _platform_from_dict(platform_name): """ Constructs a platform from a dictionary. """ - platform_path = spack.platform_path - mod_string = "spack.platforms" - - for p in list_modules(platform_path): - if platform_name == p: - mod_name = mod_string + platform_name - path = join_path(platform_path, platform_name) + ".py" - mod = imp.load_source(mod_name, path) - platform_class = mod_to_class(platform_name) - cls = getattr(mod, platform_class) - platform = cls() - return platform - return None + platform_list = all_platforms() + for p in platform_list: + if platform_name.replace("_", "").lower() == p.__name__.lower(): + return p() def arch_from_dict(d): From f751d681771499b1da46216f46beb0a074fa73e7 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 6 Jun 2016 09:03:32 -0700 Subject: [PATCH 302/481] Passing architecture to functions to find compiler. We can use the platform name if provided to find the operating system. Also changed error to operating system instead of target since operating system is now in charge of compiler finding. --- lib/spack/spack/compilers/__init__.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index ea79f655b0..d923d77da8 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -220,10 +220,11 @@ def find(compiler_spec, scope=None): @_auto_compiler_spec -def compilers_for_spec(compiler_spec, scope=None): +def compilers_for_spec(compiler_spec, scope=None, **kwargs): """This gets all compilers that satisfy the supplied CompilerSpec. Returns an empty list if none are found. """ + platform = kwargs.get("platform", None) config = all_compilers_config(scope) def get_compilers(cspec): @@ -253,7 +254,7 @@ def get_compilers(cspec): mods = [] if 'operating_system' in items: - operating_system = spack.architecture._operating_system_from_dict( items['operating_system'] ) + operating_system = spack.architecture._operating_system_from_dict(items['operating_system'], platform) else: operating_system = None @@ -275,12 +276,13 @@ def get_compilers(cspec): @_auto_compiler_spec -def compiler_for_spec(compiler_spec, operating_system): +def compiler_for_spec(compiler_spec, arch): """Get the compiler that satisfies compiler_spec. compiler_spec must be concrete.""" + operating_system = arch.platform_os assert(compiler_spec.concrete) - compilers = [c for c in compilers_for_spec(compiler_spec) + compilers = [c for c in compilers_for_spec(compiler_spec, platform=arch.platform) if c.operating_system == operating_system] if len(compilers) < 1: raise NoCompilerForSpecError(compiler_spec, operating_system) @@ -334,7 +336,7 @@ def __init__(self): class NoCompilerForSpecError(spack.error.SpackError): def __init__(self, compiler_spec, target): - super(NoCompilerForSpecError, self).__init__("No compilers for target %s satisfy spec %s" % ( + super(NoCompilerForSpecError, self).__init__("No compilers for operating system %s satisfy spec %s" % ( target, compiler_spec)) class CompilerSpecInsufficientlySpecificError(spack.error.SpackError): From c7b48f6fef92167b55d901e47a50e3f704fd65ec Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 6 Jun 2016 09:04:24 -0700 Subject: [PATCH 303/481] Passing full arch for proper compiler spec so that we can use the platform to get the operating system --- lib/spack/spack/concretize.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 58811c3419..1f37455c77 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -280,10 +280,12 @@ def concretize_compiler(self, spec): # Only use a matching compiler if it is of the proper style # Takes advantage of the proper logic already existing in compiler_for_spec # Should think whether this can be more efficient - def _proper_compiler_style(cspec, architecture): - compilers = spack.compilers.compilers_for_spec(cspec) - return filter(lambda c: c.operating_system == - architecture.platform_os, compilers) + def _proper_compiler_style(cspec, arch): + platform = arch.platform + compilers = spack.compilers.compilers_for_spec(cspec, + platform=platform) + return filter(lambda c: c.operating_system == + arch.platform_os, compilers) #return compilers @@ -368,7 +370,7 @@ def concretize_compiler_flags(self, spec): # Include the compiler flag defaults from the config files # This ensures that spack will detect conflicts that stem from a change # in default compiler flags. - compiler = spack.compilers.compiler_for_spec(spec.compiler, spec.architecture.platform_os) + compiler = spack.compilers.compiler_for_spec(spec.compiler, spec.architecture) for flag in compiler.flags: if flag not in spec.compiler_flags: spec.compiler_flags[flag] = compiler.flags[flag] From 5b023bb0a1b2eef9d950cb660b466d2ab0e04349 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 6 Jun 2016 09:05:14 -0700 Subject: [PATCH 304/481] Passing through full arch instead of just operating sysystem so that we can use the platform to get the operating system --- lib/spack/spack/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 0edb70e4ca..1a6c289bc7 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -682,7 +682,7 @@ def compiler(self): if not self.spec.concrete: raise ValueError("Can only get a compiler for a concrete package.") return spack.compilers.compiler_for_spec(self.spec.compiler, - self.spec.architecture.platform_os) + self.spec.architecture) def url_version(self, version): """ From 864191b6ed9fbd95e549b2f4852dc92113c6a190 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 6 Jun 2016 09:06:58 -0700 Subject: [PATCH 305/481] Got rid of unnecessary global keyword --- lib/spack/spack/test/mock_packages_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index 561f319b16..9c586bd8ce 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -40,7 +40,6 @@ linux_os_version = '6' if platform.name == 'linux': - global linux_os_name, linux_os_version linux_os = platform.operating_system("default_os") linux_os_name = linux_os.name linux_os_version = linux_os.version From 835982faedc0d728356600836caf2dd4690a9c5b Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 6 Jun 2016 09:08:59 -0700 Subject: [PATCH 306/481] Changed OS name to CNL10 so that we properly find the compilers for the test platform --- lib/spack/spack/platforms/test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/platforms/test.py b/lib/spack/spack/platforms/test.py index 3a4f9d5f2d..8fa2585a7a 100644 --- a/lib/spack/spack/platforms/test.py +++ b/lib/spack/spack/platforms/test.py @@ -10,8 +10,8 @@ class Test(Platform): back_end = 'x86_64' default = 'x86_64' - back_os = 'CNL' - default_os = 'CNL' + back_os = 'CNL10' + default_os = 'CNL10' def __init__(self): super(Test, self).__init__('test') From 196737c2173a7b5748b3ad59e349611ec4f6b23c Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 6 Jun 2016 10:29:19 -0700 Subject: [PATCH 307/481] Changed to passing full architecture spec to function rather than just platform_os so we can grab the operating system from the platform getters --- lib/spack/spack/spec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 89c03a13b7..7718adb72c 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1124,7 +1124,7 @@ def concretize(self): for s in self.traverse(root=False): if s.external_module: - compiler = spack.compilers.compiler_for_spec(s.compiler, s.architecture.platform_os) + compiler = spack.compilers.compiler_for_spec(s.compiler, s.architecture) for mod in compiler.modules: load_module(mod) From 4925be0bc4b63571eb9a9cb63e65409bebcc2c66 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Tue, 7 Jun 2016 13:22:47 -0700 Subject: [PATCH 308/481] Load external modules --- lib/spack/spack/build_environment.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 4483b9d132..7c65091d49 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -384,6 +384,13 @@ def parent_class_modules(cls): return result +def load_external_modules(pkg): + """ traverse the spec list and find any specs that have external modules. + """ + for dep in list(pkg.spec.traverse()): + if dep.external_module: + load_module(dep.external_module) + def setup_package(pkg): """Execute all environment setup routines.""" spack_env = EnvironmentModifications() @@ -407,7 +414,7 @@ def setup_package(pkg): set_compiler_environment_variables(pkg, spack_env) set_build_environment_variables(pkg, spack_env) - + load_external_modules(pkg) # traverse in postorder so package can use vars from its dependencies spec = pkg.spec for dspec in pkg.spec.traverse(order='post', root=False): From 5715799d4ee1c39d11106bb78a034e6affac474c Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Tue, 7 Jun 2016 14:15:28 -0700 Subject: [PATCH 309/481] Added to_dict back into operating system --- lib/spack/spack/architecture.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 818731cbf8..1f25bc14ea 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -195,6 +195,7 @@ def operating_system(self, name): name = self.back_os return self.operating_sys.get(name, None) + @classmethod def detect(self): @@ -333,6 +334,12 @@ def find_compiler(self, cmp_cls, *path): return list(compilers.values()) + def to_dict(self): + d = {} + d['name'] = self.name + d['version'] = self.version + + return d @key_ordering class Arch(object): @@ -410,7 +417,8 @@ def _operating_system_from_dict(os_name, platform=None): """ if not platform: platform = sys_type() - if isinstance(os_name, spack.util.spack_yaml.syaml_dict): + if isinstance(os_name, spack.util.spack_yaml.syaml_dict) or \ + isinstance(os_name, dict): name = os_name['name'] version = os_name['version'] return platform.operating_system(name+version) From 25b2f7b1d75d404b4e06443969cdcb3e125123e6 Mon Sep 17 00:00:00 2001 From: Greg Lee Date: Tue, 7 Jun 2016 20:02:38 -0700 Subject: [PATCH 310/481] put older versions back + version-specific dwarfdump dir --- var/spack/repos/builtin/packages/libdwarf/package.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/var/spack/repos/builtin/packages/libdwarf/package.py b/var/spack/repos/builtin/packages/libdwarf/package.py index c9c6e9404f..0e2e702b1c 100644 --- a/var/spack/repos/builtin/packages/libdwarf/package.py +++ b/var/spack/repos/builtin/packages/libdwarf/package.py @@ -23,7 +23,6 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## from spack import * -import os # Only build certain parts of dwarf because the other ones break. dwarf_dirs = ['libdwarf', 'dwarfdump2'] @@ -45,7 +44,9 @@ class Libdwarf(Package): list_url = homepage version('20160507', 'ae32d6f9ece5daf05e2d4b14822ea811') - + version('20130729', '4cc5e48693f7b93b7aa0261e63c0e21d') + version('20130207', '64b42692e947d5180e162e46c689dfbf') + version('20130126', 'ded74a5e90edb5a12aac3c29d260c5db') depends_on("libelf") parallel = False @@ -67,7 +68,11 @@ def install(self, spec, prefix): install('libdwarf.h', prefix.include) install('dwarf.h', prefix.include) - with working_dir('dwarfdump'): + if spec.satisfies('@20130126:20130729'): + dwarfdump_dir = 'dwarfdump2' + else: + dwarfdump_dir = 'dwarfdump' + with working_dir(dwarfdump_dir): configure("--prefix=" + prefix) # This makefile has strings of copy commands that From 2262bc95f1139e8599b061474569fab69d57a7f4 Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Tue, 7 Jun 2016 22:04:11 -0500 Subject: [PATCH 311/481] new package: r-packrat Manage the R packages your project depends on in an isolated, portable, and reproducible way. https://github.com/rstudio/packrat/ --- .../builtin/packages/r-packrat/package.py | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 var/spack/repos/builtin/packages/r-packrat/package.py diff --git a/var/spack/repos/builtin/packages/r-packrat/package.py b/var/spack/repos/builtin/packages/r-packrat/package.py new file mode 100644 index 0000000000..f7cebf9d49 --- /dev/null +++ b/var/spack/repos/builtin/packages/r-packrat/package.py @@ -0,0 +1,41 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class RPackrat(Package): + """Manage the R packages your project depends on in an isolated, portable, + and reproducible way.""" + + homepage = 'https://github.com/rstudio/packrat/' + url = "https://cran.r-project.org/src/contrib/packrat_0.4.7-1.tar.gz" + + version('0.4.7-1', '80c2413269b292ade163a70ba5053e84') + + extends('R') + + def install(self, spec, prefix): + R('CMD', 'INSTALL', + '--library={0}'.format(self.module.r_lib_dir), self.stage.source_path) From bb57b82794bf8434904235af42ad15914eff24ee Mon Sep 17 00:00:00 2001 From: Greg Lee Date: Tue, 7 Jun 2016 20:15:02 -0700 Subject: [PATCH 312/481] flake8 fixes --- var/spack/repos/builtin/packages/libdwarf/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/libdwarf/package.py b/var/spack/repos/builtin/packages/libdwarf/package.py index 0e2e702b1c..594271f655 100644 --- a/var/spack/repos/builtin/packages/libdwarf/package.py +++ b/var/spack/repos/builtin/packages/libdwarf/package.py @@ -27,6 +27,7 @@ # Only build certain parts of dwarf because the other ones break. dwarf_dirs = ['libdwarf', 'dwarfdump2'] + class Libdwarf(Package): """The DWARF Debugging Information Format is of interest to programmers working on compilers and debuggers (and any one @@ -51,7 +52,6 @@ class Libdwarf(Package): parallel = False - def install(self, spec, prefix): # dwarf build does not set arguments for ar properly make.add_default_arg('ARFLAGS=rcs') From 738720a000ab200cf93532de506eda71b1d582e5 Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Tue, 7 Jun 2016 23:20:46 -0500 Subject: [PATCH 313/481] new package: r-devtools This PR brings in the R devtools package and it dependencies. --- .../repos/builtin/packages/r-R6/package.py | 45 +++++++++++++++ .../repos/builtin/packages/r-curl/package.py | 50 +++++++++++++++++ .../builtin/packages/r-devtools/package.py | 49 +++++++++++++++++ .../builtin/packages/r-digest/package.py | 55 +++++++++++++++++++ .../repos/builtin/packages/r-git2r/package.py | 45 +++++++++++++++ .../repos/builtin/packages/r-httr/package.py | 48 ++++++++++++++++ .../builtin/packages/r-jsonlite/package.py | 49 +++++++++++++++++ .../builtin/packages/r-memoise/package.py | 43 +++++++++++++++ .../repos/builtin/packages/r-mime/package.py | 41 ++++++++++++++ .../builtin/packages/r-openssl/package.py | 51 +++++++++++++++++ .../builtin/packages/r-rstudioapi/package.py | 41 ++++++++++++++ .../builtin/packages/r-whisker/package.py | 41 ++++++++++++++ .../repos/builtin/packages/r-withr/package.py | 43 +++++++++++++++ 13 files changed, 601 insertions(+) create mode 100644 var/spack/repos/builtin/packages/r-R6/package.py create mode 100644 var/spack/repos/builtin/packages/r-curl/package.py create mode 100644 var/spack/repos/builtin/packages/r-devtools/package.py create mode 100644 var/spack/repos/builtin/packages/r-digest/package.py create mode 100644 var/spack/repos/builtin/packages/r-git2r/package.py create mode 100644 var/spack/repos/builtin/packages/r-httr/package.py create mode 100644 var/spack/repos/builtin/packages/r-jsonlite/package.py create mode 100644 var/spack/repos/builtin/packages/r-memoise/package.py create mode 100644 var/spack/repos/builtin/packages/r-mime/package.py create mode 100644 var/spack/repos/builtin/packages/r-openssl/package.py create mode 100644 var/spack/repos/builtin/packages/r-rstudioapi/package.py create mode 100644 var/spack/repos/builtin/packages/r-whisker/package.py create mode 100644 var/spack/repos/builtin/packages/r-withr/package.py diff --git a/var/spack/repos/builtin/packages/r-R6/package.py b/var/spack/repos/builtin/packages/r-R6/package.py new file mode 100644 index 0000000000..9dd8aebb75 --- /dev/null +++ b/var/spack/repos/builtin/packages/r-R6/package.py @@ -0,0 +1,45 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class RR6(Package): + """The R6 package allows the creation of classes with reference semantics, + similar to R's built-in reference classes. Compared to reference classes, + R6 classes are simpler and lighter-weight, and they are not built on S4 + classes so they do not require the methods package. These classes allow + public and private members, and they support inheritance, even when the + classes are defined in different packages.""" + + homepage = "https://github.com/wch/R6/" + url = "https://cran.r-project.org/src/contrib/R6_2.1.2.tar.gz" + + version('2.1.2', 'b6afb9430e48707be87638675390e457') + + extends('R') + + def install(self, spec, prefix): + R('CMD', 'INSTALL', + '--library={0}'.format(self.module.r_lib_dir), self.stage.source_path) diff --git a/var/spack/repos/builtin/packages/r-curl/package.py b/var/spack/repos/builtin/packages/r-curl/package.py new file mode 100644 index 0000000000..38ceeba5e4 --- /dev/null +++ b/var/spack/repos/builtin/packages/r-curl/package.py @@ -0,0 +1,50 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class RCurl(Package): + """The curl() and curl_download() functions provide highly configurable + drop-in replacements for base url() and download.file() with better + performance, support for encryption (https, ftps), gzip compression, + authentication, and other libcurl goodies. The core of the package + implements a framework for performing fully customized requests where data + can be processed either in memory, on disk, or streaming via the callback + or connection interfaces. Some knowledge of libcurl is recommended; for a + more-user-friendly web client see the 'httr' package which builds on this + package with http specific tools and logic.""" + + homepage = "https://github.com/jeroenooms/curl" + url = "https://cran.r-project.org/src/contrib/curl_0.9.7.tar.gz" + + version('0.9.7', 'a101f7de948cb828fef571c730f39217') + + extends('R') + + depends_on('curl') + + def install(self, spec, prefix): + R('CMD', 'INSTALL', + '--library={0}'.format(self.module.r_lib_dir), self.stage.source_path) diff --git a/var/spack/repos/builtin/packages/r-devtools/package.py b/var/spack/repos/builtin/packages/r-devtools/package.py new file mode 100644 index 0000000000..785f90107a --- /dev/null +++ b/var/spack/repos/builtin/packages/r-devtools/package.py @@ -0,0 +1,49 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class RDevtools(Package): + """Collection of package development tools.""" + + homepage = "https://github.com/hadley/devtools" + url = "https://cran.r-project.org/src/contrib/devtools_1.11.1.tar.gz" + + version('1.11.1', '242672ee27d24dddcbdaac88c586b6c2') + + extends('R') + + depends_on('r-httr') + depends_on('r-memoise') + depends_on('r-whisker') + depends_on('r-digest') + depends_on('r-rstudioapi') + depends_on('r-jsonlite') + depends_on('r-git2r') + depends_on('r-withr') + + def install(self, spec, prefix): + R('CMD', 'INSTALL', + '--library={0}'.format(self.module.r_lib_dir), self.stage.source_path) diff --git a/var/spack/repos/builtin/packages/r-digest/package.py b/var/spack/repos/builtin/packages/r-digest/package.py new file mode 100644 index 0000000000..8d6569e176 --- /dev/null +++ b/var/spack/repos/builtin/packages/r-digest/package.py @@ -0,0 +1,55 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class RDigest(Package): + """Implementation of a function 'digest()' for the creation of hash digests + of arbitrary R objects (using the md5, sha-1, sha-256, crc32, xxhash and + murmurhash algorithms) permitting easy comparison of R language objects, as + well as a function 'hmac()' to create hash-based message authentication + code. The md5 algorithm by Ron Rivest is specified in RFC 1321, the sha-1 + and sha-256 algorithms are specified in FIPS-180-1 and FIPS-180-2, and the + crc32 algorithm is described in + ftp://ftp.rocksoft.com/cliens/rocksoft/papers/crc_v3.txt. For md5, sha-1, + sha-256 and aes, this package uses small standalone implementations that + were provided by Christophe Devine. For crc32, code from the zlib library + is used. For sha-512, an implementation by Aaron D. Gifford is used. For + xxhash, the implementation by Yann Collet is used. For murmurhash, an + implementation by Shane Day is used. Please note that this package is not + meant to be deployed for cryptographic purposes for which more + comprehensive (and widely tested) libraries such as OpenSSL should be + used.""" + + homepage = "http://dirk.eddelbuettel.com/code/digest.html" + url = "https://cran.r-project.org/src/contrib/digest_0.6.9.tar.gz" + + version('0.6.9', '48048ce6c466bdb124716e45ba4a0e83') + + extends('R') + + def install(self, spec, prefix): + R('CMD', 'INSTALL', + '--library={0}'.format(self.module.r_lib_dir), self.stage.source_path) diff --git a/var/spack/repos/builtin/packages/r-git2r/package.py b/var/spack/repos/builtin/packages/r-git2r/package.py new file mode 100644 index 0000000000..cf3a401903 --- /dev/null +++ b/var/spack/repos/builtin/packages/r-git2r/package.py @@ -0,0 +1,45 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class RGit2r(Package): + """Interface to the 'libgit2' library, which is a pure C implementation of + the 'Git' core methods. Provides access to 'Git' repositories to extract + data and running some basic 'Git' commands.""" + + homepage = "https://github.com/ropensci/git2r" + url = "https://cran.r-project.org/src/contrib/git2r_0.15.0.tar.gz" + + version('0.15.0', '57658b3298f9b9aadc0dd77b4ef6a1e1') + + extends('R') + + depends_on('zlib') + depends_on('openssl') + + def install(self, spec, prefix): + R('CMD', 'INSTALL', + '--library={0}'.format(self.module.r_lib_dir), self.stage.source_path) diff --git a/var/spack/repos/builtin/packages/r-httr/package.py b/var/spack/repos/builtin/packages/r-httr/package.py new file mode 100644 index 0000000000..d10f9e6776 --- /dev/null +++ b/var/spack/repos/builtin/packages/r-httr/package.py @@ -0,0 +1,48 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class RHttr(Package): + """Useful tools for working with HTTP organised by HTTP verbs (GET(), + POST(), etc). Configuration functions make it easy to control additional + request components (authenticate(), add_headers() and so on).""" + + homepage = "https://github.com/hadley/httr" + url = "https://cran.r-project.org/src/contrib/httr_1.1.0.tar.gz" + + version('1.1.0', '5ffbbc5c2529e49f00aaa521a2b35600') + + extends('R') + + depends_on('r-jsonlite') + depends_on('r-mime') + depends_on('r-curl') + depends_on('r-openssl') + depends_on('r-R6') + + def install(self, spec, prefix): + R('CMD', 'INSTALL', + '--library={0}'.format(self.module.r_lib_dir), self.stage.source_path) diff --git a/var/spack/repos/builtin/packages/r-jsonlite/package.py b/var/spack/repos/builtin/packages/r-jsonlite/package.py new file mode 100644 index 0000000000..c8a458fa17 --- /dev/null +++ b/var/spack/repos/builtin/packages/r-jsonlite/package.py @@ -0,0 +1,49 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class RJsonlite(Package): + """A fast JSON parser and generator optimized for statistical data and the + web. Started out as a fork of 'RJSONIO', but has been completely rewritten + in recent versions. The package offers flexible, robust, high performance + tools for working with JSON in R and is particularly powerful for building + pipelines and interacting with a web API. The implementation is based on + the mapping described in the vignette (Ooms, 2014). In addition to + converting JSON data from/to R objects, 'jsonlite' contains functions to + stream, validate, and prettify JSON data. The unit tests included with the + package verify that all edge cases are encoded and decoded consistently for + use with dynamic data in systems and applications.""" + + homepage = "https://github.com/jeroenooms/jsonlite" + url = "https://cran.r-project.org/src/contrib/jsonlite_0.9.21.tar.gz" + + version('0.9.21', '4fc382747f88a79ff0718a0d06bed45d') + + extends('R') + + def install(self, spec, prefix): + R('CMD', 'INSTALL', + '--library={0}'.format(self.module.r_lib_dir), self.stage.source_path) diff --git a/var/spack/repos/builtin/packages/r-memoise/package.py b/var/spack/repos/builtin/packages/r-memoise/package.py new file mode 100644 index 0000000000..7dfda78b8a --- /dev/null +++ b/var/spack/repos/builtin/packages/r-memoise/package.py @@ -0,0 +1,43 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class RMemoise(Package): + """Cache the results of a function so that when you call it again with the + same arguments it returns the pre-computed value.""" + + homepage = "https://github.com/hadley/memoise" + url = "https://cran.r-project.org/src/contrib/memoise_1.0.0.tar.gz" + + version('1.0.0', 'd31145292e2a88ae9a504cab1602e4ac') + + extends('R') + + depends_on('r-digest') + + def install(self, spec, prefix): + R('CMD', 'INSTALL', + '--library={0}'.format(self.module.r_lib_dir), self.stage.source_path) diff --git a/var/spack/repos/builtin/packages/r-mime/package.py b/var/spack/repos/builtin/packages/r-mime/package.py new file mode 100644 index 0000000000..442280ae44 --- /dev/null +++ b/var/spack/repos/builtin/packages/r-mime/package.py @@ -0,0 +1,41 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class RMime(Package): + """Guesses the MIME type from a filename extension using the data derived + from /etc/mime.types in UNIX-type systems.""" + + homepage = "https://github.com/yihui/mime" + url = "https://cran.r-project.org/src/contrib/mime_0.4.tar.gz" + + version('0.4', '789cb33e41db2206c6fc7c3e9fbc2c02') + + extends('R') + + def install(self, spec, prefix): + R('CMD', 'INSTALL', + '--library={0}'.format(self.module.r_lib_dir), self.stage.source_path) diff --git a/var/spack/repos/builtin/packages/r-openssl/package.py b/var/spack/repos/builtin/packages/r-openssl/package.py new file mode 100644 index 0000000000..8105f421dc --- /dev/null +++ b/var/spack/repos/builtin/packages/r-openssl/package.py @@ -0,0 +1,51 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class ROpenssl(Package): + """Bindings to OpenSSL libssl and libcrypto, plus custom SSH pubkey + parsers. Supports RSA, DSA and EC curves P-256, P-384 and P-521. + Cryptographic signatures can either be created and verified manually or via + x509 certificates. AES can be used in cbc, ctr or gcm mode for symmetric + encryption; RSA for asymmetric (public key) encryption or EC for Diffie + Hellman. High-level envelope functions combine RSA and AES for encrypting + arbitrary sized data. Other utilities include key generators, hash + functions (md5, sha1, sha256, etc), base64 encoder, a secure random number + generator, and 'bignum' math methods for manually performing crypto + calculations on large multibyte integers.""" + + homepage = "https://github.com/jeroenooms/openssl#readme" + url = "https://cran.r-project.org/src/contrib/openssl_0.9.4.tar.gz" + + version('0.9.4', '82a890e71ed0e74499878bedacfb8ccb') + + extends('R') + + depends_on('openssl') + + def install(self, spec, prefix): + R('CMD', 'INSTALL', + '--library={0}'.format(self.module.r_lib_dir), self.stage.source_path) diff --git a/var/spack/repos/builtin/packages/r-rstudioapi/package.py b/var/spack/repos/builtin/packages/r-rstudioapi/package.py new file mode 100644 index 0000000000..50c92ee1c2 --- /dev/null +++ b/var/spack/repos/builtin/packages/r-rstudioapi/package.py @@ -0,0 +1,41 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class RRstudioapi(Package): + """Access the RStudio API (if available) and provide informative error + messages when it's not.""" + + homepage = "https://cran.r-project.org/web/packages/rstudioapi/index.html" + url = "https://cran.r-project.org/src/contrib/rstudioapi_0.5.tar.gz" + + version('0.5', '6ce1191da74e7bcbf06b61339486b3ba') + + extends('R') + + def install(self, spec, prefix): + R('CMD', 'INSTALL', + '--library={0}'.format(self.module.r_lib_dir), self.stage.source_path) diff --git a/var/spack/repos/builtin/packages/r-whisker/package.py b/var/spack/repos/builtin/packages/r-whisker/package.py new file mode 100644 index 0000000000..d2af271441 --- /dev/null +++ b/var/spack/repos/builtin/packages/r-whisker/package.py @@ -0,0 +1,41 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class RWhisker(Package): + """logicless templating, reuse templates in many programming languages + including R""" + + homepage = "http://github.com/edwindj/whisker" + url = "https://cran.r-project.org/src/contrib/whisker_0.3-2.tar.gz" + + version('0.3-2', 'c4b9bf9a22e69ce003fe68663ab5e8e6') + + extends('R') + + def install(self, spec, prefix): + R('CMD', 'INSTALL', + '--library={0}'.format(self.module.r_lib_dir), self.stage.source_path) diff --git a/var/spack/repos/builtin/packages/r-withr/package.py b/var/spack/repos/builtin/packages/r-withr/package.py new file mode 100644 index 0000000000..418d15e1e8 --- /dev/null +++ b/var/spack/repos/builtin/packages/r-withr/package.py @@ -0,0 +1,43 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class RWithr(Package): + """A set of functions to run code 'with' safely and temporarily modified + global state. Many of these functions were originally a part of the + 'devtools' package, this provides a simple package with limited + dependencies to provide access to these functions.""" + + homepage = "http://github.com/jimhester/withr" + url = "https://cran.r-project.org/src/contrib/withr_1.0.1.tar.gz" + + version('1.0.1', 'ac38af2c6f74027c9592dd8f0acb7598') + + extends('R') + + def install(self, spec, prefix): + R('CMD', 'INSTALL', + '--library={0}'.format(self.module.r_lib_dir), self.stage.source_path) From ad0cfa41aa970887771d3cb5a84ae41f7800432b Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Tue, 7 Jun 2016 23:37:49 -0500 Subject: [PATCH 314/481] Fix flake8 errors --- var/spack/repos/builtin/packages/r-R6/package.py | 4 ++-- var/spack/repos/builtin/packages/r-curl/package.py | 4 ++-- var/spack/repos/builtin/packages/r-devtools/package.py | 4 ++-- var/spack/repos/builtin/packages/r-digest/package.py | 4 ++-- var/spack/repos/builtin/packages/r-git2r/package.py | 4 ++-- var/spack/repos/builtin/packages/r-httr/package.py | 4 ++-- var/spack/repos/builtin/packages/r-jsonlite/package.py | 4 ++-- var/spack/repos/builtin/packages/r-memoise/package.py | 4 ++-- var/spack/repos/builtin/packages/r-mime/package.py | 4 ++-- var/spack/repos/builtin/packages/r-openssl/package.py | 4 ++-- var/spack/repos/builtin/packages/r-rstudioapi/package.py | 4 ++-- var/spack/repos/builtin/packages/r-whisker/package.py | 4 ++-- var/spack/repos/builtin/packages/r-withr/package.py | 4 ++-- 13 files changed, 26 insertions(+), 26 deletions(-) diff --git a/var/spack/repos/builtin/packages/r-R6/package.py b/var/spack/repos/builtin/packages/r-R6/package.py index 9dd8aebb75..30f489b3fd 100644 --- a/var/spack/repos/builtin/packages/r-R6/package.py +++ b/var/spack/repos/builtin/packages/r-R6/package.py @@ -41,5 +41,5 @@ class RR6(Package): extends('R') def install(self, spec, prefix): - R('CMD', 'INSTALL', - '--library={0}'.format(self.module.r_lib_dir), self.stage.source_path) + R('CMD', 'INSTALL', '--library={0}'.format(self.module.r_lib_dir), + self.stage.source_path) diff --git a/var/spack/repos/builtin/packages/r-curl/package.py b/var/spack/repos/builtin/packages/r-curl/package.py index 38ceeba5e4..23d75880a7 100644 --- a/var/spack/repos/builtin/packages/r-curl/package.py +++ b/var/spack/repos/builtin/packages/r-curl/package.py @@ -46,5 +46,5 @@ class RCurl(Package): depends_on('curl') def install(self, spec, prefix): - R('CMD', 'INSTALL', - '--library={0}'.format(self.module.r_lib_dir), self.stage.source_path) + R('CMD', 'INSTALL', '--library={0}'.format(self.module.r_lib_dir), + self.stage.source_path) diff --git a/var/spack/repos/builtin/packages/r-devtools/package.py b/var/spack/repos/builtin/packages/r-devtools/package.py index 785f90107a..5c73eab936 100644 --- a/var/spack/repos/builtin/packages/r-devtools/package.py +++ b/var/spack/repos/builtin/packages/r-devtools/package.py @@ -45,5 +45,5 @@ class RDevtools(Package): depends_on('r-withr') def install(self, spec, prefix): - R('CMD', 'INSTALL', - '--library={0}'.format(self.module.r_lib_dir), self.stage.source_path) + R('CMD', 'INSTALL', '--library={0}'.format(self.module.r_lib_dir), + self.stage.source_path) diff --git a/var/spack/repos/builtin/packages/r-digest/package.py b/var/spack/repos/builtin/packages/r-digest/package.py index 8d6569e176..a42e82f1ae 100644 --- a/var/spack/repos/builtin/packages/r-digest/package.py +++ b/var/spack/repos/builtin/packages/r-digest/package.py @@ -51,5 +51,5 @@ class RDigest(Package): extends('R') def install(self, spec, prefix): - R('CMD', 'INSTALL', - '--library={0}'.format(self.module.r_lib_dir), self.stage.source_path) + R('CMD', 'INSTALL', '--library={0}'.format(self.module.r_lib_dir), + self.stage.source_path) diff --git a/var/spack/repos/builtin/packages/r-git2r/package.py b/var/spack/repos/builtin/packages/r-git2r/package.py index cf3a401903..272dcbad93 100644 --- a/var/spack/repos/builtin/packages/r-git2r/package.py +++ b/var/spack/repos/builtin/packages/r-git2r/package.py @@ -41,5 +41,5 @@ class RGit2r(Package): depends_on('openssl') def install(self, spec, prefix): - R('CMD', 'INSTALL', - '--library={0}'.format(self.module.r_lib_dir), self.stage.source_path) + R('CMD', 'INSTALL', '--library={0}'.format(self.module.r_lib_dir), + self.stage.source_path) diff --git a/var/spack/repos/builtin/packages/r-httr/package.py b/var/spack/repos/builtin/packages/r-httr/package.py index d10f9e6776..0f379512a5 100644 --- a/var/spack/repos/builtin/packages/r-httr/package.py +++ b/var/spack/repos/builtin/packages/r-httr/package.py @@ -44,5 +44,5 @@ class RHttr(Package): depends_on('r-R6') def install(self, spec, prefix): - R('CMD', 'INSTALL', - '--library={0}'.format(self.module.r_lib_dir), self.stage.source_path) + R('CMD', 'INSTALL', '--library={0}'.format(self.module.r_lib_dir), + self.stage.source_path) diff --git a/var/spack/repos/builtin/packages/r-jsonlite/package.py b/var/spack/repos/builtin/packages/r-jsonlite/package.py index c8a458fa17..777506b984 100644 --- a/var/spack/repos/builtin/packages/r-jsonlite/package.py +++ b/var/spack/repos/builtin/packages/r-jsonlite/package.py @@ -45,5 +45,5 @@ class RJsonlite(Package): extends('R') def install(self, spec, prefix): - R('CMD', 'INSTALL', - '--library={0}'.format(self.module.r_lib_dir), self.stage.source_path) + R('CMD', 'INSTALL', '--library={0}'.format(self.module.r_lib_dir), + self.stage.source_path) diff --git a/var/spack/repos/builtin/packages/r-memoise/package.py b/var/spack/repos/builtin/packages/r-memoise/package.py index 7dfda78b8a..ca74ab02a2 100644 --- a/var/spack/repos/builtin/packages/r-memoise/package.py +++ b/var/spack/repos/builtin/packages/r-memoise/package.py @@ -39,5 +39,5 @@ class RMemoise(Package): depends_on('r-digest') def install(self, spec, prefix): - R('CMD', 'INSTALL', - '--library={0}'.format(self.module.r_lib_dir), self.stage.source_path) + R('CMD', 'INSTALL', '--library={0}'.format(self.module.r_lib_dir), + self.stage.source_path) diff --git a/var/spack/repos/builtin/packages/r-mime/package.py b/var/spack/repos/builtin/packages/r-mime/package.py index 442280ae44..ac52d785b1 100644 --- a/var/spack/repos/builtin/packages/r-mime/package.py +++ b/var/spack/repos/builtin/packages/r-mime/package.py @@ -37,5 +37,5 @@ class RMime(Package): extends('R') def install(self, spec, prefix): - R('CMD', 'INSTALL', - '--library={0}'.format(self.module.r_lib_dir), self.stage.source_path) + R('CMD', 'INSTALL', '--library={0}'.format(self.module.r_lib_dir), + self.stage.source_path) diff --git a/var/spack/repos/builtin/packages/r-openssl/package.py b/var/spack/repos/builtin/packages/r-openssl/package.py index 8105f421dc..fd6ad84aa3 100644 --- a/var/spack/repos/builtin/packages/r-openssl/package.py +++ b/var/spack/repos/builtin/packages/r-openssl/package.py @@ -47,5 +47,5 @@ class ROpenssl(Package): depends_on('openssl') def install(self, spec, prefix): - R('CMD', 'INSTALL', - '--library={0}'.format(self.module.r_lib_dir), self.stage.source_path) + R('CMD', 'INSTALL', '--library={0}'.format(self.module.r_lib_dir), + self.stage.source_path) diff --git a/var/spack/repos/builtin/packages/r-rstudioapi/package.py b/var/spack/repos/builtin/packages/r-rstudioapi/package.py index 50c92ee1c2..83424392c9 100644 --- a/var/spack/repos/builtin/packages/r-rstudioapi/package.py +++ b/var/spack/repos/builtin/packages/r-rstudioapi/package.py @@ -37,5 +37,5 @@ class RRstudioapi(Package): extends('R') def install(self, spec, prefix): - R('CMD', 'INSTALL', - '--library={0}'.format(self.module.r_lib_dir), self.stage.source_path) + R('CMD', 'INSTALL', '--library={0}'.format(self.module.r_lib_dir), + self.stage.source_path) diff --git a/var/spack/repos/builtin/packages/r-whisker/package.py b/var/spack/repos/builtin/packages/r-whisker/package.py index d2af271441..f534568ec7 100644 --- a/var/spack/repos/builtin/packages/r-whisker/package.py +++ b/var/spack/repos/builtin/packages/r-whisker/package.py @@ -37,5 +37,5 @@ class RWhisker(Package): extends('R') def install(self, spec, prefix): - R('CMD', 'INSTALL', - '--library={0}'.format(self.module.r_lib_dir), self.stage.source_path) + R('CMD', 'INSTALL', '--library={0}'.format(self.module.r_lib_dir), + self.stage.source_path) diff --git a/var/spack/repos/builtin/packages/r-withr/package.py b/var/spack/repos/builtin/packages/r-withr/package.py index 418d15e1e8..a9f0cb600f 100644 --- a/var/spack/repos/builtin/packages/r-withr/package.py +++ b/var/spack/repos/builtin/packages/r-withr/package.py @@ -39,5 +39,5 @@ class RWithr(Package): extends('R') def install(self, spec, prefix): - R('CMD', 'INSTALL', - '--library={0}'.format(self.module.r_lib_dir), self.stage.source_path) + R('CMD', 'INSTALL', '--library={0}'.format(self.module.r_lib_dir), + self.stage.source_path) From bb19dffb266ab63ff8733051a4461ecdb85bd18f Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Tue, 7 Jun 2016 23:40:12 -0500 Subject: [PATCH 315/481] Fix flake8 error. --- var/spack/repos/builtin/packages/r-packrat/package.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/r-packrat/package.py b/var/spack/repos/builtin/packages/r-packrat/package.py index f7cebf9d49..c9b02a284e 100644 --- a/var/spack/repos/builtin/packages/r-packrat/package.py +++ b/var/spack/repos/builtin/packages/r-packrat/package.py @@ -37,5 +37,5 @@ class RPackrat(Package): extends('R') def install(self, spec, prefix): - R('CMD', 'INSTALL', - '--library={0}'.format(self.module.r_lib_dir), self.stage.source_path) + R('CMD', 'INSTALL', '--library={0}'.format(self.module.r_lib_dir), + self.stage.source_path) From e124c86a4fe515ed4497fafac4bd877e22e2e25a Mon Sep 17 00:00:00 2001 From: Nicolas Richart Date: Wed, 8 Jun 2016 14:17:46 +0200 Subject: [PATCH 316/481] Corrected the shared libraries and tests in mumps package --- .../repos/builtin/packages/mumps/package.py | 92 ++++++++++--------- 1 file changed, 50 insertions(+), 42 deletions(-) diff --git a/var/spack/repos/builtin/packages/mumps/package.py b/var/spack/repos/builtin/packages/mumps/package.py index 92c45c9b95..652c85a031 100644 --- a/var/spack/repos/builtin/packages/mumps/package.py +++ b/var/spack/repos/builtin/packages/mumps/package.py @@ -23,7 +23,11 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## from spack import * -import os, sys, glob +import os +import sys +import glob +import subprocess + class Mumps(Package): """MUMPS: a MUltifrontal Massively Parallel sparse direct Solver""" @@ -54,6 +58,8 @@ class Mumps(Package): depends_on('scalapack', when='+mpi') depends_on('mpi', when='+mpi') + patch('mumps-shared.patch', when='+shared') + # this function is not a patch function because in case scalapack # is needed it uses self.spec['scalapack'].fc_link set by the # setup_dependent_environment in scalapck. This happen after patch @@ -116,43 +122,39 @@ def write_makefile_inc(self): if '+mpi' in self.spec: makefile_conf.extend( - ["CC = %s" % join_path(self.spec['mpi'].prefix.bin, 'mpicc'), - "FC = %s" % join_path(self.spec['mpi'].prefix.bin, 'mpif90'), - "FL = %s" % join_path(self.spec['mpi'].prefix.bin, 'mpif90'), + ["CC = %s" % self.spec['mpi'].mpicc, + "FC = %s" % self.spec['mpi'].mpif90, "SCALAP = %s" % self.spec['scalapack'].fc_link, "MUMPS_TYPE = par"]) else: makefile_conf.extend( ["CC = cc", "FC = fc", - "FL = fc", "MUMPS_TYPE = seq"]) # TODO: change the value to the correct one according to the # compiler possible values are -DAdd_, -DAdd__ and/or -DUPPER - makefile_conf.append("CDEFS = -DAdd_") + makefile_conf.extend([ + 'CDEFS = -DAdd_', + 'FL = $(FC)', + ]) if '+shared' in self.spec: + makefile_conf.append('SHLIBEXT = .%s' % dso_suffix) if sys.platform == 'darwin': - # Building dylibs with mpif90 causes segfaults on 10.8 and 10.10. Use gfortran. (Homebrew) - makefile_conf.extend([ - 'LIBEXT=.dylib', - 'AR=%s -dynamiclib -Wl,-install_name -Wl,%s/$(notdir $@) -undefined dynamic_lookup -o ' % (os.environ['FC'],prefix.lib), - 'RANLIB=echo' - ]) + makefile_conf.append( + 'LDFLAGS = -dynamiclib -Wl,-install_name -Wl,{0}/$(notdir $@) {1}{0} -undefined dynamic_lookup'.format(prefix.lib, self.compiler.fc_rpath_arg) + ) else: - makefile_conf.extend([ - 'LIBEXT=.so', - 'AR=$(FL) -shared -Wl,-soname -Wl,%s/$(notdir $@) -o' % prefix.lib, - 'RANLIB=echo' - ]) - else: - makefile_conf.extend([ - 'LIBEXT = .a', - 'AR = ar vr', - 'RANLIB = ranlib' - ]) + makefile_conf.append( + 'LDFLAGS = -shared {1}{0}'.format(prefix.lib, self.compiler.fc_rpath_arg) + ) + makefile_conf.extend([ + 'LIBEXT = .a', + 'AR = ar vr ', + 'RANLIB = ranlib' + ]) makefile_inc_template = join_path(os.path.dirname(self.module.__file__), 'Makefile.inc') @@ -172,38 +174,44 @@ def install(self, spec, prefix): # the choice to compile ?examples is to have kind of a sanity # check on the libraries generated. if '+float' in spec: - make_libs.append('sexamples') + make_libs.append('s') if '+complex' in spec: - make_libs.append('cexamples') + make_libs.append('c') if '+double' in spec: - make_libs.append('dexamples') + make_libs.append('d') if '+complex' in spec: - make_libs.append('zexamples') + make_libs.append('z') self.write_makefile_inc() - # Build fails in parallel - make(*make_libs, parallel=False) + make('mumps_lib', parallel=False) + make(*make_libs) install_tree('lib', prefix.lib) install_tree('include', prefix.include) if '~mpi' in spec: - lib_dsuffix = '.dylib' if sys.platform == 'darwin' else '.so' - lib_suffix = lib_dsuffix if '+shared' in spec else '.a' - install('libseq/libmpiseq%s' % lib_suffix, prefix.lib) - for f in glob.glob(join_path('libseq','*.h')): - install(f, prefix.include) + install('libseq/libmpiseq.a', prefix.lib) + if '+shared' in spec: + install('libseq/libmpiseq.{0}'.format(dso_suffix), prefix.lib) + install('libseq/mpi.h', prefix.include) + install('libseq/mpif.h', prefix.include) # FIXME: extend the tests to mpirun -np 2 (or alike) when build with MPI # FIXME: use something like numdiff to compare blessed output with the current + # TODO: test the installed mumps and not the one in stage + for t in make_libs: + make('{0}examples'.format(t)) + with working_dir('examples'): - if '+float' in spec: - os.system('./ssimpletest < input_simpletest_real') - if '+complex' in spec: - os.system('./csimpletest < input_simpletest_real') - if '+double' in spec: - os.system('./dsimpletest < input_simpletest_real') - if '+complex' in spec: - os.system('./zsimpletest < input_simpletest_cmplx') + for t in make_libs: + input_file = 'input_simpletest_{0}'.format( + 'real' if t in ['s', 'd'] else 'cmplx') + with open(input_file) as input: + test = './{0}simpletest'.format(t) + ret = subprocess.call(test, + stdin=input) + if ret is not 0: + raise RuntimeError( + 'The test {0} did not pass'.format(test)) From 498c098d11814311e7fbca0084e5298d28d30ce6 Mon Sep 17 00:00:00 2001 From: Nicolas Richart Date: Wed, 8 Jun 2016 14:57:54 +0200 Subject: [PATCH 317/481] Cleaning flake warnings in mumps package --- .../repos/builtin/packages/mumps/package.py | 75 +++++++++++-------- 1 file changed, 44 insertions(+), 31 deletions(-) diff --git a/var/spack/repos/builtin/packages/mumps/package.py b/var/spack/repos/builtin/packages/mumps/package.py index 652c85a031..c3cd01a2aa 100644 --- a/var/spack/repos/builtin/packages/mumps/package.py +++ b/var/spack/repos/builtin/packages/mumps/package.py @@ -25,7 +25,6 @@ from spack import * import os import sys -import glob import subprocess @@ -48,7 +47,6 @@ class Mumps(Package): variant('idx64', default=False, description='Use int64_t/integer*8 as default index type') variant('shared', default=True, description='Build shared libraries') - depends_on('scotch + esmumps', when='~ptscotch+scotch') depends_on('scotch + esmumps + mpi', when='+ptscotch') depends_on('metis@5:', when='+metis') @@ -59,50 +57,63 @@ class Mumps(Package): depends_on('mpi', when='+mpi') patch('mumps-shared.patch', when='+shared') - + # this function is not a patch function because in case scalapack # is needed it uses self.spec['scalapack'].fc_link set by the # setup_dependent_environment in scalapck. This happen after patch # end before install # def patch(self): def write_makefile_inc(self): - if ('+parmetis' in self.spec or '+ptscotch' in self.spec) and '+mpi' not in self.spec: - raise RuntimeError('You cannot use the variants parmetis or ptscotch without mpi') + if (('+parmetis' in self.spec or + '+ptscotch' in self.spec)) and '+mpi' not in self.spec: + raise RuntimeError('You cannot use the variants parmetis or ptscotch without mpi') # NOQA: E501 - makefile_conf = ["LIBBLAS = -L%s -lblas" % self.spec['blas'].prefix.lib] + makefile_conf = [ + "LIBBLAS = -L%s -lblas" % self.spec['blas'].prefix.lib + ] orderings = ['-Dpord'] if '+ptscotch' in self.spec or '+scotch' in self.spec: join_lib = ' -l%s' % ('pt' if '+ptscotch' in self.spec else '') - makefile_conf.extend( - ["ISCOTCH = -I%s" % self.spec['scotch'].prefix.include, - "LSCOTCH = -L%s %s%s" % (self.spec['scotch'].prefix.lib, - join_lib, - join_lib.join(['esmumps', 'scotch', 'scotcherr']))]) + makefile_conf.extend([ + "ISCOTCH = -I%s" % self.spec['scotch'].prefix.include, + "LSCOTCH = -L%s %s%s" % (self.spec['scotch'].prefix.lib, + join_lib, + join_lib.join(['esmumps', + 'scotch', + 'scotcherr'])) + ]) orderings.append('-Dscotch') if '+ptscotch' in self.spec: orderings.append('-Dptscotch') if '+parmetis' in self.spec and '+metis' in self.spec: - libname = 'parmetis' if '+parmetis' in self.spec else 'metis' - makefile_conf.extend( - ["IMETIS = -I%s" % self.spec['parmetis'].prefix.include, - "LMETIS = -L%s -l%s -L%s -l%s" % (self.spec['parmetis'].prefix.lib, 'parmetis',self.spec['metis'].prefix.lib, 'metis')]) + makefile_conf.extend([ + "IMETIS = -I%s" % self.spec['parmetis'].prefix.include, + "LMETIS = -L%s -l%s -L%s -l%s" % ( + self.spec['parmetis'].prefix.lib, 'parmetis', + self.spec['metis'].prefix.lib, 'metis') + ]) orderings.append('-Dparmetis') elif '+metis' in self.spec: - makefile_conf.extend( - ["IMETIS = -I%s" % self.spec['metis'].prefix.include, - "LMETIS = -L%s -l%s" % (self.spec['metis'].prefix.lib, 'metis')]) + makefile_conf.extend([ + "IMETIS = -I%s" % self.spec['metis'].prefix.include, + "LMETIS = -L%s -l%s" % (self.spec['metis'].prefix.lib, + 'metis') + ]) orderings.append('-Dmetis') makefile_conf.append("ORDERINGSF = %s" % (' '.join(orderings))) - # when building shared libs need -fPIC, otherwise - # /usr/bin/ld: graph.o: relocation R_X86_64_32 against `.rodata.str1.1' can not be used when making a shared object; recompile with -fPIC + # when building shared libs need -fPIC, otherwise /usr/bin/ld: + # graph.o: relocation R_X86_64_32 against `.rodata.str1.1' can + # not be used when making a shared object; recompile with + # -fPIC fpic = '-fPIC' if '+shared' in self.spec else '' + # TODO: test this part, it needs a full blas, scalapack and # partitionning environment with 64bit integers if '+idx64' in self.spec: @@ -110,7 +121,7 @@ def write_makefile_inc(self): # the fortran compilation flags most probably are # working only for intel and gnu compilers this is # perhaps something the compiler should provide - ['OPTF = %s -O -DALLOW_NON_INIT %s' % (fpic,'-fdefault-integer-8' if self.compiler.name == "gcc" else '-i8'), + ['OPTF = %s -O -DALLOW_NON_INIT %s' % (fpic, '-fdefault-integer-8' if self.compiler.name == "gcc" else '-i8'), # NOQA: E501 'OPTL = %s -O ' % fpic, 'OPTC = %s -O -DINTSIZE64' % fpic]) else: @@ -119,7 +130,6 @@ def write_makefile_inc(self): 'OPTL = %s -O ' % fpic, 'OPTC = %s -O ' % fpic]) - if '+mpi' in self.spec: makefile_conf.extend( ["CC = %s" % self.spec['mpi'].mpicc, @@ -143,11 +153,13 @@ def write_makefile_inc(self): makefile_conf.append('SHLIBEXT = .%s' % dso_suffix) if sys.platform == 'darwin': makefile_conf.append( - 'LDFLAGS = -dynamiclib -Wl,-install_name -Wl,{0}/$(notdir $@) {1}{0} -undefined dynamic_lookup'.format(prefix.lib, self.compiler.fc_rpath_arg) + 'LDFLAGS = -dynamiclib -Wl,-install_name -Wl,{0}/$(notdir $@) {1}{0} -undefined dynamic_lookup'.format(prefix.lib, self.compiler.fc_rpath_arg) # NOQA: E501 ) else: makefile_conf.append( - 'LDFLAGS = -shared {1}{0}'.format(prefix.lib, self.compiler.fc_rpath_arg) + 'LDFLAGS = -shared {0}{1}'.format( + self.compiler.fc_rpath_arg, + prefix.lib) ) makefile_conf.extend([ @@ -156,8 +168,9 @@ def write_makefile_inc(self): 'RANLIB = ranlib' ]) - makefile_inc_template = join_path(os.path.dirname(self.module.__file__), - 'Makefile.inc') + makefile_inc_template = \ + join_path(os.path.dirname(self.module.__file__), + 'Makefile.inc') with open(makefile_inc_template, "r") as fh: makefile_conf.extend(fh.read().split('\n')) @@ -166,8 +179,6 @@ def write_makefile_inc(self): makefile_inc = '\n'.join(makefile_conf) fh.write(makefile_inc) - - def install(self, spec, prefix): make_libs = [] @@ -191,15 +202,17 @@ def install(self, spec, prefix): install_tree('lib', prefix.lib) install_tree('include', prefix.include) - if '~mpi' in spec: + if '~mpi' in spec: install('libseq/libmpiseq.a', prefix.lib) if '+shared' in spec: install('libseq/libmpiseq.{0}'.format(dso_suffix), prefix.lib) install('libseq/mpi.h', prefix.include) install('libseq/mpif.h', prefix.include) - # FIXME: extend the tests to mpirun -np 2 (or alike) when build with MPI - # FIXME: use something like numdiff to compare blessed output with the current + # FIXME: extend the tests to mpirun -np 2 (or alike) when + # build with MPI + # FIXME: use something like numdiff to compare blessed output + # with the current # TODO: test the installed mumps and not the one in stage for t in make_libs: make('{0}examples'.format(t)) From 165411f4a5bae4d398630664f863def130492a22 Mon Sep 17 00:00:00 2001 From: Nicolas Richart Date: Wed, 8 Jun 2016 15:15:39 +0200 Subject: [PATCH 318/481] bug fix in the +mpi case --- .../repos/builtin/packages/mumps/package.py | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/var/spack/repos/builtin/packages/mumps/package.py b/var/spack/repos/builtin/packages/mumps/package.py index c3cd01a2aa..8a90ef72d0 100644 --- a/var/spack/repos/builtin/packages/mumps/package.py +++ b/var/spack/repos/builtin/packages/mumps/package.py @@ -133,7 +133,7 @@ def write_makefile_inc(self): if '+mpi' in self.spec: makefile_conf.extend( ["CC = %s" % self.spec['mpi'].mpicc, - "FC = %s" % self.spec['mpi'].mpif90, + "FC = %s" % self.spec['mpi'].mpifc, "SCALAP = %s" % self.spec['scalapack'].fc_link, "MUMPS_TYPE = par"]) else: @@ -214,17 +214,18 @@ def install(self, spec, prefix): # FIXME: use something like numdiff to compare blessed output # with the current # TODO: test the installed mumps and not the one in stage - for t in make_libs: - make('{0}examples'.format(t)) - - with working_dir('examples'): + if '~mpi' in spec: for t in make_libs: - input_file = 'input_simpletest_{0}'.format( - 'real' if t in ['s', 'd'] else 'cmplx') - with open(input_file) as input: - test = './{0}simpletest'.format(t) - ret = subprocess.call(test, - stdin=input) - if ret is not 0: - raise RuntimeError( - 'The test {0} did not pass'.format(test)) + make('{0}examples'.format(t)) + + with working_dir('examples'): + for t in make_libs: + input_file = 'input_simpletest_{0}'.format( + 'real' if t in ['s', 'd'] else 'cmplx') + with open(input_file) as input: + test = './{0}simpletest'.format(t) + ret = subprocess.call(test, + stdin=input) + if ret is not 0: + raise RuntimeError( + 'The test {0} did not pass'.format(test)) From f4a39457de662376b5f9da300a06e2dff91b1aeb Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 8 Jun 2016 15:35:15 +0200 Subject: [PATCH 319/481] lmod : updated package and dependencies --- lib/spack/spack/version.py | 11 +++++ .../repos/builtin/packages/lmod/package.py | 16 +++---- .../packages/lua-luafilesystem/package.py | 46 +++++++++++++++++++ .../repos/builtin/packages/lua/package.py | 6 ++- 4 files changed, 69 insertions(+), 10 deletions(-) create mode 100644 var/spack/repos/builtin/packages/lua-luafilesystem/package.py diff --git a/lib/spack/spack/version.py b/lib/spack/spack/version.py index 247f6d2362..d86bcef3e0 100644 --- a/lib/spack/spack/version.py +++ b/lib/spack/spack/version.py @@ -124,6 +124,17 @@ def __init__(self, string): # last element of separators is '' self.separators = tuple(re.split(segment_regex, string)[1:-1]) + @property + def dotted(self): + return '.'.join(str(x) for x in self.version) + + @property + def underscored(self): + return '_'.join(str(x) for x in self.version) + + @property + def dashed(self): + return '-'.join(str(x) for x in self.version) def up_to(self, index): """Return a version string up to the specified component, exclusive. diff --git a/var/spack/repos/builtin/packages/lmod/package.py b/var/spack/repos/builtin/packages/lmod/package.py index 0a8b9b4577..9084c34881 100644 --- a/var/spack/repos/builtin/packages/lmod/package.py +++ b/var/spack/repos/builtin/packages/lmod/package.py @@ -34,17 +34,17 @@ class Lmod(Package): variable. Modulefiles for Library packages provide environment variables that specify where the library and header files can be found. """ - homepage = "https://www.tacc.utexas.edu/research-development/tacc-projects/lmod" - url = "http://sourceforge.net/projects/lmod/files/Lmod-6.0.1.tar.bz2/download" + homepage = 'https://www.tacc.utexas.edu/research-development/tacc-projects/lmod' + url = 'https://github.com/TACC/Lmod/archive/6.4.1.tar.gz' + version('6.4.1', '7978ba777c8aa41a4d8c05fec5f780f4') + version('6.3.7', '0fa4d5a24c41cae03776f781aa2dedc1', preferred=True) version('6.0.1', '91abf52fe5033bd419ffe2842ebe7af9') - depends_on("lua@5.2:") + depends_on('lua@5.2:') + depends_on('lua-luaposix') + depends_on('lua-luafilesystem') def install(self, spec, prefix): - # Add our lua to PATH - os.environ['PATH'] = spec['lua'].prefix.bin + ';' + os.environ['PATH'] - configure('--prefix=%s' % prefix) - make() - make("install") + make('install') diff --git a/var/spack/repos/builtin/packages/lua-luafilesystem/package.py b/var/spack/repos/builtin/packages/lua-luafilesystem/package.py new file mode 100644 index 0000000000..14278bbe72 --- /dev/null +++ b/var/spack/repos/builtin/packages/lua-luafilesystem/package.py @@ -0,0 +1,46 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class LuaLuafilesystem(Package): + """ + LuaFileSystem is a Lua library developed to complement the set of functions related to file + systems offered by the standard Lua distribution. + + LuaFileSystem offers a portable way to access the underlying directory structure and file attributes. LuaFileSystem + is free software and uses the same license as Lua 5.1 + """ + homepage = 'http://keplerproject.github.io/luafilesystem' + url = 'https://github.com/keplerproject/luafilesystem/archive/v_1_6_3.tar.gz' + + version('1_6_3', 'd0552c7e5a082f5bb2865af63fb9dc95') + + extends('lua') + + def install(self, spec, prefix): + version = self.spec.version + rockspec_format = join_path(self.stage.path, 'luafilesystem-v_{version.underscored}', 'rockspecs', 'luafilesystem-{version.dotted}-1.rockspec') + luarocks('--tree=' + prefix, 'install', rockspec_format.format(version=self.spec.version)) \ No newline at end of file diff --git a/var/spack/repos/builtin/packages/lua/package.py b/var/spack/repos/builtin/packages/lua/package.py index 170f90516a..320e25899e 100644 --- a/var/spack/repos/builtin/packages/lua/package.py +++ b/var/spack/repos/builtin/packages/lua/package.py @@ -105,6 +105,9 @@ def setup_dependent_environment(self, spack_env, run_env, extension_spec): spack_env.set('LUA_PATH', ';'.join(lua_patterns), separator=';') spack_env.set('LUA_CPATH', ';'.join(lua_cpatterns), separator=';') + # Add LUA to PATH for dependent packages + spack_env.prepend_path('PATH', self.prefix.bin) + # For run time environment set only the path for extension_spec and # prepend it to LUAPATH if extension_spec.package.extends(self.spec): @@ -153,5 +156,4 @@ def setup_dependent_package(self, module, ext_spec): """ # Lua extension builds can have lua and luarocks executable functions module.lua = Executable(join_path(self.spec.prefix.bin, 'lua')) - module.luarocks = Executable(join_path(self.spec.prefix.bin, - 'luarocks')) + module.luarocks = Executable(join_path(self.spec.prefix.bin, 'luarocks')) From 4a61efe641c9bf987a62bff729334c7cce3f141b Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 8 Jun 2016 16:21:09 +0200 Subject: [PATCH 320/481] lmod : fixed issues with LUA_PATH during installation --- var/spack/repos/builtin/packages/lmod/package.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/lmod/package.py b/var/spack/repos/builtin/packages/lmod/package.py index 9084c34881..e50c2cfc4b 100644 --- a/var/spack/repos/builtin/packages/lmod/package.py +++ b/var/spack/repos/builtin/packages/lmod/package.py @@ -23,7 +23,7 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## from spack import * -import os + class Lmod(Package): """ @@ -38,13 +38,19 @@ class Lmod(Package): url = 'https://github.com/TACC/Lmod/archive/6.4.1.tar.gz' version('6.4.1', '7978ba777c8aa41a4d8c05fec5f780f4') - version('6.3.7', '0fa4d5a24c41cae03776f781aa2dedc1', preferred=True) + version('6.3.7', '0fa4d5a24c41cae03776f781aa2dedc1') version('6.0.1', '91abf52fe5033bd419ffe2842ebe7af9') depends_on('lua@5.2:') depends_on('lua-luaposix') depends_on('lua-luafilesystem') + parallel = False + + def setup_environment(self, spack_env, run_env): + stage_lua_path = join_path(self.stage.path, 'Lmod-{version}', 'src', '?.lua') + spack_env.append_path('LUA_PATH', stage_lua_path.format(version=self.version), separator=';') + def install(self, spec, prefix): configure('--prefix=%s' % prefix) make('install') From 564483936f32abfd5ee79dea6263af4bd6e9bb9d Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 8 Jun 2016 16:33:49 +0200 Subject: [PATCH 321/481] qa : fixed flake8 issues --- lib/spack/spack/version.py | 139 ++++++------------ .../repos/builtin/packages/lmod/package.py | 8 +- .../packages/lua-luafilesystem/package.py | 19 ++- .../repos/builtin/packages/lua/package.py | 5 +- 4 files changed, 64 insertions(+), 107 deletions(-) diff --git a/lib/spack/spack/version.py b/lib/spack/spack/version.py index d86bcef3e0..858d581472 100644 --- a/lib/spack/spack/version.py +++ b/lib/spack/spack/version.py @@ -43,16 +43,16 @@ intersection concrete """ -import os -import sys import re from bisect import bisect_left from functools import wraps + from functools_backport import total_ordering # Valid version characters VALID_VERSION = r'[A-Za-z0-9_.-]' + def int_if_int(string): """Convert a string to int if possible. Otherwise, return a string.""" try: @@ -62,10 +62,11 @@ def int_if_int(string): def coerce_versions(a, b): - """Convert both a and b to the 'greatest' type between them, in this order: + """ + Convert both a and b to the 'greatest' type between them, in this order: Version < VersionRange < VersionList - This is used to simplify comparison operations below so that we're always - comparing things that are of the same type. + This is used to simplify comparison operations below so that we're always + comparing things that are of the same type. """ order = (Version, VersionRange, VersionList) ta, tb = type(a), type(b) @@ -105,6 +106,7 @@ def coercing_method(a, b, *args, **kwargs): @total_ordering class Version(object): """Class to represent versions""" + def __init__(self, string): string = str(string) @@ -142,15 +144,12 @@ def up_to(self, index): """ return '.'.join(str(x) for x in self[:index]) - def lowest(self): return self - def highest(self): return self - @coerced def satisfies(self, other): """A Version 'satisfies' another if it is at least as specific and has a @@ -158,11 +157,10 @@ def satisfies(self, other): gcc@4.7 so that when a user asks to build with gcc@4.7, we can find a suitable compiler. """ - nself = len(self.version) + nself = len(self.version) nother = len(other.version) return nother <= nself and self.version[:nother] == other.version - def wildcard(self): """Create a regex that will match variants of this version string.""" def a_or_n(seg): @@ -192,28 +190,22 @@ def a_or_n(seg): wc += '(?:[a-z]|alpha|beta)?)?' * (len(segments) - 1) return wc - def __iter__(self): return iter(self.version) - def __getitem__(self, idx): return tuple(self.version[idx]) - def __repr__(self): return self.string - def __str__(self): return self.string - @property def concrete(self): return self - @coerced def __lt__(self, other): """Version comparison is designed for consistency with the way RPM @@ -246,28 +238,23 @@ def __lt__(self, other): # If the common prefix is equal, the one with more segments is bigger. return len(self.version) < len(other.version) - @coerced def __eq__(self, other): return (other is not None and type(other) == Version and self.version == other.version) - def __ne__(self, other): return not (self == other) - def __hash__(self): return hash(self.version) - @coerced def __contains__(self, other): if other is None: return False return other.version[:len(self.version)] == self.version - def is_predecessor(self, other): """True if the other version is the immediate predecessor of this one. That is, NO versions v exist such that: @@ -280,16 +267,13 @@ def is_predecessor(self, other): ol = other.version[-1] return type(sl) == int and type(ol) == int and (ol - sl == 1) - def is_successor(self, other): return other.is_predecessor(self) - @coerced def overlaps(self, other): return self in other or other in self - @coerced def union(self, other): if self == other or other in self: @@ -299,7 +283,6 @@ def union(self, other): else: return VersionList([self, other]) - @coerced def intersection(self, other): if self == other: @@ -310,6 +293,7 @@ def intersection(self, other): @total_ordering class VersionRange(object): + def __init__(self, start, end): if isinstance(start, basestring): start = Version(start) @@ -321,15 +305,12 @@ def __init__(self, start, end): if start and end and end < start: raise ValueError("Invalid Version range: %s" % self) - def lowest(self): return self.start - def highest(self): return self.end - @coerced def __lt__(self, other): """Sort VersionRanges lexicographically so that they are ordered first @@ -342,28 +323,24 @@ def __lt__(self, other): s, o = self, other if s.start != o.start: - return s.start is None or (o.start is not None and s.start < o.start) + return s.start is None or (o.start is not None and s.start < o.start) # NOQA: ignore=E501 return (s.end != o.end and o.end is None or (s.end is not None and s.end < o.end)) - @coerced def __eq__(self, other): return (other is not None and type(other) == VersionRange and self.start == other.start and self.end == other.end) - def __ne__(self, other): return not (self == other) - @property def concrete(self): return self.start if self.start == self.end else None - @coerced def __contains__(self, other): if other is None: @@ -384,57 +361,55 @@ def __contains__(self, other): other.end in self.end))) return in_upper - @coerced def satisfies(self, other): - """A VersionRange satisfies another if some version in this range - would satisfy some version in the other range. To do this it must - either: - a) Overlap with the other range - b) The start of this range satisfies the end of the other range. + """ + A VersionRange satisfies another if some version in this range + would satisfy some version in the other range. To do this it must + either: + a) Overlap with the other range + b) The start of this range satisfies the end of the other range. - This is essentially the same as overlaps(), but overlaps assumes - that its arguments are specific. That is, 4.7 is interpreted as - 4.7.0.0.0.0... . This funciton assumes that 4.7 woudl be satisfied - by 4.7.3.5, etc. + This is essentially the same as overlaps(), but overlaps assumes + that its arguments are specific. That is, 4.7 is interpreted as + 4.7.0.0.0.0... . This funciton assumes that 4.7 woudl be satisfied + by 4.7.3.5, etc. - Rationale: - If a user asks for gcc@4.5:4.7, and a package is only compatible with - gcc@4.7.3:4.8, then that package should be able to build under the - constraints. Just using overlaps() would not work here. + Rationale: + If a user asks for gcc@4.5:4.7, and a package is only compatible with + gcc@4.7.3:4.8, then that package should be able to build under the + constraints. Just using overlaps() would not work here. - Note that we don't need to check whether the end of this range - would satisfy the start of the other range, because overlaps() - already covers that case. + Note that we don't need to check whether the end of this range + would satisfy the start of the other range, because overlaps() + already covers that case. - Note further that overlaps() is a symmetric operation, while - satisfies() is not. + Note further that overlaps() is a symmetric operation, while + satisfies() is not. """ return (self.overlaps(other) or # if either self.start or other.end are None, then this can't # satisfy, or overlaps() would've taken care of it. self.start and other.end and self.start.satisfies(other.end)) - @coerced def overlaps(self, other): - return ((self.start == None or other.end is None or + return ((self.start is None or other.end is None or self.start <= other.end or other.end in self.start or self.start in other.end) and - (other.start is None or self.end == None or + (other.start is None or self.end is None or other.start <= self.end or other.start in self.end or self.end in other.start)) - @coerced def union(self, other): if not self.overlaps(other): if (self.end is not None and other.start is not None and - self.end.is_predecessor(other.start)): + self.end.is_predecessor(other.start)): return VersionRange(self.start, other.end) if (other.end is not None and self.start is not None and - other.end.is_predecessor(self.start)): + other.end.is_predecessor(self.start)): return VersionRange(other.start, self.end) return VersionList([self, other]) @@ -453,13 +428,12 @@ def union(self, other): else: end = self.end # TODO: See note in intersection() about < and in discrepancy. - if not other.end in self.end: + if other.end not in self.end: if end in other.end or other.end > self.end: end = other.end return VersionRange(start, end) - @coerced def intersection(self, other): if self.overlaps(other): @@ -481,7 +455,7 @@ def intersection(self, other): # 1.6 < 1.6.5 = True (lexicographic) # Should 1.6 NOT be less than 1.6.5? Hm. # Here we test (not end in other.end) first to avoid paradox. - if other.end is not None and not end in other.end: + if other.end is not None and end not in other.end: if other.end < end or other.end in end: end = other.end @@ -490,15 +464,12 @@ def intersection(self, other): else: return VersionList() - def __hash__(self): return hash((self.start, self.end)) - def __repr__(self): return self.__str__() - def __str__(self): out = "" if self.start: @@ -512,6 +483,7 @@ def __str__(self): @total_ordering class VersionList(object): """Sorted, non-redundant list of Versions and VersionRanges.""" + def __init__(self, vlist=None): self.versions = [] if vlist is not None: @@ -526,7 +498,6 @@ def __init__(self, vlist=None): for v in vlist: self.add(ver(v)) - def add(self, version): if type(version) in (Version, VersionRange): # This normalizes single-value version ranges. @@ -535,9 +506,9 @@ def add(self, version): i = bisect_left(self, version) - while i-1 >= 0 and version.overlaps(self[i-1]): - version = version.union(self[i-1]) - del self.versions[i-1] + while i - 1 >= 0 and version.overlaps(self[i - 1]): + version = version.union(self[i - 1]) + del self.versions[i - 1] i -= 1 while i < len(self) and version.overlaps(self[i]): @@ -553,7 +524,6 @@ def add(self, version): else: raise TypeError("Can't add %s to VersionList" % type(version)) - @property def concrete(self): if len(self) == 1: @@ -561,11 +531,9 @@ def concrete(self): else: return None - def copy(self): return VersionList(self) - def lowest(self): """Get the lowest version in the list.""" if not self: @@ -573,7 +541,6 @@ def lowest(self): else: return self[0].lowest() - def highest(self): """Get the highest version in the list.""" if not self: @@ -581,7 +548,6 @@ def highest(self): else: return self[-1].highest() - @coerced def overlaps(self, other): if not other or not self: @@ -597,14 +563,12 @@ def overlaps(self, other): o += 1 return False - def to_dict(self): """Generate human-readable dict for YAML.""" if self.concrete: - return { 'version' : str(self[0]) } + return {'version': str(self[0])} else: - return { 'versions' : [str(v) for v in self] } - + return {'versions': [str(v) for v in self]} @staticmethod def from_dict(dictionary): @@ -616,7 +580,6 @@ def from_dict(dictionary): else: raise ValueError("Dict must have 'version' or 'versions' in it.") - @coerced def satisfies(self, other, strict=False): """A VersionList satisfies another if some version in the list @@ -644,20 +607,17 @@ def satisfies(self, other, strict=False): o += 1 return False - @coerced def update(self, other): for v in other.versions: self.add(v) - @coerced def union(self, other): result = self.copy() result.update(other) return result - @coerced def intersection(self, other): # TODO: make this faster. This is O(n^2). @@ -667,7 +627,6 @@ def intersection(self, other): result.add(s.intersection(o)) return result - @coerced def intersect(self, other): """Intersect this spec's list with other. @@ -689,50 +648,40 @@ def __contains__(self, other): if i == 0: if version not in self[0]: return False - elif all(version not in v for v in self[i-1:]): + elif all(version not in v for v in self[i - 1:]): return False return True - def __getitem__(self, index): return self.versions[index] - def __iter__(self): return iter(self.versions) - def __reversed__(self): return reversed(self.versions) - def __len__(self): return len(self.versions) - @coerced def __eq__(self, other): return other is not None and self.versions == other.versions - def __ne__(self, other): return not (self == other) - @coerced def __lt__(self, other): return other is not None and self.versions < other.versions - def __hash__(self): return hash(tuple(self.versions)) - def __str__(self): return ",".join(str(v) for v in self.versions) - def __repr__(self): return str(self.versions) @@ -741,7 +690,7 @@ def _string_to_version(string): """Converts a string to a Version, VersionList, or VersionRange. This is private. Client code should use ver(). """ - string = string.replace(' ','') + string = string.replace(' ', '') if ',' in string: return VersionList(string.split(',')) @@ -749,7 +698,7 @@ def _string_to_version(string): elif ':' in string: s, e = string.split(':') start = Version(s) if s else None - end = Version(e) if e else None + end = Version(e) if e else None return VersionRange(start, end) else: diff --git a/var/spack/repos/builtin/packages/lmod/package.py b/var/spack/repos/builtin/packages/lmod/package.py index e50c2cfc4b..7d75866d52 100644 --- a/var/spack/repos/builtin/packages/lmod/package.py +++ b/var/spack/repos/builtin/packages/lmod/package.py @@ -34,7 +34,7 @@ class Lmod(Package): variable. Modulefiles for Library packages provide environment variables that specify where the library and header files can be found. """ - homepage = 'https://www.tacc.utexas.edu/research-development/tacc-projects/lmod' + homepage = 'https://www.tacc.utexas.edu/research-development/tacc-projects/lmod' # NOQA: ignore=E501 url = 'https://github.com/TACC/Lmod/archive/6.4.1.tar.gz' version('6.4.1', '7978ba777c8aa41a4d8c05fec5f780f4') @@ -48,8 +48,10 @@ class Lmod(Package): parallel = False def setup_environment(self, spack_env, run_env): - stage_lua_path = join_path(self.stage.path, 'Lmod-{version}', 'src', '?.lua') - spack_env.append_path('LUA_PATH', stage_lua_path.format(version=self.version), separator=';') + stage_lua_path = join_path( + self.stage.path, 'Lmod-{version}', 'src', '?.lua') + spack_env.append_path('LUA_PATH', stage_lua_path.format( + version=self.version), separator=';') def install(self, spec, prefix): configure('--prefix=%s' % prefix) diff --git a/var/spack/repos/builtin/packages/lua-luafilesystem/package.py b/var/spack/repos/builtin/packages/lua-luafilesystem/package.py index 14278bbe72..a61b9dd675 100644 --- a/var/spack/repos/builtin/packages/lua-luafilesystem/package.py +++ b/var/spack/repos/builtin/packages/lua-luafilesystem/package.py @@ -27,11 +27,13 @@ class LuaLuafilesystem(Package): """ - LuaFileSystem is a Lua library developed to complement the set of functions related to file - systems offered by the standard Lua distribution. + LuaFileSystem is a Lua library developed to complement the set of + functions related to file systems offered by the standard Lua distribution. - LuaFileSystem offers a portable way to access the underlying directory structure and file attributes. LuaFileSystem - is free software and uses the same license as Lua 5.1 + LuaFileSystem offers a portable way to access the underlying directory + structure and file attributes. + + LuaFileSystem is free software and uses the same license as Lua 5.1 """ homepage = 'http://keplerproject.github.io/luafilesystem' url = 'https://github.com/keplerproject/luafilesystem/archive/v_1_6_3.tar.gz' @@ -41,6 +43,9 @@ class LuaLuafilesystem(Package): extends('lua') def install(self, spec, prefix): - version = self.spec.version - rockspec_format = join_path(self.stage.path, 'luafilesystem-v_{version.underscored}', 'rockspecs', 'luafilesystem-{version.dotted}-1.rockspec') - luarocks('--tree=' + prefix, 'install', rockspec_format.format(version=self.spec.version)) \ No newline at end of file + rockspec_fmt = join_path(self.stage.path, + 'luafilesystem-v_{version.underscored}', + 'rockspecs', + 'luafilesystem-{version.dotted}-1.rockspec') + luarocks('--tree=' + prefix, 'install', + rockspec_fmt.format(version=self.spec.version)) diff --git a/var/spack/repos/builtin/packages/lua/package.py b/var/spack/repos/builtin/packages/lua/package.py index 320e25899e..8ce4a7b94d 100644 --- a/var/spack/repos/builtin/packages/lua/package.py +++ b/var/spack/repos/builtin/packages/lua/package.py @@ -57,7 +57,7 @@ class Lua(Package): placement='luarocks') def install(self, spec, prefix): - if spec.satisfies("arch=darwin-i686") or spec.satisfies("arch=darwin-x86_64"): + if spec.satisfies("arch=darwin-i686") or spec.satisfies("arch=darwin-x86_64"): # NOQA: ignore=E501 target = 'macosx' else: target = 'linux' @@ -156,4 +156,5 @@ def setup_dependent_package(self, module, ext_spec): """ # Lua extension builds can have lua and luarocks executable functions module.lua = Executable(join_path(self.spec.prefix.bin, 'lua')) - module.luarocks = Executable(join_path(self.spec.prefix.bin, 'luarocks')) + module.luarocks = Executable( + join_path(self.spec.prefix.bin, 'luarocks')) From 2d21693b9d98219db29eac4ceb9163f32aaf51e4 Mon Sep 17 00:00:00 2001 From: Nicolas Richart Date: Wed, 8 Jun 2016 17:09:13 +0200 Subject: [PATCH 322/481] Missing file to patch mumps --- .../builtin/packages/mumps/mumps-shared.patch | 119 ++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 var/spack/repos/builtin/packages/mumps/mumps-shared.patch diff --git a/var/spack/repos/builtin/packages/mumps/mumps-shared.patch b/var/spack/repos/builtin/packages/mumps/mumps-shared.patch new file mode 100644 index 0000000000..592f48037c --- /dev/null +++ b/var/spack/repos/builtin/packages/mumps/mumps-shared.patch @@ -0,0 +1,119 @@ +diff -Naur MUMPS_5.0.1/libseq/Makefile MUMPS_5.0.1.new/libseq/Makefile +--- MUMPS_5.0.1/libseq/Makefile 2015-07-23 19:08:32.000000000 +0200 ++++ MUMPS_5.0.1.new/libseq/Makefile 2016-06-07 10:41:16.585179151 +0200 +@@ -8,11 +8,15 @@ + + include ../Makefile.inc + +-libmpiseq: libmpiseq$(PLAT)$(LIBEXT) ++libmpiseq: libmpiseq$(PLAT)$(LIBEXT) libmpiseq$(PLAT)$(SHLIBEXT) + + libmpiseq$(PLAT)$(LIBEXT): mpi.o mpic.o elapse.o + $(AR)$@ mpi.o mpic.o elapse.o + $(RANLIB) $@ ++ ++libmpiseq$(PLAT)$(SHLIBEXT): mpi.o mpic.o elapse.o ++ $(FC) $(LDFLAGS) $^ -o libmpiseq$(PLAT)$(SHLIBEXT) ++ + .f.o: + $(FC) $(OPTF) -c $*.f $(OUTF)$*.o + .c.o: +diff -Naur MUMPS_5.0.1/Makefile MUMPS_5.0.1.new/Makefile +--- MUMPS_5.0.1/Makefile 2015-07-23 19:08:29.000000000 +0200 ++++ MUMPS_5.0.1.new/Makefile 2016-06-07 10:50:21.863281217 +0200 +@@ -51,7 +51,7 @@ + dexamples: d + (cd examples ; $(MAKE) d) + +-requiredobj: Makefile.inc $(LIBSEQNEEDED) $(libdir)/libpord$(PLAT)$(LIBEXT) ++requiredobj: Makefile.inc $(LIBSEQNEEDED) $(libdir)/libpord$(PLAT)$(LIBEXT) $(libdir)/libpord$(PLAT)$(SHLIBEXT) + + # dummy MPI library (sequential version) + +@@ -62,16 +62,25 @@ + $(libdir)/libpord$(PLAT)$(LIBEXT): + if [ "$(LPORDDIR)" != "" ] ; then \ + cd $(LPORDDIR); \ +- $(MAKE) CC="$(CC)" CFLAGS="$(OPTC)" AR="$(AR)" RANLIB="$(RANLIB)" OUTC="$(OUTC)" LIBEXT=$(LIBEXT); \ ++ $(MAKE) CC="$(CC)" CFLAGS="$(OPTC)" AR="$(AR)" RANLIB="$(RANLIB)" LDFLAGS="$(LDFLAGS)" OUTC="$(OUTC)" LIBEXT=$(LIBEXT) PLAT=$(PLAT) SHLIBEXT=$(SHLIBEXT); \ + fi; + if [ "$(LPORDDIR)" != "" ] ; then \ + cp $(LPORDDIR)/libpord$(LIBEXT) $@; \ + fi; + ++$(libdir)/libpord$(PLAT)$(SHLIBEXT): ++ if [ "$(LPORDDIR)" != "" ] ; then \ ++ cd $(LPORDDIR); \ ++ $(MAKE) CC="$(CC)" CFLAGS="$(OPTC)" AR="$(AR)" RANLIB="$(RANLIB)" LDFLAGS="$(LDFLAGS)" OUTC="$(OUTC)" LIBEXT=$(LIBEXT) PLAT=$(PLAT) SHLIBEXT=$(SHLIBEXT) libpord$(PLAT)$(SHLIBEXT); \ ++ fi; ++ if [ "$(LPORDDIR)" != "" ] ; then \ ++ cp $(LPORDDIR)/libpord$(PLAT)$(SHLIBEXT) $@; \ ++ fi; ++ + clean: + (cd src; $(MAKE) clean) + (cd examples; $(MAKE) clean) +- (cd $(libdir); $(RM) *$(PLAT)$(LIBEXT)) ++ (cd $(libdir); $(RM) *$(PLAT)$(LIBEXT) *$(PLAT)$(SHLIBEXT)) + (cd libseq; $(MAKE) clean) + if [ "$(LPORDDIR)" != "" ] ; then \ + cd $(LPORDDIR); $(MAKE) realclean; \ +diff -Naur MUMPS_5.0.1/PORD/lib/Makefile MUMPS_5.0.1.new/PORD/lib/Makefile +--- MUMPS_5.0.1/PORD/lib/Makefile 2015-07-23 19:08:29.000000000 +0200 ++++ MUMPS_5.0.1.new/PORD/lib/Makefile 2016-06-07 10:49:48.889000958 +0200 +@@ -13,7 +13,7 @@ + + OBJS = graph.o gbipart.o gbisect.o ddcreate.o ddbisect.o nestdiss.o \ + multisector.o gelim.o bucket.o tree.o \ +- symbfac.o interface.o sort.o minpriority.o ++ symbfac.o interface.o sort.o minpriority.o + + # Note: numfac.c read.c mapping.c triangular.c matrix.c kernel.c + # were not direcly used by MUMPS and have been removed from the +@@ -24,12 +24,15 @@ + .c.o: + $(CC) $(COPTIONS) -c $*.c $(OUTC)$*.o + +-libpord$(LIBEXT):$(OBJS) ++libpord$(PLAT)$(LIBEXT):$(OBJS) + $(AR)$@ $(OBJS) + $(RANLIB) $@ + ++libpord$(PLAT)$(SHLIBEXT): $(OBJS) ++ $(CC) $(LDFLAGS) $(OBJS) -o libpord$(PLAT)$(SHLIBEXT) ++ + clean: + rm -f *.o + + realclean: +- rm -f *.o libpord.a ++ rm -f *.o libpord$(PLAT)$(SHLIBEXT) libpord$(PLAT)$(LIBEXT) +diff -Naur MUMPS_5.0.1/src/Makefile MUMPS_5.0.1.new/src/Makefile +--- MUMPS_5.0.1/src/Makefile 2015-07-23 19:08:29.000000000 +0200 ++++ MUMPS_5.0.1.new/src/Makefile 2016-06-07 10:40:52.534703722 +0200 +@@ -24,7 +24,10 @@ + include $(topdir)/Makefile.inc + + mumps_lib: $(libdir)/libmumps_common$(PLAT)$(LIBEXT) \ +- $(libdir)/lib$(ARITH)mumps$(PLAT)$(LIBEXT) ++ $(libdir)/libmumps_common$(PLAT)$(SHLIBEXT) \ ++ $(libdir)/lib$(ARITH)mumps$(PLAT)$(LIBEXT) \ ++ $(libdir)/lib$(ARITH)mumps$(PLAT)$(SHLIBEXT) ++ + + OBJS_COMMON_MOD = \ + ana_omp_m.o\ +@@ -162,6 +165,13 @@ + $(AR)$@ $? + $(RANLIB) $@ + ++$(libdir)/libmumps_common$(PLAT)$(SHLIBEXT): $(OBJS_COMMON_MOD) $(OBJS_COMMON_OTHER) ++ $(FC) $(LDFLAGS) $^ -L$(libdir) $(LORDERINGS) $(LIBS) $(LIBBLAS) $(LIBOTHERS) -o $(libdir)/libmumps_common$(PLAT)$(SHLIBEXT) ++ ++ ++$(libdir)/lib$(ARITH)mumps$(PLAT)$(SHLIBEXT): $(OBJS_MOD) $(OBJS_OTHER) ++ $(FC) $(LDFLAGS) $^ -L$(libdir) -lmumps_common$(PLAT) $(LORDERINGS) $(LIBS) $(LIBBLAS) $(LIBOTHERS) -o $(libdir)/lib$(ARITH)mumps$(PLAT)$(SHLIBEXT) ++ + # Dependencies between modules: + $(ARITH)mumps_load.o: $(ARITH)mumps_comm_buffer.o \ + $(ARITH)mumps_struc_def.o \ From 42a10d56905727529065d2c492f4c8dc9e4f73cb Mon Sep 17 00:00:00 2001 From: James Riley Wynne III Date: Wed, 8 Jun 2016 12:43:05 -0400 Subject: [PATCH 323/481] Fixes issue #438 --- var/spack/repos/builtin/packages/boost/package.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/var/spack/repos/builtin/packages/boost/package.py b/var/spack/repos/builtin/packages/boost/package.py index 2f2965eb12..6da86d6da4 100644 --- a/var/spack/repos/builtin/packages/boost/package.py +++ b/var/spack/repos/builtin/packages/boost/package.py @@ -109,6 +109,7 @@ class Boost(Package): variant('multithreaded', default=True, description="Build multi-threaded versions of libraries") variant('singlethreaded', default=True, description="Build single-threaded versions of libraries") variant('icu_support', default=False, description="Include ICU support (for regex/locale libraries)") + variant('graph', default=False, description="Build the Boost Graph library") depends_on('icu', when='+icu_support') depends_on('python', when='+python') @@ -235,6 +236,10 @@ def install(self, spec, prefix): withLibs.remove('chrono') if not spec.satisfies('@1.43.0:'): withLibs.remove('random') + + if '+graph' in spec and '+mpi' in spec: + withLibs.remove('graph') + withLibs.append('graph_parallel') # to make Boost find the user-config.jam env['BOOST_BUILD_PATH'] = './' From 4b9707f8ea3ef947239f6f269aa57849295dd294 Mon Sep 17 00:00:00 2001 From: James Riley Wynne III Date: Wed, 8 Jun 2016 14:36:45 -0400 Subject: [PATCH 324/481] Fixed flake8 formatting problems --- .../repos/builtin/packages/boost/package.py | 74 ++++++++++--------- 1 file changed, 39 insertions(+), 35 deletions(-) diff --git a/var/spack/repos/builtin/packages/boost/package.py b/var/spack/repos/builtin/packages/boost/package.py index 6da86d6da4..8a49672824 100644 --- a/var/spack/repos/builtin/packages/boost/package.py +++ b/var/spack/repos/builtin/packages/boost/package.py @@ -27,7 +27,7 @@ import sys import os -import sys + class Boost(Package): """Boost provides free peer-reviewed portable C++ source @@ -75,23 +75,23 @@ class Boost(Package): version('1.34.0', 'ed5b9291ffad776f8757a916e1726ad0') default_install_libs = set(['atomic', - 'chrono', - 'date_time', - 'filesystem', - 'graph', - 'iostreams', - 'locale', - 'log', - 'math', - 'program_options', - 'random', - 'regex', - 'serialization', - 'signals', - 'system', - 'test', - 'thread', - 'wave']) + 'chrono', + 'date_time', + 'filesystem', + 'graph', + 'iostreams', + 'locale', + 'log', + 'math', + 'program_options', + 'random', + 'regex', + 'serialization', + 'signals', + 'system', + 'test', + 'thread', + 'wave']) # mpi/python are not installed by default because they pull in many # dependencies and/or because there is a great deal of customization @@ -121,12 +121,15 @@ class Boost(Package): patch('boost_11856.patch', when='@1.60.0%gcc@4.4.7') def url_for_version(self, version): - """Handle Boost's weird URLs, which write the version two different ways.""" + """ + Handle Boost's weird URLs, + which write the version two different ways. + """ parts = [str(p) for p in Version(version)] dots = ".".join(parts) underscores = "_".join(parts) - return "http://downloads.sourceforge.net/project/boost/boost/%s/boost_%s.tar.bz2" % ( - dots, underscores) + return "http://downloads.sourceforge.net/project/boost" \ + "/boost/%s/boost_%s.tar.bz2" % (dots, underscores) def determine_toolset(self, spec): if spec.satisfies("arch=darwin-x86_64"): @@ -150,20 +153,20 @@ def determine_bootstrap_options(self, spec, withLibs, options): if '+python' in spec: options.append('--with-python=%s' % - join_path(spec['python'].prefix.bin, 'python')) + join_path(spec['python'].prefix.bin, 'python')) with open('user-config.jam', 'w') as f: compiler_wrapper = join_path(spack.build_env_path, 'c++') f.write("using {0} : : {1} ;\n".format(boostToolsetId, - compiler_wrapper)) + compiler_wrapper)) if '+mpi' in spec: f.write('using mpi : %s ;\n' % - join_path(spec['mpi'].prefix.bin, 'mpicxx')) + join_path(spec['mpi'].prefix.bin, 'mpicxx')) if '+python' in spec: f.write('using python : %s : %s ;\n' % - (spec['python'].version, - join_path(spec['python'].prefix.bin, 'python'))) + (spec['python'].version, + join_path(spec['python'].prefix.bin, 'python'))) def determine_b2_options(self, spec, options): if '+debug' in spec: @@ -179,8 +182,7 @@ def determine_b2_options(self, spec, options): '-s', 'BZIP2_INCLUDE=%s' % spec['bzip2'].prefix.include, '-s', 'BZIP2_LIBPATH=%s' % spec['bzip2'].prefix.lib, '-s', 'ZLIB_INCLUDE=%s' % spec['zlib'].prefix.include, - '-s', 'ZLIB_LIBPATH=%s' % spec['zlib'].prefix.lib, - ]) + '-s', 'ZLIB_LIBPATH=%s' % spec['zlib'].prefix.lib]) linkTypes = ['static'] if '+shared' in spec: @@ -192,7 +194,8 @@ def determine_b2_options(self, spec, options): if '+singlethreaded' in spec: threadingOpts.append('single') if not threadingOpts: - raise RuntimeError("At least one of {singlethreaded, multithreaded} must be enabled") + raise RuntimeError("""At least one of {singlethreaded, + multithreaded} must be enabled""") options.extend([ 'toolset=%s' % self.determine_toolset(spec), @@ -203,9 +206,9 @@ def determine_b2_options(self, spec, options): def install(self, spec, prefix): # On Darwin, Boost expects the Darwin libtool. However, one of the - # dependencies may have pulled in Spack's GNU libtool, and these two are - # not compatible. We thus create a symlink to Darwin's libtool and add - # it at the beginning of PATH. + # dependencies may have pulled in Spack's GNU libtool, and these two + # are not compatible. We thus create a symlink to Darwin's libtool + # and add it at the beginning of PATH. if sys.platform == 'darwin': newdir = os.path.abspath('darwin-libtool') mkdirp(newdir) @@ -218,7 +221,8 @@ def install(self, spec, prefix): withLibs.append(lib) if not withLibs: # if no libraries are specified for compilation, then you dont have - # to configure/build anything, just copy over to the prefix directory. + # to configure/build anything, just copy over to the prefix + # directory. src = join_path(self.stage.source_path, 'boost') mkdirp(join_path(prefix, 'include')) dst = join_path(prefix, 'include', 'boost') @@ -236,7 +240,6 @@ def install(self, spec, prefix): withLibs.remove('chrono') if not spec.satisfies('@1.43.0:'): withLibs.remove('random') - if '+graph' in spec and '+mpi' in spec: withLibs.remove('graph') withLibs.append('graph_parallel') @@ -264,6 +267,7 @@ def install(self, spec, prefix): for threadingOpt in threadingOpts: b2('install', 'threading=%s' % threadingOpt, *b2_options) - # The shared libraries are not installed correctly on Darwin; correct this + # The shared libraries are not installed correctly + # on Darwin; correct this if (sys.platform == 'darwin') and ('+shared' in spec): fix_darwin_install_name(prefix.lib) From 65896ff2ed809776e54f012a622b13a450c64ea6 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Wed, 8 Jun 2016 14:11:14 -0500 Subject: [PATCH 325/481] Add Armadillo and SuperLU packages --- .../builtin/packages/armadillo/package.py | 67 +++++++++++++++++++ .../repos/builtin/packages/arpack/package.py | 40 ++++++----- .../repos/builtin/packages/superlu/package.py | 55 +++++++++++++++ 3 files changed, 146 insertions(+), 16 deletions(-) create mode 100644 var/spack/repos/builtin/packages/armadillo/package.py create mode 100644 var/spack/repos/builtin/packages/superlu/package.py diff --git a/var/spack/repos/builtin/packages/armadillo/package.py b/var/spack/repos/builtin/packages/armadillo/package.py new file mode 100644 index 0000000000..b3e5994e30 --- /dev/null +++ b/var/spack/repos/builtin/packages/armadillo/package.py @@ -0,0 +1,67 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class Armadillo(Package): + """Armadillo is a high quality linear algebra library (matrix maths) + for the C++ language, aiming towards a good balance between speed and + ease of use.""" + + homepage = "http://arma.sourceforge.net/" + url = "http://sourceforge.net/projects/arma/files/armadillo-7.200.1.tar.xz" + + version('7.200.1', 'ed86d6df0058979e107502e1fe3e469e') + + variant('hdf5', default=False, description='Include HDF5 support') + + depends_on('arpack') + depends_on('blas') + depends_on('lapack') + depends_on('superlu@5.2:') + depends_on('hdf5', when='+hdf5') + + def install(self, spec, prefix): + cmake_args = [ + # ARPACK support + '-DARPACK_LIBRARY={0}/libarpack.a'.format( + spec['arpack'].prefix.lib), + # BLAS support + '-DBLAS_LIBRARY={0}'.format(spec['blas'].blas_shared_lib), + # LAPACK support + '-DLAPACK_LIBRARY={0}'.format(spec['lapack'].lapack_shared_lib), + # SuperLU support + '-DSuperLU_INCLUDE_DIR={0}'.format(spec['superlu'].prefix.include), + '-DSuperLU_LIBRARY={0}/libsuperlu.a'.format( + spec['superlu'].prefix.lib64), + # HDF5 support + '-DDETECT_HDF5={0}'.format('ON' if '+hdf5' in spec else 'OFF') + ] + + cmake_args.extend(std_cmake_args) + cmake('.', *cmake_args) + + make() + make('install') diff --git a/var/spack/repos/builtin/packages/arpack/package.py b/var/spack/repos/builtin/packages/arpack/package.py index 75158776fe..91b5f06a4a 100644 --- a/var/spack/repos/builtin/packages/arpack/package.py +++ b/var/spack/repos/builtin/packages/arpack/package.py @@ -24,12 +24,12 @@ ############################################################################## from spack import * import os -import shutil + class Arpack(Package): """A collection of Fortran77 subroutines designed to solve large scale - eigenvalue problems. - """ + eigenvalue problems.""" + homepage = "http://www.caam.rice.edu/software/ARPACK/" url = "http://www.caam.rice.edu/software/ARPACK/SRC/arpack96.tar.gz" @@ -39,27 +39,35 @@ class Arpack(Package): depends_on('lapack') def patch(self): - # Filter the cray makefile to make a spack one. - shutil.move('ARMAKES/ARmake.CRAY', 'ARmake.inc') makefile = FileFilter('ARmake.inc') - # Be sure to use Spack F77 wrapper - makefile.filter('^FC.*', 'FC = f77') - makefile.filter('^FFLAGS.*', 'FFLAGS = -O2 -g') + # Section 1: Paths and Libraries - # Set up some variables. - makefile.filter('^PLAT.*', 'PLAT = ') - makefile.filter('^home.*', 'home = %s' % os.getcwd()) - makefile.filter('^BLASdir.*', 'BLASdir = %s' % self.spec['blas'].prefix) - makefile.filter('^LAPACKdir.*', 'LAPACKdir = %s' % self.spec['lapack'].prefix) + # Change the build directory + makefile.filter('^home.*', 'home = %s' % os.getcwd()) - # build the library in our own prefix. - makefile.filter('^ARPACKLIB.*', 'ARPACKLIB = %s/libarpack.a' % os.getcwd()) + # Use external BLAS/LAPACK + makefile.filter('^BLASdir.*', + 'BLASdir = %s' % self.spec['blas'].prefix) + makefile.filter('^LAPACKdir.*', + 'LAPACKdir = %s' % self.spec['lapack'].prefix) + # Do not include the platform in the library name + makefile.filter('^PLAT.*', 'PLAT = ') + makefile.filter('^ARPACKLIB.*', 'ARPACKLIB = $(home)/libarpack.a') + + # Section 2: Compilers + + # Be sure to use the Spack compiler wrapper + makefile.filter('^FC.*', 'FC = {0}'.format(os.environ['F77'])) + makefile.filter('^FFLAGS.*', 'FFLAGS = -O2 -g -fPIC') + + if not which('ranlib'): + makefile.filter('^RANLIB.*', 'RANLIB = touch') def install(self, spec, prefix): with working_dir('SRC'): make('all') - mkdirp(prefix.lib) + mkdir(prefix.lib) install('libarpack.a', prefix.lib) diff --git a/var/spack/repos/builtin/packages/superlu/package.py b/var/spack/repos/builtin/packages/superlu/package.py new file mode 100644 index 0000000000..91e8693924 --- /dev/null +++ b/var/spack/repos/builtin/packages/superlu/package.py @@ -0,0 +1,55 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class Superlu(Package): + """SuperLU is a general purpose library for the direct solution of large, + sparse, nonsymmetric systems of linear equations on high performance + machines. SuperLU is designed for sequential machines.""" + + homepage = "http://crd-legacy.lbl.gov/~xiaoye/SuperLU/#superlu" + url = "http://crd-legacy.lbl.gov/~xiaoye/SuperLU/superlu_5.2.1.tar.gz" + + version('5.2.1', '3a1a9bff20cb06b7d97c46d337504447') + + depends_on('blas') + + def install(self, spec, prefix): + cmake_args = [ + '-DCMAKE_C_FLAGS=-fPIC', + '-DCMAKE_Fortran_FLAGS=-fPIC', + # BLAS support + '-Denable_blaslib=OFF', + '-DBLAS_blas_LIBRARY={0}'.format(spec['blas'].blas_shared_lib) + ] + + cmake_args.extend(std_cmake_args) + + with working_dir('spack-build', create=True): + cmake('..', *cmake_args) + + make() + make('install') From e3ca5e66765d06eca9ec40fed215bd82c4bdd49c Mon Sep 17 00:00:00 2001 From: Orion Poplawski Date: Wed, 8 Jun 2016 16:23:04 -0600 Subject: [PATCH 326/481] Add patch to compile pcre with Intel 16.0.3 compiler --- var/spack/repos/builtin/packages/pcre/intel.patch | 12 ++++++++++++ var/spack/repos/builtin/packages/pcre/package.py | 2 ++ 2 files changed, 14 insertions(+) create mode 100644 var/spack/repos/builtin/packages/pcre/intel.patch diff --git a/var/spack/repos/builtin/packages/pcre/intel.patch b/var/spack/repos/builtin/packages/pcre/intel.patch new file mode 100644 index 0000000000..f160f55e1b --- /dev/null +++ b/var/spack/repos/builtin/packages/pcre/intel.patch @@ -0,0 +1,12 @@ +diff -up pcre-8.38/pcrecpp.cc.intel pcre-8.38/pcrecpp.cc +--- pcre-8.38/pcrecpp.cc.intel 2014-09-15 07:48:59.000000000 -0600 ++++ pcre-8.38/pcrecpp.cc 2016-06-08 16:16:56.702721214 -0600 +@@ -66,7 +66,7 @@ Arg RE::no_arg((void*)NULL); + // inclusive test if we ever needed it. (Note that not only the + // __attribute__ syntax, but also __USER_LABEL_PREFIX__, are + // gnu-specific.) +-#if defined(__GNUC__) && __GNUC__ >= 3 && defined(__ELF__) ++#if defined(__GNUC__) && __GNUC__ >= 3 && defined(__ELF__) && !defined(__INTEL_COMPILER) + # define ULP_AS_STRING(x) ULP_AS_STRING_INTERNAL(x) + # define ULP_AS_STRING_INTERNAL(x) #x + # define USER_LABEL_PREFIX_STR ULP_AS_STRING(__USER_LABEL_PREFIX__) diff --git a/var/spack/repos/builtin/packages/pcre/package.py b/var/spack/repos/builtin/packages/pcre/package.py index 7a9f3b911d..b75a50736a 100644 --- a/var/spack/repos/builtin/packages/pcre/package.py +++ b/var/spack/repos/builtin/packages/pcre/package.py @@ -34,6 +34,8 @@ class Pcre(Package): version('8.36', 'b767bc9af0c20bc9c1fe403b0d41ad97') version('8.38', '00aabbfe56d5a48b270f999b508c5ad2') + patch("intel.patch") + def install(self, spec, prefix): configure("--prefix=%s" % prefix) make() From 454acf58af0ea22b6f14f7527226234303d40e99 Mon Sep 17 00:00:00 2001 From: Orion Poplawski Date: Wed, 8 Jun 2016 16:49:42 -0600 Subject: [PATCH 327/481] Add additional blank line for flake8 --- var/spack/repos/builtin/packages/pcre/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/pcre/package.py b/var/spack/repos/builtin/packages/pcre/package.py index b75a50736a..8e0f83110e 100644 --- a/var/spack/repos/builtin/packages/pcre/package.py +++ b/var/spack/repos/builtin/packages/pcre/package.py @@ -24,6 +24,7 @@ ############################################################################## from spack import * + class Pcre(Package): """The PCRE package contains Perl Compatible Regular Expression libraries. These are useful for implementing regular expression From 2a4d440003c467f34b072d3128ce4869dc5cd5d2 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Wed, 8 Jun 2016 19:14:01 -0400 Subject: [PATCH 328/481] Add quotes around file name $input_log will expand to file names with special characters (e.g. "@"), thus bash requires quotes. --- lib/spack/env/cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index 9758b74f37..a604785904 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -324,8 +324,8 @@ fi if [[ $SPACK_DEBUG == TRUE ]]; then input_log="$SPACK_DEBUG_LOG_DIR/spack-cc-$SPACK_SHORT_SPEC.in.log" output_log="$SPACK_DEBUG_LOG_DIR/spack-cc-$SPACK_SHORT_SPEC.out.log" - echo "[$mode] $command $input_command" >> $input_log - echo "[$mode] ${full_command[@]}" >> $output_log + echo "[$mode] $command $input_command" >> "$input_log" + echo "[$mode] ${full_command[@]}" >> "$output_log" fi exec "${full_command[@]}" From 915115386af09583dc7dfff094b259990179bb4a Mon Sep 17 00:00:00 2001 From: "Kelly (KT) Thompson" Date: Thu, 9 Jun 2016 18:30:28 -0600 Subject: [PATCH 329/481] Package serf also depends on zlib. + Add zlib as a required dependency. + Point scons build system to the spack provided zlib installation. --- var/spack/repos/builtin/packages/serf/package.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/serf/package.py b/var/spack/repos/builtin/packages/serf/package.py index 3b1d08889c..7ca72096aa 100644 --- a/var/spack/repos/builtin/packages/serf/package.py +++ b/var/spack/repos/builtin/packages/serf/package.py @@ -36,6 +36,7 @@ class Serf(Package): depends_on('scons') depends_on('expat') depends_on('openssl') + depends_on('zlib') def install(self, spec, prefix): scons = which("scons") @@ -44,8 +45,10 @@ def install(self, spec, prefix): options.append('APR=%s' % spec['apr'].prefix) options.append('APU=%s' % spec['apr-util'].prefix) options.append('OPENSSL=%s' % spec['openssl'].prefix) - options.append('LINKFLAGS=-L%s/lib' % spec['expat'].prefix) - options.append('CPPFLAGS=-I%s/include' % spec['expat'].prefix) + options.append('LINKFLAGS=-L%s/lib -L%s/lib' % + ( spec['expat'].prefix, spec['zlib'].prefix )) + options.append('CPPFLAGS=-I%s/include -I%s/include' % + ( spec['expat'].prefix, spec['zlib'].prefix )) scons(*options) scons('install') From 883f601bca7c6c8f26937b295dadfd2f6a4f023e Mon Sep 17 00:00:00 2001 From: "Kelly (KT) Thompson" Date: Thu, 9 Jun 2016 19:18:25 -0600 Subject: [PATCH 330/481] Fix style issues. --- var/spack/repos/builtin/packages/serf/package.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/var/spack/repos/builtin/packages/serf/package.py b/var/spack/repos/builtin/packages/serf/package.py index 7ca72096aa..817db68241 100644 --- a/var/spack/repos/builtin/packages/serf/package.py +++ b/var/spack/repos/builtin/packages/serf/package.py @@ -24,8 +24,10 @@ ############################################################################## from spack import * + class Serf(Package): - """Apache Serf - a high performance C-based HTTP client library built upon the Apache Portable Runtime (APR) library""" + """Apache Serf - a high performance C-based HTTP client library + built upon the Apache Portable Runtime (APR) library""" homepage = 'https://serf.apache.org/' url = 'https://archive.apache.org/dist/serf/serf-1.3.8.tar.bz2' @@ -46,9 +48,9 @@ def install(self, spec, prefix): options.append('APU=%s' % spec['apr-util'].prefix) options.append('OPENSSL=%s' % spec['openssl'].prefix) options.append('LINKFLAGS=-L%s/lib -L%s/lib' % - ( spec['expat'].prefix, spec['zlib'].prefix )) + (spec['expat'].prefix, spec['zlib'].prefix)) options.append('CPPFLAGS=-I%s/include -I%s/include' % - ( spec['expat'].prefix, spec['zlib'].prefix )) + (spec['expat'].prefix, spec['zlib'].prefix)) scons(*options) scons('install') From 85502e34f9d8a0a91267247d9b72287012befd0f Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Fri, 10 Jun 2016 14:35:19 -0500 Subject: [PATCH 331/481] CMake knows how to create PIC --- var/spack/repos/builtin/packages/superlu/package.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/superlu/package.py b/var/spack/repos/builtin/packages/superlu/package.py index 91e8693924..c634c1d1ba 100644 --- a/var/spack/repos/builtin/packages/superlu/package.py +++ b/var/spack/repos/builtin/packages/superlu/package.py @@ -39,8 +39,7 @@ class Superlu(Package): def install(self, spec, prefix): cmake_args = [ - '-DCMAKE_C_FLAGS=-fPIC', - '-DCMAKE_Fortran_FLAGS=-fPIC', + '-DCMAKE_POSITION_INDEPENDENT_CODE=ON', # BLAS support '-Denable_blaslib=OFF', '-DBLAS_blas_LIBRARY={0}'.format(spec['blas'].blas_shared_lib) From 992bcac7949883b74a9250d5835ac6d9de28a6c0 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 10 Jun 2016 16:26:02 -0700 Subject: [PATCH 332/481] changed compiler yaml format --- lib/spack/spack/compilers/__init__.py | 34 ++++----- lib/spack/spack/config.py | 24 +++---- lib/spack/spack/test/config.py | 82 +++++++++------------- lib/spack/spack/test/mock_packages_test.py | 72 +++++++------------ lib/spack/spack/yaml_version_check.py | 4 +- 5 files changed, 86 insertions(+), 130 deletions(-) diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index d923d77da8..3cf8f2297f 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -73,16 +73,14 @@ def _to_dict(compiler): d = {} d['spec'] = str(compiler.spec) d['paths'] = dict( (attr, getattr(compiler, attr, None)) for attr in _path_instance_vars ) - d['operating_system'] = compiler.operating_system.to_dict() + d['operating_system'] = str(compiler.operating_system) d['modules'] = compiler.modules - if not compiler.alias: - yaml_text = yaml.dump( - d, default_flow_style=True, width=sys.maxint) - sha = hashlib.sha1(yaml_text) - compiler.alias = base64.b32encode(sha.digest()).lower()[:8] + if compiler.alias: + d['alias'] = compiler.alias + return { - compiler.alias: d + 'compiler': d } @@ -91,11 +89,11 @@ def get_compiler_config(scope=None): """ def init_compiler_config(): """Compiler search used when Spack has no compilers.""" - config = {} compilers = find_compilers() + compilers_dict = [] for compiler in compilers: - config.update(_to_dict(compiler)) - spack.config.update_config('compilers', config, scope=scope) + compilers_dict.append(_to_dict(compiler)) + spack.config.update_config('compilers', compilers_dict, scope=scope) config = spack.config.get_config('compilers', scope=scope) # Update the configuration if there are currently no compilers @@ -127,7 +125,7 @@ def add_compilers_to_config(compilers, scope=None): """ compiler_config = get_compiler_config(scope) for compiler in compilers: - compiler_config = _to_dict(compiler) + compiler_config.append(_to_dict(compiler)) spack.config.update_config('compilers', compiler_config, scope) @@ -167,8 +165,8 @@ def all_compilers_config(scope=None): def all_compilers(scope=None): # Return compiler specs from the merged config. - return [spack.spec.CompilerSpec(s['spec']) - for s in all_compilers_config(scope).values()] + return [spack.spec.CompilerSpec(s['compiler']['spec']) + for s in all_compilers_config(scope)] def default_compiler(): @@ -230,11 +228,10 @@ def compilers_for_spec(compiler_spec, scope=None, **kwargs): def get_compilers(cspec): compilers = [] - for aka, cmp in config.items(): - if cmp['spec'] != str(cspec): + for items in config: + if items['compiler']['spec'] != str(cspec): continue - items = cmp - alias = aka + items = items['compiler'] if not ('paths' in items and all(n in items['paths'] for n in _path_instance_vars)): raise InvalidCompilerConfigurationError(cspec) @@ -258,6 +255,9 @@ def get_compilers(cspec): else: operating_system = None + + alias = items['alias'] if 'alias' in items else None + flags = {} for f in spack.spec.FlagMap.valid_compiler_flags(): if f in items: diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index ec04c81787..e51016998c 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -146,11 +146,9 @@ 'additionalProperties': False, 'patternProperties': { 'compilers:?': { # optional colon for overriding site config. - 'type': 'object', - 'default': {}, - 'additionalProperties': False, - 'patternProperties': { - r'\w[\w-]*': { # alias + 'type': 'array', + 'items': { + 'compiler': { 'type': 'object', 'additionalProperties': False, 'required': ['paths', 'spec', 'modules', 'operating_system'], @@ -180,15 +178,10 @@ {'type' : 'null' }]}, 'ldlibs': { 'anyOf': [ {'type' : 'string' }, {'type' : 'null' }]}}}, - 'spec': { 'type': 'string'},#r'\w[\w-]*@\w[\w-]*' - 'operating_system': { - 'type': 'object', - 'required': ['name', 'version'], - 'additionalProperties': False, - 'properties': { - 'name': {'type': 'string'}, - 'version': {'type': 'string'} - }}, + 'spec': { 'type': 'string'}, + 'operating_system': { 'type': 'string'}, + 'alias': { 'anyOf': [ {'type' : 'string'}, + {'type' : 'null' }]}, 'modules': { 'anyOf': [ {'type' : 'string'}, {'type' : 'null' }, {'type': 'array'}, @@ -591,8 +584,7 @@ def they_are(t): # Source list is prepended (for precedence) if they_are(list): - seen = set(source) - dest[:] = source + [x for x in dest if x not in seen] + dest[:] = source + [x for x in dest if x not in source] return dest # Source dict is merged into dest. diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index 8fffc09437..252d77e66b 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -32,8 +32,8 @@ from spack.test.mock_packages_test import * # Some sample compiler config data -a_comps = { - 'gcc473': { +a_comps = [ + {'compiler': { 'paths': { "cc" : "/gcc473", "cxx": "/g++473", @@ -42,12 +42,9 @@ }, 'modules': None, 'spec': 'gcc@4.7.3', - 'operating_system': { - 'name': 'CNL', - 'version': '10' - } - }, - 'gcc450': { + 'operating_system': 'CNL10' + }}, + {'compiler': { 'paths': { "cc" : "/gcc450", "cxx": "/g++450", @@ -56,12 +53,9 @@ }, 'modules': None, 'spec': 'gcc@4.5.0', - 'operating_system': { - 'name': 'CNL', - 'version': '10' - } - }, - 'clang33': { + 'operating_system': 'CNL10' + }}, + {'compiler': { 'paths': { "cc" : "", "cxx": "", @@ -69,15 +63,12 @@ "fc" : '' }, 'modules': None, 'spec': 'clang@3.3', - 'operating_system': { - 'name': 'CNL', - 'version': '10' - } - } -} + 'operating_system': 'CNL10' + }} +] -b_comps = { - 'icc100': { +b_comps = [ + {'compiler': { 'paths': { "cc" : "/icc100", "cxx": "/icp100", @@ -86,12 +77,9 @@ }, 'modules': None, 'spec': 'icc@10.0', - 'operating_system': { - 'name': 'CNL', - 'version': '10' - } - }, - 'icc111': { + 'operating_system': 'CNL10' + }}, + {'compiler': { 'paths': { "cc" : "/icc111", "cxx": "/icp111", @@ -100,12 +88,9 @@ }, 'modules': None, 'spec': 'icc@11.1', - 'operating_system': { - 'name': 'CNL', - 'version': '10' - } - }, - 'clang33': { + 'operating_system': 'CNL10' + }}, + {'compiler': { 'paths': { "cc" : "", "cxx": "", @@ -113,12 +98,9 @@ "fc" : '' }, 'modules': None, 'spec': 'clang@3.3', - 'operating_system': { - 'name': 'CNL', - 'version': '10' - } - } -} + 'operating_system': 'CNL10' + }} +] # Some Sample repo data repos_low = [ "/some/path" ] @@ -143,15 +125,21 @@ def check_config(self, comps, *compiler_names): config = spack.config.get_config('compilers') compiler_list = ['cc', 'cxx', 'f77', 'fc'] param_list = ['modules', 'paths', 'spec', 'operating_system'] - for alias, compiler in config.items(): - if compiler['spec'] in compiler_names: + for compiler in config: + conf = compiler['compiler'] + if conf['spec'] in compiler_names: + comp = None + for c in comps: + if c['compiler']['spec'] == conf['spec']: + comp = c['compiler'] + break + if not comp: + self.fail('Bad config spec') for p in param_list: - expected = comps[alias][p] - actual = config[alias][p] - self.assertEqual(expected, actual) + self.assertEqual(conf[p], comp[p]) for c in compiler_list: - expected = comps[alias]['paths'][c] - actual = config[alias]['paths'][c] + expected = comp['paths'][c] + actual = conf['paths'][c] self.assertEqual(expected, actual) def test_write_list_in_memory(self): diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index 9c586bd8ce..a56bd8ebdc 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -46,132 +46,108 @@ mock_compiler_config = """\ compilers: - clang3.3GENLINUX: +- compiler: spec: clang@3.3 - operating_system: - name: {0} - version: '{1}' + operating_system: {0}{1} paths: cc: /path/to/clang cxx: /path/to/clang++ f77: None fc: None modules: 'None' - gcc4.5GENLINUX: +- compiler: spec: gcc@4.5.0 - operating_system: - name: {0} - version: '{1}' + operating_system: {0}{1} paths: cc: /path/to/gcc cxx: /path/to/g++ f77: None fc: None modules: 'None' - clang3.3CNL: +- compiler: spec: clang@3.3 - operating_system: - name: CNL - version: '10' + operating_system: CNL10 paths: cc: /path/to/clang cxx: /path/to/clang++ f77: None fc: None modules: 'None' - clang3.3SUSE: +- compiler: spec: clang@3.3 - operating_system: - name: SuSE - version: '11' + operating_system: SuSE11 paths: cc: /path/to/clang cxx: /path/to/clang++ f77: None fc: None modules: 'None' - clang3.3RHL: +- compiler: spec: clang@3.3 - operating_system: - name: redhat - version: '6' + operating_system: redhat6 paths: cc: /path/to/clang cxx: /path/to/clang++ f77: None fc: None modules: 'None' - clang3.3OSX: +- compiler: spec: clang@3.3 - operating_system: - name: yosemite - version: '10.10' + operating_system: yosemite paths: cc: /path/to/clang cxx: /path/to/clang++ f77: None fc: None modules: 'None' - gcc4.5.0CNL: +- compiler: paths: cc: /path/to/gcc cxx: /path/to/g++ f77: /path/to/gfortran fc: /path/to/gfortran - operating_system: - name: CNL - version: '10' + operating_system: CNL10 spec: gcc@4.5.0 modules: 'None' - gcc4.5.0SUSE: +- compiler: paths: cc: /path/to/gcc cxx: /path/to/g++ f77: /path/to/gfortran fc: /path/to/gfortran - operating_system: - name: SuSE - version: '11' + operating_system: SuSE11 spec: gcc@4.5.0 modules: 'None' - gcc4.5.0RHL: +- compiler: paths: cc: /path/to/gcc cxx: /path/to/g++ f77: /path/to/gfortran fc: /path/to/gfortran - operating_system: - name: redhat - version: '6' + operating_system: redhat6 spec: gcc@4.5.0 modules: 'None' - gcc4.5.0OSX: +- compiler: paths: cc: /path/to/gcc cxx: /path/to/g++ f77: /path/to/gfortran fc: /path/to/gfortran - operating_system: - name: yosemite - version: '10.10' + operating_system: yosemite spec: gcc@4.5.0 modules: 'None' - gcc4.5.0ELCAP: +- compiler: paths: cc: /path/to/gcc cxx: /path/to/g++ f77: /path/to/gfortran fc: /path/to/gfortran - operating_system: - name: elcapitan - version: '10.11' + operating_system: elcapitan spec: gcc@4.5.0 modules: 'None' - clang3.3ELCAP: +- compiler: spec: clang@3.3 - operating_system: - name: elcapitan - version: '10.11' + operating_system: elcapitan paths: cc: /path/to/clang cxx: /path/to/clang++ diff --git a/lib/spack/spack/yaml_version_check.py b/lib/spack/spack/yaml_version_check.py index 7e7bae4edf..5930eefafa 100644 --- a/lib/spack/spack/yaml_version_check.py +++ b/lib/spack/spack/yaml_version_check.py @@ -44,9 +44,9 @@ def check_compiler_yaml_version(): data = syaml.load(f) if data: - compilers = data['compilers'].items() + compilers = data['compilers'] if len(compilers) > 0: - if 'operating_system' not in compilers[0][1]: + if 'operating_system' not in compilers[0]['compiler']: new_file = os.path.join(scope.path, '_old_compilers.yaml') tty.warn('%s in out of date compilers format. ' 'Moved to %s. Spack automatically generate ' From eb9b00b210b3c58dc1ea4fdf2b59cd1922266561 Mon Sep 17 00:00:00 2001 From: alalazo Date: Sat, 11 Jun 2016 12:44:21 +0200 Subject: [PATCH 333/481] versions : added unit tests for the new properties --- lib/spack/spack/test/versions.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/spack/spack/test/versions.py b/lib/spack/spack/test/versions.py index a026403e2e..892073feab 100644 --- a/lib/spack/spack/test/versions.py +++ b/lib/spack/spack/test/versions.py @@ -423,3 +423,11 @@ def test_satisfaction_with_lists(self): self.assert_satisfies('4.8.0', '4.2, 4.3:4.8') self.assert_satisfies('4.8.2', '4.2, 4.3:4.8') + + def test_formatted_strings(self): + versions = '1.2.3', '1_2_3', '1-2-3' + for item in versions: + v = Version(item) + self.assertEqual(v.dotted, '1.2.3') + self.assertEqual(v.dashed, '1-2-3') + self.assertEqual(v.underscored, '1_2_3') \ No newline at end of file From 3737169450228643848572ba2605d9f20aba2039 Mon Sep 17 00:00:00 2001 From: alalazo Date: Sat, 11 Jun 2016 12:57:46 +0200 Subject: [PATCH 334/481] qa : fixed style and a bug reported by flake8 lib/spack/spack/test/versions.py:313: [F811] redefinition of unused 'test_union_with_containment' from line 306 Flake8 found errors. --- lib/spack/spack/test/versions.py | 48 ++------------------------------ 1 file changed, 3 insertions(+), 45 deletions(-) diff --git a/lib/spack/spack/test/versions.py b/lib/spack/spack/test/versions.py index 892073feab..4624f901c8 100644 --- a/lib/spack/spack/test/versions.py +++ b/lib/spack/spack/test/versions.py @@ -43,7 +43,6 @@ def assert_ver_lt(self, a, b): self.assertFalse(a > b) self.assertFalse(a >= b) - def assert_ver_gt(self, a, b): a, b = ver(a), ver(b) self.assertTrue(a > b) @@ -53,7 +52,6 @@ def assert_ver_gt(self, a, b): self.assertFalse(a < b) self.assertFalse(a <= b) - def assert_ver_eq(self, a, b): a, b = ver(a), ver(b) self.assertFalse(a > b) @@ -63,55 +61,43 @@ def assert_ver_eq(self, a, b): self.assertFalse(a < b) self.assertTrue(a <= b) - def assert_in(self, needle, haystack): self.assertTrue(ver(needle) in ver(haystack)) - def assert_not_in(self, needle, haystack): self.assertFalse(ver(needle) in ver(haystack)) - def assert_canonical(self, canonical_list, version_list): self.assertEqual(ver(canonical_list), ver(version_list)) - def assert_overlaps(self, v1, v2): self.assertTrue(ver(v1).overlaps(ver(v2))) - def assert_no_overlap(self, v1, v2): self.assertFalse(ver(v1).overlaps(ver(v2))) - def assert_satisfies(self, v1, v2): self.assertTrue(ver(v1).satisfies(ver(v2))) - def assert_does_not_satisfy(self, v1, v2): self.assertFalse(ver(v1).satisfies(ver(v2))) - def check_intersection(self, expected, a, b): self.assertEqual(ver(expected), ver(a).intersection(ver(b))) - def check_union(self, expected, a, b): self.assertEqual(ver(expected), ver(a).union(ver(b))) - def test_two_segments(self): self.assert_ver_eq('1.0', '1.0') self.assert_ver_lt('1.0', '2.0') self.assert_ver_gt('2.0', '1.0') - def test_three_segments(self): self.assert_ver_eq('2.0.1', '2.0.1') self.assert_ver_lt('2.0', '2.0.1') self.assert_ver_gt('2.0.1', '2.0') - def test_alpha(self): # TODO: not sure whether I like this. 2.0.1a is *usually* # TODO: less than 2.0.1, but special-casing it makes version @@ -120,7 +106,6 @@ def test_alpha(self): self.assert_ver_gt('2.0.1a', '2.0.1') self.assert_ver_lt('2.0.1', '2.0.1a') - def test_patch(self): self.assert_ver_eq('5.5p1', '5.5p1') self.assert_ver_lt('5.5p1', '5.5p2') @@ -129,7 +114,6 @@ def test_patch(self): self.assert_ver_lt('5.5p1', '5.5p10') self.assert_ver_gt('5.5p10', '5.5p1') - def test_num_alpha_with_no_separator(self): self.assert_ver_lt('10xyz', '10.1xyz') self.assert_ver_gt('10.1xyz', '10xyz') @@ -137,7 +121,6 @@ def test_num_alpha_with_no_separator(self): self.assert_ver_lt('xyz10', 'xyz10.1') self.assert_ver_gt('xyz10.1', 'xyz10') - def test_alpha_with_dots(self): self.assert_ver_eq('xyz.4', 'xyz.4') self.assert_ver_lt('xyz.4', '8') @@ -145,30 +128,25 @@ def test_alpha_with_dots(self): self.assert_ver_lt('xyz.4', '2') self.assert_ver_gt('2', 'xyz.4') - def test_nums_and_patch(self): self.assert_ver_lt('5.5p2', '5.6p1') self.assert_ver_gt('5.6p1', '5.5p2') self.assert_ver_lt('5.6p1', '6.5p1') self.assert_ver_gt('6.5p1', '5.6p1') - def test_rc_versions(self): self.assert_ver_gt('6.0.rc1', '6.0') self.assert_ver_lt('6.0', '6.0.rc1') - def test_alpha_beta(self): self.assert_ver_gt('10b2', '10a1') self.assert_ver_lt('10a2', '10b2') - def test_double_alpha(self): self.assert_ver_eq('1.0aa', '1.0aa') self.assert_ver_lt('1.0a', '1.0aa') self.assert_ver_gt('1.0aa', '1.0a') - def test_padded_numbers(self): self.assert_ver_eq('10.0001', '10.0001') self.assert_ver_eq('10.0001', '10.1') @@ -176,24 +154,20 @@ def test_padded_numbers(self): self.assert_ver_lt('10.0001', '10.0039') self.assert_ver_gt('10.0039', '10.0001') - def test_close_numbers(self): self.assert_ver_lt('4.999.9', '5.0') self.assert_ver_gt('5.0', '4.999.9') - def test_date_stamps(self): self.assert_ver_eq('20101121', '20101121') self.assert_ver_lt('20101121', '20101122') self.assert_ver_gt('20101122', '20101121') - def test_underscores(self): self.assert_ver_eq('2_0', '2_0') self.assert_ver_eq('2.0', '2_0') self.assert_ver_eq('2_0', '2.0') - def test_rpm_oddities(self): self.assert_ver_eq('1b.fc17', '1b.fc17') self.assert_ver_lt('1b.fc17', '1.fc17') @@ -202,7 +176,6 @@ def test_rpm_oddities(self): self.assert_ver_gt('1g.fc17', '1.fc17') self.assert_ver_lt('1.fc17', '1g.fc17') - # Stuff below here is not taken from RPM's tests and is # unique to spack def test_version_ranges(self): @@ -214,7 +187,6 @@ def test_version_ranges(self): self.assert_ver_lt('1.2:1.4', '1.5:1.6') self.assert_ver_gt('1.5:1.6', '1.2:1.4') - def test_contains(self): self.assert_in('1.3', '1.2:1.4') self.assert_in('1.2.5', '1.2:1.4') @@ -233,7 +205,6 @@ def test_contains(self): self.assert_in('1.4.1', '1.2.7:1.4') self.assert_not_in('1.4.1', '1.2.7:1.4.0') - def test_in_list(self): self.assert_in('1.2', ['1.5', '1.2', '1.3']) self.assert_in('1.2.5', ['1.5', '1.2:1.3']) @@ -245,7 +216,6 @@ def test_in_list(self): self.assert_not_in('1.2.5:1.5', ['1.5', '1.2:1.3']) self.assert_not_in('1.1:1.2.5', ['1.5', '1.2:1.3']) - def test_ranges_overlap(self): self.assert_overlaps('1.2', '1.2') self.assert_overlaps('1.2.1', '1.2.1') @@ -262,7 +232,6 @@ def test_ranges_overlap(self): self.assert_overlaps(':', '1.6:1.9') self.assert_overlaps('1.6:1.9', ':') - def test_overlap_with_containment(self): self.assert_in('1.6.5', '1.6') self.assert_in('1.6.5', ':1.6') @@ -273,7 +242,6 @@ def test_overlap_with_containment(self): self.assert_not_in(':1.6', '1.6.5') self.assert_in('1.6.5', ':1.6') - def test_lists_overlap(self): self.assert_overlaps('1.2b:1.7,5', '1.6:1.9,1') self.assert_overlaps('1,2,3,4,5', '3,4,5,6,7') @@ -287,7 +255,6 @@ def test_lists_overlap(self): self.assert_no_overlap('1,2,3,4,5', '6,7') self.assert_no_overlap('1,2,3,4,5', '6:7') - def test_canonicalize_list(self): self.assert_canonical(['1.2', '1.3', '1.4'], ['1.2', '1.3', '1.3', '1.4']) @@ -316,7 +283,6 @@ def test_canonicalize_list(self): self.assert_canonical([':'], [':,1.3, 1.3.1,1.3.9,1.4 : 1.5 , 1.3 : 1.4']) - def test_intersection(self): self.check_intersection('2.5', '1.0:2.5', '2.5:3.0') @@ -325,12 +291,11 @@ def test_intersection(self): self.check_intersection('0:1', ':', '0:1') self.check_intersection(['1.0', '2.5:2.7'], - ['1.0:2.7'], ['2.5:3.0','1.0']) + ['1.0:2.7'], ['2.5:3.0', '1.0']) self.check_intersection(['2.5:2.7'], - ['1.1:2.7'], ['2.5:3.0','1.0']) + ['1.1:2.7'], ['2.5:3.0', '1.0']) self.check_intersection(['0:1'], [':'], ['0:1']) - def test_intersect_with_containment(self): self.check_intersection('1.6.5', '1.6.5', ':1.6') self.check_intersection('1.6.5', ':1.6', '1.6.5') @@ -338,7 +303,6 @@ def test_intersect_with_containment(self): self.check_intersection('1.6:1.6.5', ':1.6.5', '1.6') self.check_intersection('1.6:1.6.5', '1.6', ':1.6.5') - def test_union_with_containment(self): self.check_union(':1.6', '1.6.5', ':1.6') self.check_union(':1.6', ':1.6', '1.6.5') @@ -346,8 +310,6 @@ def test_union_with_containment(self): self.check_union(':1.6', ':1.6.5', '1.6') self.check_union(':1.6', '1.6', ':1.6.5') - - def test_union_with_containment(self): self.check_union(':', '1.0:', ':2.0') self.check_union('1:4', '1:3', '2:4') @@ -356,7 +318,6 @@ def test_union_with_containment(self): # Tests successor/predecessor case. self.check_union('1:4', '1:2', '3:4') - def test_basic_version_satisfaction(self): self.assert_satisfies('4.7.3', '4.7.3') @@ -372,7 +333,6 @@ def test_basic_version_satisfaction(self): self.assert_does_not_satisfy('4.8', '4.9') self.assert_does_not_satisfy('4', '4.9') - def test_basic_version_satisfaction_in_lists(self): self.assert_satisfies(['4.7.3'], ['4.7.3']) @@ -388,7 +348,6 @@ def test_basic_version_satisfaction_in_lists(self): self.assert_does_not_satisfy(['4.8'], ['4.9']) self.assert_does_not_satisfy(['4'], ['4.9']) - def test_version_range_satisfaction(self): self.assert_satisfies('4.7b6', '4.3:4.7') self.assert_satisfies('4.3.0', '4.3:4.7') @@ -400,7 +359,6 @@ def test_version_range_satisfaction(self): self.assert_satisfies('4.7b6', '4.3:4.7') self.assert_does_not_satisfy('4.8.0', '4.3:4.7') - def test_version_range_satisfaction_in_lists(self): self.assert_satisfies(['4.7b6'], ['4.3:4.7']) self.assert_satisfies(['4.3.0'], ['4.3:4.7']) @@ -430,4 +388,4 @@ def test_formatted_strings(self): v = Version(item) self.assertEqual(v.dotted, '1.2.3') self.assertEqual(v.dashed, '1-2-3') - self.assertEqual(v.underscored, '1_2_3') \ No newline at end of file + self.assertEqual(v.underscored, '1_2_3') From 6c0ea77d1d5e349329ad36aecaadff13c0ccf6c4 Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Sat, 11 Jun 2016 15:36:43 -0500 Subject: [PATCH 335/481] Add list_url for older versions. --- var/spack/repos/builtin/packages/r-packrat/package.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/r-packrat/package.py b/var/spack/repos/builtin/packages/r-packrat/package.py index c9b02a284e..1fa62da5d4 100644 --- a/var/spack/repos/builtin/packages/r-packrat/package.py +++ b/var/spack/repos/builtin/packages/r-packrat/package.py @@ -30,7 +30,8 @@ class RPackrat(Package): and reproducible way.""" homepage = 'https://github.com/rstudio/packrat/' - url = "https://cran.r-project.org/src/contrib/packrat_0.4.7-1.tar.gz" + url = "https://cran.r-project.org/src/contrib/packrat_0.4.7-1.tar.gz" + list_url = 'https://cran.r-project.org/src/contrib/Archive/packrat' version('0.4.7-1', '80c2413269b292ade163a70ba5053e84') From 953b65abccf392e2a565fe9c1b3c1101f0cb5b66 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Sat, 11 Jun 2016 16:41:39 -0400 Subject: [PATCH 336/481] Do not pass "/usr" or "/usr/local" as path to configure --- var/spack/repos/builtin/packages/openmpi/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/openmpi/package.py b/var/spack/repos/builtin/packages/openmpi/package.py index 4e465e1784..0638628a6c 100644 --- a/var/spack/repos/builtin/packages/openmpi/package.py +++ b/var/spack/repos/builtin/packages/openmpi/package.py @@ -142,7 +142,7 @@ def install(self, spec, prefix): ]) if '+verbs' in spec: path = _verbs_dir() - if path is not None: + if path is not None and path not in ('/usr', '/usr/local'): config_args.append('--with-%s=%s' % (self.verbs, path)) else: config_args.append('--with-%s' % self.verbs) From 0e9f8bd38d4d660db2a03d235535d2d964a7ddec Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Sat, 11 Jun 2016 15:53:28 -0500 Subject: [PATCH 337/481] Add list_url for old versions. --- var/spack/repos/builtin/packages/r-R6/package.py | 3 ++- var/spack/repos/builtin/packages/r-curl/package.py | 3 ++- var/spack/repos/builtin/packages/r-devtools/package.py | 3 ++- var/spack/repos/builtin/packages/r-digest/package.py | 3 ++- var/spack/repos/builtin/packages/r-git2r/package.py | 3 ++- var/spack/repos/builtin/packages/r-httr/package.py | 3 ++- var/spack/repos/builtin/packages/r-jsonlite/package.py | 3 ++- var/spack/repos/builtin/packages/r-memoise/package.py | 3 ++- var/spack/repos/builtin/packages/r-mime/package.py | 3 ++- var/spack/repos/builtin/packages/r-openssl/package.py | 3 ++- var/spack/repos/builtin/packages/r-rstudioapi/package.py | 3 ++- var/spack/repos/builtin/packages/r-whisker/package.py | 3 ++- var/spack/repos/builtin/packages/r-withr/package.py | 3 ++- 13 files changed, 26 insertions(+), 13 deletions(-) diff --git a/var/spack/repos/builtin/packages/r-R6/package.py b/var/spack/repos/builtin/packages/r-R6/package.py index 30f489b3fd..0b75888e79 100644 --- a/var/spack/repos/builtin/packages/r-R6/package.py +++ b/var/spack/repos/builtin/packages/r-R6/package.py @@ -34,7 +34,8 @@ class RR6(Package): classes are defined in different packages.""" homepage = "https://github.com/wch/R6/" - url = "https://cran.r-project.org/src/contrib/R6_2.1.2.tar.gz" + url = "https://cran.r-project.org/src/contrib/R6_2.1.2.tar.gz" + list_url = "https://cran.r-project.org/src/contrib/Archive/R6" version('2.1.2', 'b6afb9430e48707be87638675390e457') diff --git a/var/spack/repos/builtin/packages/r-curl/package.py b/var/spack/repos/builtin/packages/r-curl/package.py index 23d75880a7..c6e8f22a94 100644 --- a/var/spack/repos/builtin/packages/r-curl/package.py +++ b/var/spack/repos/builtin/packages/r-curl/package.py @@ -37,7 +37,8 @@ class RCurl(Package): package with http specific tools and logic.""" homepage = "https://github.com/jeroenooms/curl" - url = "https://cran.r-project.org/src/contrib/curl_0.9.7.tar.gz" + url = "https://cran.r-project.org/src/contrib/curl_0.9.7.tar.gz" + list_url = "https://cran.r-project.org/src/contrib/Archive/RCurl" version('0.9.7', 'a101f7de948cb828fef571c730f39217') diff --git a/var/spack/repos/builtin/packages/r-devtools/package.py b/var/spack/repos/builtin/packages/r-devtools/package.py index 5c73eab936..5f0b7b8779 100644 --- a/var/spack/repos/builtin/packages/r-devtools/package.py +++ b/var/spack/repos/builtin/packages/r-devtools/package.py @@ -29,7 +29,8 @@ class RDevtools(Package): """Collection of package development tools.""" homepage = "https://github.com/hadley/devtools" - url = "https://cran.r-project.org/src/contrib/devtools_1.11.1.tar.gz" + url = "https://cran.r-project.org/src/contrib/devtools_1.11.1.tar.gz" + list_url = "https://cran.r-project.org/src/contrib/Archive/devtools" version('1.11.1', '242672ee27d24dddcbdaac88c586b6c2') diff --git a/var/spack/repos/builtin/packages/r-digest/package.py b/var/spack/repos/builtin/packages/r-digest/package.py index a42e82f1ae..1d11afb139 100644 --- a/var/spack/repos/builtin/packages/r-digest/package.py +++ b/var/spack/repos/builtin/packages/r-digest/package.py @@ -44,7 +44,8 @@ class RDigest(Package): used.""" homepage = "http://dirk.eddelbuettel.com/code/digest.html" - url = "https://cran.r-project.org/src/contrib/digest_0.6.9.tar.gz" + url = "https://cran.r-project.org/src/contrib/digest_0.6.9.tar.gz" + list_url = "https://cran.r-project.org/src/contrib/Archive/digest" version('0.6.9', '48048ce6c466bdb124716e45ba4a0e83') diff --git a/var/spack/repos/builtin/packages/r-git2r/package.py b/var/spack/repos/builtin/packages/r-git2r/package.py index 272dcbad93..1f08379d6e 100644 --- a/var/spack/repos/builtin/packages/r-git2r/package.py +++ b/var/spack/repos/builtin/packages/r-git2r/package.py @@ -31,7 +31,8 @@ class RGit2r(Package): data and running some basic 'Git' commands.""" homepage = "https://github.com/ropensci/git2r" - url = "https://cran.r-project.org/src/contrib/git2r_0.15.0.tar.gz" + url = "https://cran.r-project.org/src/contrib/git2r_0.15.0.tar.gz" + list_url = "https://cran.r-project.org/src/contrib/Archive/git2r" version('0.15.0', '57658b3298f9b9aadc0dd77b4ef6a1e1') diff --git a/var/spack/repos/builtin/packages/r-httr/package.py b/var/spack/repos/builtin/packages/r-httr/package.py index 0f379512a5..77ec34ab03 100644 --- a/var/spack/repos/builtin/packages/r-httr/package.py +++ b/var/spack/repos/builtin/packages/r-httr/package.py @@ -31,7 +31,8 @@ class RHttr(Package): request components (authenticate(), add_headers() and so on).""" homepage = "https://github.com/hadley/httr" - url = "https://cran.r-project.org/src/contrib/httr_1.1.0.tar.gz" + url = "https://cran.r-project.org/src/contrib/httr_1.1.0.tar.gz" + list_url = "https://cran.r-project.org/src/contrib/Archive/httr" version('1.1.0', '5ffbbc5c2529e49f00aaa521a2b35600') diff --git a/var/spack/repos/builtin/packages/r-jsonlite/package.py b/var/spack/repos/builtin/packages/r-jsonlite/package.py index 777506b984..6e231ed345 100644 --- a/var/spack/repos/builtin/packages/r-jsonlite/package.py +++ b/var/spack/repos/builtin/packages/r-jsonlite/package.py @@ -38,7 +38,8 @@ class RJsonlite(Package): use with dynamic data in systems and applications.""" homepage = "https://github.com/jeroenooms/jsonlite" - url = "https://cran.r-project.org/src/contrib/jsonlite_0.9.21.tar.gz" + url = "https://cran.r-project.org/src/contrib/jsonlite_0.9.21.tar.gz" + list_url = "https://cran.r-project.org/src/contrib/Archive/jsonlite" version('0.9.21', '4fc382747f88a79ff0718a0d06bed45d') diff --git a/var/spack/repos/builtin/packages/r-memoise/package.py b/var/spack/repos/builtin/packages/r-memoise/package.py index ca74ab02a2..6a0fb78650 100644 --- a/var/spack/repos/builtin/packages/r-memoise/package.py +++ b/var/spack/repos/builtin/packages/r-memoise/package.py @@ -30,7 +30,8 @@ class RMemoise(Package): same arguments it returns the pre-computed value.""" homepage = "https://github.com/hadley/memoise" - url = "https://cran.r-project.org/src/contrib/memoise_1.0.0.tar.gz" + url = "https://cran.r-project.org/src/contrib/memoise_1.0.0.tar.gz" + list_url = "https://cran.r-project.org/src/contrib/Archive/memoise" version('1.0.0', 'd31145292e2a88ae9a504cab1602e4ac') diff --git a/var/spack/repos/builtin/packages/r-mime/package.py b/var/spack/repos/builtin/packages/r-mime/package.py index ac52d785b1..fb079f44c5 100644 --- a/var/spack/repos/builtin/packages/r-mime/package.py +++ b/var/spack/repos/builtin/packages/r-mime/package.py @@ -30,7 +30,8 @@ class RMime(Package): from /etc/mime.types in UNIX-type systems.""" homepage = "https://github.com/yihui/mime" - url = "https://cran.r-project.org/src/contrib/mime_0.4.tar.gz" + url = "https://cran.r-project.org/src/contrib/mime_0.4.tar.gz" + list_url = "https://cran.r-project.org/src/contrib/Archive/mime" version('0.4', '789cb33e41db2206c6fc7c3e9fbc2c02') diff --git a/var/spack/repos/builtin/packages/r-openssl/package.py b/var/spack/repos/builtin/packages/r-openssl/package.py index fd6ad84aa3..3e77923d76 100644 --- a/var/spack/repos/builtin/packages/r-openssl/package.py +++ b/var/spack/repos/builtin/packages/r-openssl/package.py @@ -38,7 +38,8 @@ class ROpenssl(Package): calculations on large multibyte integers.""" homepage = "https://github.com/jeroenooms/openssl#readme" - url = "https://cran.r-project.org/src/contrib/openssl_0.9.4.tar.gz" + url = "https://cran.r-project.org/src/contrib/openssl_0.9.4.tar.gz" + list_url = "https://cran.r-project.org/src/contrib/Archive/openssl" version('0.9.4', '82a890e71ed0e74499878bedacfb8ccb') diff --git a/var/spack/repos/builtin/packages/r-rstudioapi/package.py b/var/spack/repos/builtin/packages/r-rstudioapi/package.py index 83424392c9..0ef2d9b987 100644 --- a/var/spack/repos/builtin/packages/r-rstudioapi/package.py +++ b/var/spack/repos/builtin/packages/r-rstudioapi/package.py @@ -30,7 +30,8 @@ class RRstudioapi(Package): messages when it's not.""" homepage = "https://cran.r-project.org/web/packages/rstudioapi/index.html" - url = "https://cran.r-project.org/src/contrib/rstudioapi_0.5.tar.gz" + url = "https://cran.r-project.org/src/contrib/rstudioapi_0.5.tar.gz" + list_url = "https://cran.r-project.org/src/contrib/Archive/rstudioapi" version('0.5', '6ce1191da74e7bcbf06b61339486b3ba') diff --git a/var/spack/repos/builtin/packages/r-whisker/package.py b/var/spack/repos/builtin/packages/r-whisker/package.py index f534568ec7..f338d150f1 100644 --- a/var/spack/repos/builtin/packages/r-whisker/package.py +++ b/var/spack/repos/builtin/packages/r-whisker/package.py @@ -30,7 +30,8 @@ class RWhisker(Package): including R""" homepage = "http://github.com/edwindj/whisker" - url = "https://cran.r-project.org/src/contrib/whisker_0.3-2.tar.gz" + url = "https://cran.r-project.org/src/contrib/whisker_0.3-2.tar.gz" + list_url = "https://cran.r-project.org/src/contrib/Archive/whisker" version('0.3-2', 'c4b9bf9a22e69ce003fe68663ab5e8e6') diff --git a/var/spack/repos/builtin/packages/r-withr/package.py b/var/spack/repos/builtin/packages/r-withr/package.py index a9f0cb600f..5ce7437c84 100644 --- a/var/spack/repos/builtin/packages/r-withr/package.py +++ b/var/spack/repos/builtin/packages/r-withr/package.py @@ -32,7 +32,8 @@ class RWithr(Package): dependencies to provide access to these functions.""" homepage = "http://github.com/jimhester/withr" - url = "https://cran.r-project.org/src/contrib/withr_1.0.1.tar.gz" + url = "https://cran.r-project.org/src/contrib/withr_1.0.1.tar.gz" + list_url = "https://cran.r-project.org/src/contrib/Archive/withr" version('1.0.1', 'ac38af2c6f74027c9592dd8f0acb7598') From 59c9816dbf6e8979a6cfcb28ecb244e92f70f2e6 Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Sat, 11 Jun 2016 16:01:23 -0500 Subject: [PATCH 338/481] Make quotations in variables consistent. --- var/spack/repos/builtin/packages/r-packrat/package.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/r-packrat/package.py b/var/spack/repos/builtin/packages/r-packrat/package.py index 1fa62da5d4..9520952bd7 100644 --- a/var/spack/repos/builtin/packages/r-packrat/package.py +++ b/var/spack/repos/builtin/packages/r-packrat/package.py @@ -29,9 +29,9 @@ class RPackrat(Package): """Manage the R packages your project depends on in an isolated, portable, and reproducible way.""" - homepage = 'https://github.com/rstudio/packrat/' + homepage = "https://github.com/rstudio/packrat/" url = "https://cran.r-project.org/src/contrib/packrat_0.4.7-1.tar.gz" - list_url = 'https://cran.r-project.org/src/contrib/Archive/packrat' + list_url = "https://cran.r-project.org/src/contrib/Archive/packrat" version('0.4.7-1', '80c2413269b292ade163a70ba5053e84') From a6681f2d7f0c3ff1a1ec359751710e98d2d52b63 Mon Sep 17 00:00:00 2001 From: alalazo Date: Sun, 12 Jun 2016 15:06:17 +0200 Subject: [PATCH 339/481] environment modules : added function to construct them from source files --- lib/spack/spack/environment.py | 76 +++++++++++++++++++- lib/spack/spack/test/data/sourceme_first.sh | 3 + lib/spack/spack/test/data/sourceme_second.sh | 3 + lib/spack/spack/test/environment.py | 39 +++++++++- 4 files changed, 119 insertions(+), 2 deletions(-) create mode 100644 lib/spack/spack/test/data/sourceme_first.sh create mode 100644 lib/spack/spack/test/data/sourceme_second.sh diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py index af642dcc9b..87759b36e1 100644 --- a/lib/spack/spack/environment.py +++ b/lib/spack/spack/environment.py @@ -26,7 +26,9 @@ import inspect import os import os.path - +import subprocess +import shlex +import json class NameModifier(object): def __init__(self, name, **kwargs): @@ -240,6 +242,78 @@ def apply_modifications(self): for x in actions: x.execute() + @staticmethod + def from_sourcing_files(*args, **kwargs): + """ + Creates an instance of EnvironmentModifications that, if executed, + has the same effect on the environment as sourcing the files passed as + parameters + + Args: + *args: list of files to be sourced + + Returns: + instance of EnvironmentModifications + """ + env = EnvironmentModifications() + # Check if the files are actually there + if not all(os.path.isfile(file) for file in args): + raise RuntimeError('trying to source non-existing files') + # Relevant kwd parameters and formats + info = dict(kwargs) + info.setdefault('shell', '/bin/bash') + info.setdefault('shell_options', '-c') + info.setdefault('source_command', 'source') + info.setdefault('suppress_output', '&> /dev/null') + info.setdefault('concatenate_on_success', '&&') + + shell = '{shell}'.format(**info) + shell_options = '{shell_options}'.format(**info) + source_file = '{source_command} {file} {concatenate_on_success}' + dump_environment = 'python -c "import os, json; print json.dumps(dict(os.environ))"' + # Construct the command that will be executed + command = [source_file.format(file=file, **info) for file in args] + command.append(dump_environment) + command = ' '.join(command) + command = [ + shell, + shell_options, + command + ] + + # Try to source all the files, + proc = subprocess.Popen(command, stdout=subprocess.PIPE, env=os.environ) + proc.wait() + if proc.returncode != 0: + raise RuntimeError('sourcing files returned a non-zero exit code') + output = ''.join([line for line in proc.stdout]) + # Construct a dictionary with all the variables in the environment + after_source_env = dict(json.loads(output)) + + # Filter variables that are due to how we source + after_source_env.pop('SHLVL') + after_source_env.pop('_') + after_source_env.pop('PWD') + + # Fill the EnvironmentModifications instance + this_environment = dict(os.environ) + # New variables + new_variables = set(after_source_env) - set(this_environment) + for x in new_variables: + env.set(x, after_source_env[x]) + # Variables that have been unset + unset_variables = set(this_environment) - set(after_source_env) + for x in unset_variables: + env.unset(x) + # Variables that have been modified + common_variables = set(this_environment).intersection(set(after_source_env)) + modified_variables = [x for x in common_variables if this_environment[x] != after_source_env[x]] + for x in modified_variables: + # TODO : we may be smarter here, and try to parse if we could compose append_path + # TODO : and prepend_path to modify the value + env.set(x, after_source_env[x]) + return env + def concatenate_paths(paths, separator=':'): """ diff --git a/lib/spack/spack/test/data/sourceme_first.sh b/lib/spack/spack/test/data/sourceme_first.sh new file mode 100644 index 0000000000..800f639ac8 --- /dev/null +++ b/lib/spack/spack/test/data/sourceme_first.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +export NEW_VAR='new' +export UNSET_ME='overridden' diff --git a/lib/spack/spack/test/data/sourceme_second.sh b/lib/spack/spack/test/data/sourceme_second.sh new file mode 100644 index 0000000000..db88b8334a --- /dev/null +++ b/lib/spack/spack/test/data/sourceme_second.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +export PATH_LIST='/path/first:/path/second:/path/third:/path/fourth' +unset EMPTY_PATH_LIST \ No newline at end of file diff --git a/lib/spack/spack/test/environment.py b/lib/spack/spack/test/environment.py index ded1539e18..34b4485f8e 100644 --- a/lib/spack/spack/test/environment.py +++ b/lib/spack/spack/test/environment.py @@ -24,7 +24,10 @@ ############################################################################## import unittest import os -from spack.environment import EnvironmentModifications + +from spack import spack_root +from llnl.util.filesystem import join_path +from spack.environment import EnvironmentModifications, SetEnv, UnsetEnv class EnvironmentTest(unittest.TestCase): @@ -95,3 +98,37 @@ def test_extend(self): self.assertEqual(len(copy_construct), 2) for x, y in zip(env, copy_construct): assert x is y + + def test_source_files(self): + datadir = join_path(spack_root, 'lib', 'spack', 'spack', 'test', 'data') + files = [ + join_path(datadir, 'sourceme_first.sh'), + join_path(datadir, 'sourceme_second.sh') + ] + env = EnvironmentModifications.from_sourcing_files(*files) + modifications = env.group_by_name() + + self.assertEqual(len(modifications), 4) + # Set new variables + self.assertEqual(len(modifications['NEW_VAR']), 1) + self.assertTrue(isinstance(modifications['NEW_VAR'][0], SetEnv)) + self.assertEqual(modifications['NEW_VAR'][0].value, 'new') + # Unset variables + self.assertEqual(len(modifications['EMPTY_PATH_LIST']), 1) + self.assertTrue(isinstance(modifications['EMPTY_PATH_LIST'][0], UnsetEnv)) + # Modified variables + self.assertEqual(len(modifications['UNSET_ME']), 1) + self.assertTrue(isinstance(modifications['UNSET_ME'][0], SetEnv)) + self.assertEqual(modifications['UNSET_ME'][0].value, 'overridden') + + self.assertEqual(len(modifications['PATH_LIST']), 1) + self.assertTrue(isinstance(modifications['PATH_LIST'][0], SetEnv)) + self.assertEqual(modifications['PATH_LIST'][0].value, '/path/first:/path/second:/path/third:/path/fourth') + + # TODO : with reference to the TODO in spack/environment.py + # TODO : remove the above and insert + # self.assertEqual(len(modifications['PATH_LIST']), 2) + # self.assertTrue(isinstance(modifications['PATH_LIST'][0], PrependPath)) + # self.assertEqual(modifications['PATH_LIST'][0].value, '/path/first') + # self.assertTrue(isinstance(modifications['PATH_LIST'][1], AppendPath)) + # self.assertEqual(modifications['PATH_LIST'][1].value, '/path/fourth') From 5300ffac7f1e4f77fcb1ba52083d4a7ba7183467 Mon Sep 17 00:00:00 2001 From: alalazo Date: Sun, 12 Jun 2016 15:11:26 +0200 Subject: [PATCH 340/481] qa : fixed flak8 checks --- lib/spack/spack/environment.py | 25 +++++++++++++------ lib/spack/spack/test/environment.py | 37 ++++++++++++++++++++++------- 2 files changed, 46 insertions(+), 16 deletions(-) diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py index 87759b36e1..623bfa6ed2 100644 --- a/lib/spack/spack/environment.py +++ b/lib/spack/spack/environment.py @@ -24,13 +24,14 @@ ############################################################################## import collections import inspect +import json import os import os.path import subprocess -import shlex -import json + class NameModifier(object): + def __init__(self, name, **kwargs): self.name = name self.args = {'name': name} @@ -38,6 +39,7 @@ def __init__(self, name, **kwargs): class NameValueModifier(object): + def __init__(self, name, value, **kwargs): self.name = name self.value = value @@ -47,23 +49,27 @@ def __init__(self, name, value, **kwargs): class SetEnv(NameValueModifier): + def execute(self): os.environ[self.name] = str(self.value) class UnsetEnv(NameModifier): + def execute(self): # Avoid throwing if the variable was not set os.environ.pop(self.name, None) class SetPath(NameValueModifier): + def execute(self): string_path = concatenate_paths(self.value, separator=self.separator) os.environ[self.name] = string_path class AppendPath(NameValueModifier): + def execute(self): environment_value = os.environ.get(self.name, '') directories = environment_value.split( @@ -73,6 +79,7 @@ def execute(self): class PrependPath(NameValueModifier): + def execute(self): environment_value = os.environ.get(self.name, '') directories = environment_value.split( @@ -82,6 +89,7 @@ def execute(self): class RemovePath(NameValueModifier): + def execute(self): environment_value = os.environ.get(self.name, '') directories = environment_value.split( @@ -270,7 +278,7 @@ def from_sourcing_files(*args, **kwargs): shell = '{shell}'.format(**info) shell_options = '{shell_options}'.format(**info) source_file = '{source_command} {file} {concatenate_on_success}' - dump_environment = 'python -c "import os, json; print json.dumps(dict(os.environ))"' + dump_environment = 'python -c "import os, json; print json.dumps(dict(os.environ))"' # NOQA: ignore=E501 # Construct the command that will be executed command = [source_file.format(file=file, **info) for file in args] command.append(dump_environment) @@ -282,7 +290,8 @@ def from_sourcing_files(*args, **kwargs): ] # Try to source all the files, - proc = subprocess.Popen(command, stdout=subprocess.PIPE, env=os.environ) + proc = subprocess.Popen( + command, stdout=subprocess.PIPE, env=os.environ) proc.wait() if proc.returncode != 0: raise RuntimeError('sourcing files returned a non-zero exit code') @@ -306,10 +315,12 @@ def from_sourcing_files(*args, **kwargs): for x in unset_variables: env.unset(x) # Variables that have been modified - common_variables = set(this_environment).intersection(set(after_source_env)) - modified_variables = [x for x in common_variables if this_environment[x] != after_source_env[x]] + common_variables = set(this_environment).intersection( + set(after_source_env)) + modified_variables = [x for x in common_variables if this_environment[x] != after_source_env[x]] # NOQA: ignore=E501 for x in modified_variables: - # TODO : we may be smarter here, and try to parse if we could compose append_path + # TODO : we may be smarter here, and try to parse + # TODO : if we could compose append_path # TODO : and prepend_path to modify the value env.set(x, after_source_env[x]) return env diff --git a/lib/spack/spack/test/environment.py b/lib/spack/spack/test/environment.py index 34b4485f8e..31a85f9ee3 100644 --- a/lib/spack/spack/test/environment.py +++ b/lib/spack/spack/test/environment.py @@ -31,12 +31,13 @@ class EnvironmentTest(unittest.TestCase): + def setUp(self): os.environ.clear() os.environ['UNSET_ME'] = 'foo' os.environ['EMPTY_PATH_LIST'] = '' os.environ['PATH_LIST'] = '/path/second:/path/third' - os.environ['REMOVE_PATH_LIST'] = '/a/b:/duplicate:/a/c:/remove/this:/a/d:/duplicate/:/f/g' + os.environ['REMOVE_PATH_LIST'] = '/a/b:/duplicate:/a/c:/remove/this:/a/d:/duplicate/:/f/g' # NOQA: ignore=E501 def test_set(self): env = EnvironmentModifications() @@ -77,9 +78,18 @@ def test_path_manipulation(self): env.remove_path('REMOVE_PATH_LIST', '/duplicate/') env.apply_modifications() - self.assertEqual('/path/first:/path/second:/path/third:/path/last', os.environ['PATH_LIST']) - self.assertEqual('/path/first:/path/middle:/path/last', os.environ['EMPTY_PATH_LIST']) - self.assertEqual('/path/first:/path/middle:/path/last', os.environ['NEWLY_CREATED_PATH_LIST']) + self.assertEqual( + '/path/first:/path/second:/path/third:/path/last', + os.environ['PATH_LIST'] + ) + self.assertEqual( + '/path/first:/path/middle:/path/last', + os.environ['EMPTY_PATH_LIST'] + ) + self.assertEqual( + '/path/first:/path/middle:/path/last', + os.environ['NEWLY_CREATED_PATH_LIST'] + ) self.assertEqual('/a/b:/a/c:/a/d:/f/g', os.environ['REMOVE_PATH_LIST']) def test_extra_arguments(self): @@ -100,7 +110,8 @@ def test_extend(self): assert x is y def test_source_files(self): - datadir = join_path(spack_root, 'lib', 'spack', 'spack', 'test', 'data') + datadir = join_path(spack_root, 'lib', 'spack', + 'spack', 'test', 'data') files = [ join_path(datadir, 'sourceme_first.sh'), join_path(datadir, 'sourceme_second.sh') @@ -115,7 +126,8 @@ def test_source_files(self): self.assertEqual(modifications['NEW_VAR'][0].value, 'new') # Unset variables self.assertEqual(len(modifications['EMPTY_PATH_LIST']), 1) - self.assertTrue(isinstance(modifications['EMPTY_PATH_LIST'][0], UnsetEnv)) + self.assertTrue(isinstance( + modifications['EMPTY_PATH_LIST'][0], UnsetEnv)) # Modified variables self.assertEqual(len(modifications['UNSET_ME']), 1) self.assertTrue(isinstance(modifications['UNSET_ME'][0], SetEnv)) @@ -123,12 +135,19 @@ def test_source_files(self): self.assertEqual(len(modifications['PATH_LIST']), 1) self.assertTrue(isinstance(modifications['PATH_LIST'][0], SetEnv)) - self.assertEqual(modifications['PATH_LIST'][0].value, '/path/first:/path/second:/path/third:/path/fourth') + self.assertEqual( + modifications['PATH_LIST'][0].value, + '/path/first:/path/second:/path/third:/path/fourth' + ) # TODO : with reference to the TODO in spack/environment.py # TODO : remove the above and insert # self.assertEqual(len(modifications['PATH_LIST']), 2) - # self.assertTrue(isinstance(modifications['PATH_LIST'][0], PrependPath)) + # self.assertTrue( + # isinstance(modifications['PATH_LIST'][0], PrependPath) + # ) # self.assertEqual(modifications['PATH_LIST'][0].value, '/path/first') - # self.assertTrue(isinstance(modifications['PATH_LIST'][1], AppendPath)) + # self.assertTrue( + # isinstance(modifications['PATH_LIST'][1], AppendPath) + # ) # self.assertEqual(modifications['PATH_LIST'][1].value, '/path/fourth') From 30c9d976f6e7b74b6713215e8f7c7924a9f43f43 Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Sun, 12 Jun 2016 18:49:12 +0200 Subject: [PATCH 341/481] environment : added more logic to treat append and prepend path --- lib/spack/spack/environment.py | 60 +++++++++++++++++--- lib/spack/spack/test/data/sourceme_second.sh | 2 +- lib/spack/spack/test/environment.py | 33 +++++------ 3 files changed, 68 insertions(+), 27 deletions(-) diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py index 623bfa6ed2..cfba060459 100644 --- a/lib/spack/spack/environment.py +++ b/lib/spack/spack/environment.py @@ -1,4 +1,4 @@ -############################################################################## +# # Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory. # @@ -21,7 +21,7 @@ # You should have received a copy of the GNU Lesser General Public # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## +# import collections import inspect import json @@ -100,6 +100,7 @@ def execute(self): class EnvironmentModifications(object): + """ Keeps track of requests to modify the current environment. @@ -315,14 +316,57 @@ def from_sourcing_files(*args, **kwargs): for x in unset_variables: env.unset(x) # Variables that have been modified - common_variables = set(this_environment).intersection( - set(after_source_env)) + common_variables = set(this_environment).intersection(set(after_source_env)) # NOQA: ignore=E501 modified_variables = [x for x in common_variables if this_environment[x] != after_source_env[x]] # NOQA: ignore=E501 + + def return_separator_if_any(first_value, second_value): + separators = ':', ';' + for separator in separators: + if separator in first_value and separator in second_value: + return separator + return None + for x in modified_variables: - # TODO : we may be smarter here, and try to parse - # TODO : if we could compose append_path - # TODO : and prepend_path to modify the value - env.set(x, after_source_env[x]) + current = this_environment[x] + modified = after_source_env[x] + sep = return_separator_if_any(current, modified) + if sep is None: + # We just need to set the variable to the new value + env.set(x, after_source_env[x]) + else: + current_list = current.split(sep) + modified_list = modified.split(sep) + # Paths that have been removed + remove_list = [ + ii for ii in current_list if ii not in modified_list] + # Check that nothing has been added in the middle of vurrent + # list + remaining_list = [ + ii for ii in current_list if ii in modified_list] + start = modified_list.index(remaining_list[0]) + end = modified_list.index(remaining_list[-1]) + search = sep.join(modified_list[start:end + 1]) + if search not in current: + # We just need to set the variable to the new value + env.set(x, after_source_env[x]) + break + else: + try: + prepend_list = modified_list[:start] + except KeyError: + prepend_list = [] + try: + append_list = modified_list[end + 1:] + except KeyError: + append_list = [] + + for item in remove_list: + env.remove_path(x, item) + for item in append_list: + env.append_path(x, item) + for item in prepend_list: + env.prepend_path(x, item) + return env diff --git a/lib/spack/spack/test/data/sourceme_second.sh b/lib/spack/spack/test/data/sourceme_second.sh index db88b8334a..9955a0e6d6 100644 --- a/lib/spack/spack/test/data/sourceme_second.sh +++ b/lib/spack/spack/test/data/sourceme_second.sh @@ -1,3 +1,3 @@ #!/usr/bin/env bash -export PATH_LIST='/path/first:/path/second:/path/third:/path/fourth' +export PATH_LIST='/path/first:/path/second:/path/fourth' unset EMPTY_PATH_LIST \ No newline at end of file diff --git a/lib/spack/spack/test/environment.py b/lib/spack/spack/test/environment.py index 31a85f9ee3..219c68e5a8 100644 --- a/lib/spack/spack/test/environment.py +++ b/lib/spack/spack/test/environment.py @@ -27,7 +27,9 @@ from spack import spack_root from llnl.util.filesystem import join_path -from spack.environment import EnvironmentModifications, SetEnv, UnsetEnv +from spack.environment import EnvironmentModifications +from spack.environment import SetEnv, UnsetEnv +from spack.environment import RemovePath, PrependPath, AppendPath class EnvironmentTest(unittest.TestCase): @@ -133,21 +135,16 @@ def test_source_files(self): self.assertTrue(isinstance(modifications['UNSET_ME'][0], SetEnv)) self.assertEqual(modifications['UNSET_ME'][0].value, 'overridden') - self.assertEqual(len(modifications['PATH_LIST']), 1) - self.assertTrue(isinstance(modifications['PATH_LIST'][0], SetEnv)) - self.assertEqual( - modifications['PATH_LIST'][0].value, - '/path/first:/path/second:/path/third:/path/fourth' + self.assertEqual(len(modifications['PATH_LIST']), 3) + self.assertTrue( + isinstance(modifications['PATH_LIST'][0], RemovePath) ) - - # TODO : with reference to the TODO in spack/environment.py - # TODO : remove the above and insert - # self.assertEqual(len(modifications['PATH_LIST']), 2) - # self.assertTrue( - # isinstance(modifications['PATH_LIST'][0], PrependPath) - # ) - # self.assertEqual(modifications['PATH_LIST'][0].value, '/path/first') - # self.assertTrue( - # isinstance(modifications['PATH_LIST'][1], AppendPath) - # ) - # self.assertEqual(modifications['PATH_LIST'][1].value, '/path/fourth') + self.assertEqual(modifications['PATH_LIST'][0].value, '/path/third') + self.assertTrue( + isinstance(modifications['PATH_LIST'][1], AppendPath) + ) + self.assertEqual(modifications['PATH_LIST'][1].value, '/path/fourth') + self.assertTrue( + isinstance(modifications['PATH_LIST'][2], PrependPath) + ) + self.assertEqual(modifications['PATH_LIST'][2].value, '/path/first') From b89bcdff62e10223a737c0ac26feb996c8202941 Mon Sep 17 00:00:00 2001 From: "Kelly (KT) Thompson" Date: Sun, 12 Jun 2016 19:48:45 -0600 Subject: [PATCH 342/481] Make Boost an optional feature of Trilinos. I have been unable to build Boost with some toolset/platform combinations. The users I support don't need the Boost features in Trilinos, so I have been manually hacking trilinos/package.py to disable boost. I would like to propose that these changes be added to the released version. + Add boost as a variant, default to true so there is no change to existing behavior. + Move the CMake configure options related to Boost into a conditional block that is triggered by "if '+boost' in spec". --- .../repos/builtin/packages/trilinos/package.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/var/spack/repos/builtin/packages/trilinos/package.py b/var/spack/repos/builtin/packages/trilinos/package.py index 1eaec86405..28c0bea7c6 100644 --- a/var/spack/repos/builtin/packages/trilinos/package.py +++ b/var/spack/repos/builtin/packages/trilinos/package.py @@ -58,11 +58,12 @@ class Trilinos(Package): variant('python', default=False, description='Build python wrappers') variant('shared', default=True, description='Enables the build of shared libraries') variant('debug', default=False, description='Builds a debug version of the libraries') + variant('boost', default=True, description='Compile with Boost') # Everything should be compiled with -fpic depends_on('blas') depends_on('lapack') - depends_on('boost') + depends_on('boost', when='+boost') depends_on('matio') depends_on('glm') depends_on('swig') @@ -121,9 +122,6 @@ def install(self, spec, prefix): '-DTPL_ENABLE_LAPACK=ON', '-DLAPACK_LIBRARY_NAMES=lapack', '-DLAPACK_LIBRARY_DIRS=%s' % spec['lapack'].prefix, - '-DTPL_ENABLE_Boost:BOOL=ON', - '-DBoost_INCLUDE_DIRS:PATH=%s' % spec['boost'].prefix.include, - '-DBoost_LIBRARY_DIRS:PATH=%s' % spec['boost'].prefix.lib, '-DTrilinos_ENABLE_EXPLICIT_INSTANTIATION:BOOL=ON', '-DTrilinos_ENABLE_CXX11:BOOL=ON', '-DTPL_ENABLE_Netcdf:BOOL=ON', @@ -131,6 +129,14 @@ def install(self, spec, prefix): '-DTPL_ENABLE_HDF5:BOOL=%s' % ('ON' if '+hdf5' in spec else 'OFF'), ]) + if '+boost' in spec: + options.extend([ + '-DTPL_ENABLE_Boost:BOOL=ON', + '-DBoost_INCLUDE_DIRS:PATH=%s' % spec['boost'].prefix.include, + '-DBoost_LIBRARY_DIRS:PATH=%s' % spec['boost'].prefix.lib + ]) + else: + options.extend(['-DTPL_ENABLE_Boost:BOOL=OFF']) # Fortran lib libgfortran = os.path.dirname (os.popen('%s --print-file-name libgfortran.a' % join_path(mpi_bin,'mpif90') ).read()) options.extend([ From 31d2c386875364a9524727279575955ec32972cd Mon Sep 17 00:00:00 2001 From: "Kelly (KT) Thompson" Date: Sun, 12 Jun 2016 19:56:29 -0600 Subject: [PATCH 343/481] Update formatting to meet code style requirements. --- .../builtin/packages/trilinos/package.py | 193 ++++++++++-------- 1 file changed, 113 insertions(+), 80 deletions(-) diff --git a/var/spack/repos/builtin/packages/trilinos/package.py b/var/spack/repos/builtin/packages/trilinos/package.py index 28c0bea7c6..6913d79dcc 100644 --- a/var/spack/repos/builtin/packages/trilinos/package.py +++ b/var/spack/repos/builtin/packages/trilinos/package.py @@ -23,18 +23,23 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## from spack import * -import os, sys, glob +import os +import sys -# Trilinos is complicated to build, as an inspiration a couple of links to other repositories which build it: +# Trilinos is complicated to build, as an inspiration a couple of links to +# other repositories which build it: # https://github.com/hpcugent/easybuild-easyblocks/blob/master/easybuild/easyblocks/t/trilinos.py#L111 # https://github.com/koecher/candi/blob/master/deal.II-toolchain/packages/trilinos.package # https://gitlab.com/configurations/cluster-config/blob/master/trilinos.sh -# https://github.com/Homebrew/homebrew-science/blob/master/trilinos.rb -# and some relevant documentation/examples: +# https://github.com/Homebrew/homebrew-science/blob/master/trilinos.rb and some +# relevant documentation/examples: # https://github.com/trilinos/Trilinos/issues/175 + + class Trilinos(Package): - """The Trilinos Project is an effort to develop algorithms and enabling technologies within an object-oriented - software framework for the solution of large-scale, complex multi-physics engineering and scientific problems. + """The Trilinos Project is an effort to develop algorithms and enabling + technologies within an object-oriented software framework for the solution + of large-scale, complex multi-physics engineering and scientific problems. A unique design feature of Trilinos is its focus on packages. """ homepage = "https://trilinos.org/" @@ -54,7 +59,8 @@ class Trilinos(Package): variant('hypre', default=True, description='Compile with Hypre preconditioner') variant('hdf5', default=True, description='Compile with HDF5') variant('suite-sparse', default=True, description='Compile with SuiteSparse solvers') - # not everyone has py-numpy activated, keep it disabled by default to avoid configure errors + # not everyone has py-numpy activated, keep it disabled by default to avoid + # configure errors variant('python', default=False, description='Build python wrappers') variant('shared', default=True, description='Enables the build of shared libraries') variant('debug', default=False, description='Builds a debug version of the libraries') @@ -67,37 +73,37 @@ class Trilinos(Package): depends_on('matio') depends_on('glm') depends_on('swig') - depends_on('metis@5:',when='+metis') - depends_on('suite-sparse',when='+suite-sparse') + depends_on('metis@5:', when='+metis') + depends_on('suite-sparse', when='+suite-sparse') # MPI related dependencies depends_on('mpi') depends_on('netcdf+mpi') - depends_on('parmetis',when='+metis') - # Trilinos' Tribits config system is limited which makes it - # very tricky to link Amesos with static MUMPS, see + depends_on('parmetis', when='+metis') + # Trilinos' Tribits config system is limited which makes it very tricky to + # link Amesos with static MUMPS, see # https://trilinos.org/docs/dev/packages/amesos2/doc/html/classAmesos2_1_1MUMPS.html - # One could work it out by getting linking flags from mpif90 --showme:link (or alike) - # and adding results to -DTrilinos_EXTRA_LINK_FLAGS - # together with Blas and Lapack and ScaLAPACK and Blacs and -lgfortran and - # it may work at the end. But let's avoid all this by simply using shared libs - depends_on('mumps@5.0:+mpi+shared',when='+mumps') - depends_on('scalapack',when='+mumps') - depends_on('superlu-dist',when='+superlu-dist') - depends_on('hypre~internal-superlu',when='+hypre') - depends_on('hdf5+mpi',when='+hdf5') - - depends_on('python',when='+python') + # One could work it out by getting linking flags from mpif90 --showme:link + # (or alike) and adding results to -DTrilinos_EXTRA_LINK_FLAGS together + # with Blas and Lapack and ScaLAPACK and Blacs and -lgfortran and it may + # work at the end. But let's avoid all this by simply using shared libs + depends_on('mumps@5.0:+mpi+shared', when='+mumps') + depends_on('scalapack', when='+mumps') + depends_on('superlu-dist', when='+superlu-dist') + depends_on('hypre~internal-superlu', when='+hypre') + depends_on('hdf5+mpi', when='+hdf5') + depends_on('python', when='+python') patch('umfpack_from_suitesparse.patch') # check that the combination of variants makes sense def variants_check(self): if '+superlu-dist' in self.spec and self.spec.satisfies('@:11.4.3'): - # For Trilinos v11 we need to force SuperLUDist=OFF, - # since only the deprecated SuperLUDist v3.3 together with an Amesos patch - # is working. - raise RuntimeError('The superlu-dist variant can only be used with Trilinos @12.0.1:') + # For Trilinos v11 we need to force SuperLUDist=OFF, since only the + # deprecated SuperLUDist v3.3 together with an Amesos patch is + # working. + raise RuntimeError('The superlu-dist variant can only be used' + + ' with Trilinos @12.0.1:') def install(self, spec, prefix): self.variants_check() @@ -107,27 +113,32 @@ def install(self, spec, prefix): options.extend(std_cmake_args) mpi_bin = spec['mpi'].prefix.bin - options.extend(['-DTrilinos_ENABLE_ALL_PACKAGES:BOOL=ON', - '-DTrilinos_ENABLE_ALL_OPTIONAL_PACKAGES:BOOL=ON', - '-DTrilinos_VERBOSE_CONFIGURE:BOOL=OFF', - '-DTrilinos_ENABLE_TESTS:BOOL=OFF', - '-DTrilinos_ENABLE_EXAMPLES:BOOL=OFF', - '-DCMAKE_BUILD_TYPE:STRING=%s' % ('DEBUG' if '+debug' in spec else 'RELEASE'), - '-DBUILD_SHARED_LIBS:BOOL=%s' % ('ON' if '+shared' in spec else 'OFF'), - '-DTPL_ENABLE_MPI:BOOL=ON', - '-DMPI_BASE_DIR:PATH=%s' % spec['mpi'].prefix, - '-DTPL_ENABLE_BLAS=ON', - '-DBLAS_LIBRARY_NAMES=blas', # FIXME: don't hardcode names - '-DBLAS_LIBRARY_DIRS=%s' % spec['blas'].prefix.lib, - '-DTPL_ENABLE_LAPACK=ON', - '-DLAPACK_LIBRARY_NAMES=lapack', - '-DLAPACK_LIBRARY_DIRS=%s' % spec['lapack'].prefix, - '-DTrilinos_ENABLE_EXPLICIT_INSTANTIATION:BOOL=ON', - '-DTrilinos_ENABLE_CXX11:BOOL=ON', - '-DTPL_ENABLE_Netcdf:BOOL=ON', - '-DTPL_ENABLE_HYPRE:BOOL=%s' % ('ON' if '+hypre' in spec else 'OFF'), - '-DTPL_ENABLE_HDF5:BOOL=%s' % ('ON' if '+hdf5' in spec else 'OFF'), - ]) + options.extend([ + '-DTrilinos_ENABLE_ALL_PACKAGES:BOOL=ON', + '-DTrilinos_ENABLE_ALL_OPTIONAL_PACKAGES:BOOL=ON', + '-DTrilinos_VERBOSE_CONFIGURE:BOOL=OFF', + '-DTrilinos_ENABLE_TESTS:BOOL=OFF', + '-DTrilinos_ENABLE_EXAMPLES:BOOL=OFF', + '-DCMAKE_BUILD_TYPE:STRING=%s' % ( + 'DEBUG' if '+debug' in spec else 'RELEASE'), + '-DBUILD_SHARED_LIBS:BOOL=%s' % ( + 'ON' if '+shared' in spec else 'OFF'), + '-DTPL_ENABLE_MPI:BOOL=ON', + '-DMPI_BASE_DIR:PATH=%s' % spec['mpi'].prefix, + '-DTPL_ENABLE_BLAS=ON', + '-DBLAS_LIBRARY_NAMES=blas', # FIXME: don't hardcode names + '-DBLAS_LIBRARY_DIRS=%s' % spec['blas'].prefix.lib, + '-DTPL_ENABLE_LAPACK=ON', + '-DLAPACK_LIBRARY_NAMES=lapack', + '-DLAPACK_LIBRARY_DIRS=%s' % spec['lapack'].prefix, + '-DTrilinos_ENABLE_EXPLICIT_INSTANTIATION:BOOL=ON', + '-DTrilinos_ENABLE_CXX11:BOOL=ON', + '-DTPL_ENABLE_Netcdf:BOOL=ON', + '-DTPL_ENABLE_HYPRE:BOOL=%s' % ( + 'ON' if '+hypre' in spec else 'OFF'), + '-DTPL_ENABLE_HDF5:BOOL=%s' % ( + 'ON' if '+hdf5' in spec else 'OFF'), + ]) if '+boost' in spec: options.extend([ @@ -137,29 +148,40 @@ def install(self, spec, prefix): ]) else: options.extend(['-DTPL_ENABLE_Boost:BOOL=OFF']) + # Fortran lib - libgfortran = os.path.dirname (os.popen('%s --print-file-name libgfortran.a' % join_path(mpi_bin,'mpif90') ).read()) + libgfortran = os.path.dirname(os.popen( + '%s --print-file-name libgfortran.a' % + join_path(mpi_bin, 'mpif90')).read()) options.extend([ - '-DTrilinos_EXTRA_LINK_FLAGS:STRING=-L%s/ -lgfortran' % libgfortran, + '-DTrilinos_EXTRA_LINK_FLAGS:STRING=-L%s/ -lgfortran' % ( + libgfortran), '-DTrilinos_ENABLE_Fortran=ON' ]) # for build-debug only: - #options.extend([ - # '-DCMAKE_VERBOSE_MAKEFILE:BOOL=TRUE' - #]) + # options.extend([ + # '-DCMAKE_VERBOSE_MAKEFILE:BOOL=TRUE' + # ]) # suite-sparse related if '+suite-sparse' in spec: options.extend([ - '-DTPL_ENABLE_Cholmod:BOOL=OFF', # FIXME: Trilinos seems to be looking for static libs only, patch CMake TPL file? - #'-DTPL_ENABLE_Cholmod:BOOL=ON', - #'-DCholmod_LIBRARY_DIRS:PATH=%s' % spec['suite-sparse'].prefix.lib, - #'-DCholmod_INCLUDE_DIRS:PATH=%s' % spec['suite-sparse'].prefix.include, + # FIXME: Trilinos seems to be looking for static libs only, + # patch CMake TPL file? + '-DTPL_ENABLE_Cholmod:BOOL=OFF', + # '-DTPL_ENABLE_Cholmod:BOOL=ON', + # '-DCholmod_LIBRARY_DIRS:PATH=%s' % ( + # spec['suite-sparse'].prefix.lib, + # '-DCholmod_INCLUDE_DIRS:PATH=%s' % ( + # spec['suite-sparse'].prefix.include, '-DTPL_ENABLE_UMFPACK:BOOL=ON', - '-DUMFPACK_LIBRARY_DIRS:PATH=%s' % spec['suite-sparse'].prefix.lib, - '-DUMFPACK_INCLUDE_DIRS:PATH=%s' % spec['suite-sparse'].prefix.include, - '-DUMFPACK_LIBRARY_NAMES=umfpack;amd;colamd;cholmod;suitesparseconfig' + '-DUMFPACK_LIBRARY_DIRS:PATH=%s' % ( + spec['suite-sparse'].prefix.lib), + '-DUMFPACK_INCLUDE_DIRS:PATH=%s' % ( + spec['suite-sparse'].prefix.include), + '-DUMFPACK_LIBRARY_NAMES=umfpack;amd;colamd;cholmod;' + + 'suitesparseconfig' ]) else: options.extend([ @@ -175,9 +197,11 @@ def install(self, spec, prefix): '-DMETIS_LIBRARY_NAMES=metis', '-DTPL_METIS_INCLUDE_DIRS=%s' % spec['metis'].prefix.include, '-DTPL_ENABLE_ParMETIS:BOOL=ON', - '-DParMETIS_LIBRARY_DIRS=%s;%s' % (spec['parmetis'].prefix.lib,spec['metis'].prefix.lib), + '-DParMETIS_LIBRARY_DIRS=%s;%s' % ( + spec['parmetis'].prefix.lib, spec['metis'].prefix.lib), '-DParMETIS_LIBRARY_NAMES=parmetis;metis', - '-DTPL_ParMETIS_INCLUDE_DIRS=%s' % spec['parmetis'].prefix.include + '-DTPL_ParMETIS_INCLUDE_DIRS=%s' % ( + spec['parmetis'].prefix.include) ]) else: options.extend([ @@ -190,11 +214,14 @@ def install(self, spec, prefix): options.extend([ '-DTPL_ENABLE_MUMPS:BOOL=ON', '-DMUMPS_LIBRARY_DIRS=%s' % spec['mumps'].prefix.lib, - '-DMUMPS_LIBRARY_NAMES=dmumps;mumps_common;pord', # order is important! + # order is important! + '-DMUMPS_LIBRARY_NAMES=dmumps;mumps_common;pord', '-DTPL_ENABLE_SCALAPACK:BOOL=ON', - '-DSCALAPACK_LIBRARY_NAMES=scalapack' # FIXME: for MKL it's mkl_scalapack_lp64;mkl_blacs_mpich_lp64 + # FIXME: for MKL it's mkl_scalapack_lp64;mkl_blacs_mpich_lp64 + '-DSCALAPACK_LIBRARY_NAMES=scalapack' ]) - # see https://github.com/trilinos/Trilinos/blob/master/packages/amesos/README-MUMPS + # see + # https://github.com/trilinos/Trilinos/blob/master/packages/amesos/README-MUMPS cxx_flags.extend([ '-DMUMPS_5_0' ]) @@ -207,16 +234,20 @@ def install(self, spec, prefix): # superlu-dist: if '+superlu-dist' in spec: # Amesos, conflicting types of double and complex SLU_D - # see https://trilinos.org/pipermail/trilinos-users/2015-March/004731.html - # and https://trilinos.org/pipermail/trilinos-users/2015-March/004802.html + # see + # https://trilinos.org/pipermail/trilinos-users/2015-March/004731.html + # and + # https://trilinos.org/pipermail/trilinos-users/2015-March/004802.html options.extend([ '-DTeuchos_ENABLE_COMPLEX:BOOL=OFF', '-DKokkosTSQR_ENABLE_Complex:BOOL=OFF' ]) options.extend([ '-DTPL_ENABLE_SuperLUDist:BOOL=ON', - '-DSuperLUDist_LIBRARY_DIRS=%s' % spec['superlu-dist'].prefix.lib, - '-DSuperLUDist_INCLUDE_DIRS=%s' % spec['superlu-dist'].prefix.include + '-DSuperLUDist_LIBRARY_DIRS=%s' % + spec['superlu-dist'].prefix.lib, + '-DSuperLUDist_INCLUDE_DIRS=%s' % + spec['superlu-dist'].prefix.include ]) if spec.satisfies('^superlu-dist@4.0:'): options.extend([ @@ -227,7 +258,6 @@ def install(self, spec, prefix): '-DTPL_ENABLE_SuperLUDist:BOOL=OFF', ]) - # python if '+python' in spec: options.extend([ @@ -254,23 +284,26 @@ def install(self, spec, prefix): '-DTrilinos_ENABLE_FEI=OFF' ]) - with working_dir('spack-build', create=True): cmake('..', *options) make() make('install') - # When trilinos is built with Python, libpytrilinos is included through - # cmake configure files. Namely, Trilinos_LIBRARIES in TrilinosConfig.cmake - # contains pytrilinos. This leads to a run-time error: - # Symbol not found: _PyBool_Type - # and prevents Trilinos to be used in any C++ code, which links executable - # against the libraries listed in Trilinos_LIBRARIES. - # See https://github.com/Homebrew/homebrew-science/issues/2148#issuecomment-103614509 + # When trilinos is built with Python, libpytrilinos is included + # through cmake configure files. Namely, Trilinos_LIBRARIES in + # TrilinosConfig.cmake contains pytrilinos. This leads to a + # run-time error: Symbol not found: _PyBool_Type and prevents + # Trilinos to be used in any C++ code, which links executable + # against the libraries listed in Trilinos_LIBRARIES. See + # https://github.com/Homebrew/homebrew-science/issues/2148#issuecomment-103614509 # A workaround it to remove PyTrilinos from the COMPONENTS_LIST : if '+python' in self.spec: - filter_file(r'(SET\(COMPONENTS_LIST.*)(PyTrilinos;)(.*)', (r'\1\3'), '%s/cmake/Trilinos/TrilinosConfig.cmake' % prefix.lib) + filter_file(r'(SET\(COMPONENTS_LIST.*)(PyTrilinos;)(.*)', + (r'\1\3'), + '%s/cmake/Trilinos/TrilinosConfig.cmake' % + prefix.lib) - # The shared libraries are not installed correctly on Darwin; correct this + # The shared libraries are not installed correctly on Darwin; + # correct this if (sys.platform == 'darwin') and ('+shared' in spec): fix_darwin_install_name(prefix.lib) From 1601cf1b148bf326f84ebaeb142f2951c99fb788 Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Sun, 12 Jun 2016 21:46:11 -0500 Subject: [PATCH 344/481] Patch binutils This commit adds a patch to update symbol versions from linker scripts. This is seen with libmysqlclient.so which has to maintain compatibility across two versions. Only one can be the default. https://sourceware.org/git/gitweb.cgi?p=binutils-gdb.git;a=commitdiff;h=544ddf9322b1b83982e5cb84a54d084ee7e718ea --- .../builtin/packages/binutils/package.py | 1 + .../binutils/update_symbol-2.26.patch | 104 ++++++++++++++++++ 2 files changed, 105 insertions(+) create mode 100644 var/spack/repos/builtin/packages/binutils/update_symbol-2.26.patch diff --git a/var/spack/repos/builtin/packages/binutils/package.py b/var/spack/repos/builtin/packages/binutils/package.py index 9e4cc98ae6..7ecb219df5 100644 --- a/var/spack/repos/builtin/packages/binutils/package.py +++ b/var/spack/repos/builtin/packages/binutils/package.py @@ -46,6 +46,7 @@ class Binutils(Package): patch('binutilskrell-2.24.patch', when='@2.24+krellpatch') patch('cr16.patch') + patch('update_symbol-2.26.patch', when='@2.26') variant('libiberty', default=False, description='Also install libiberty.') diff --git a/var/spack/repos/builtin/packages/binutils/update_symbol-2.26.patch b/var/spack/repos/builtin/packages/binutils/update_symbol-2.26.patch new file mode 100644 index 0000000000..2601f63a6b --- /dev/null +++ b/var/spack/repos/builtin/packages/binutils/update_symbol-2.26.patch @@ -0,0 +1,104 @@ +From 544ddf9322b1b83982e5cb84a54d084ee7e718ea Mon Sep 17 00:00:00 2001 +From: H.J. Lu +Date: Wed, 24 Feb 2016 15:13:35 -0800 +Subject: [PATCH] Update symbol version for symbol from linker script + +We need to update symbol version for symbols from linker script. + +Backport from master + +bfd/ + + PR ld/19698 + * elflink.c (bfd_elf_record_link_assignment): Set versioned if + symbol version is unknown. + +ld/ + + PR ld/19698 + * testsuite/ld-elf/pr19698.d: New file. + * testsuite/ld-elf/pr19698.s: Likewise. + * testsuite/ld-elf/pr19698.t: Likewise. +--- + bfd/ChangeLog | 9 +++++++++ + bfd/elflink.c | 13 +++++++++++++ + ld/ChangeLog | 10 ++++++++++ + ld/testsuite/ld-elf/pr19698.d | 10 ++++++++++ + ld/testsuite/ld-elf/pr19698.s | 5 +++++ + ld/testsuite/ld-elf/pr19698.t | 11 +++++++++++ + 6 files changed, 58 insertions(+), 0 deletions(-) + create mode 100644 ld/testsuite/ld-elf/pr19698.d + create mode 100644 ld/testsuite/ld-elf/pr19698.s + create mode 100644 ld/testsuite/ld-elf/pr19698.t + +diff --git a/bfd/elflink.c b/bfd/elflink.c +index ae8d148..8fcaadd 100644 +--- a/bfd/elflink.c ++++ b/bfd/elflink.c +@@ -555,6 +555,19 @@ bfd_elf_record_link_assignment (bfd *output_bfd, + if (h == NULL) + return provide; + ++ if (h->versioned == unknown) ++ { ++ /* Set versioned if symbol version is unknown. */ ++ char *version = strrchr (name, ELF_VER_CHR); ++ if (version) ++ { ++ if (version > name && version[-1] != ELF_VER_CHR) ++ h->versioned = versioned_hidden; ++ else ++ h->versioned = versioned; ++ } ++ } ++ + switch (h->root.type) + { + case bfd_link_hash_defined: +diff --git a/ld/testsuite/ld-elf/pr19698.d b/ld/testsuite/ld-elf/pr19698.d +new file mode 100644 +index 0000000..a39f67a +--- /dev/null ++++ b/ld/testsuite/ld-elf/pr19698.d +@@ -0,0 +1,10 @@ ++#ld: -shared $srcdir/$subdir/pr19698.t ++#readelf : --dyn-syms --wide ++#target: *-*-linux* *-*-gnu* *-*-solaris* ++ ++Symbol table '\.dynsym' contains [0-9]+ entries: ++#... ++ +[0-9]+: +[0-9a-f]+ +[0-9a-f]+ +FUNC +GLOBAL +DEFAULT +[0-9]+ +foo@VERS.1 ++#... ++ +[0-9]+: +[0-9a-f]+ +[0-9a-f]+ +FUNC +GLOBAL +DEFAULT +[0-9]+ +foo@@VERS.2 ++#pass +diff --git a/ld/testsuite/ld-elf/pr19698.s b/ld/testsuite/ld-elf/pr19698.s +new file mode 100644 +index 0000000..875dca4 +--- /dev/null ++++ b/ld/testsuite/ld-elf/pr19698.s +@@ -0,0 +1,5 @@ ++ .text ++ .globl foo ++ .type foo, %function ++foo: ++ .byte 0 +diff --git a/ld/testsuite/ld-elf/pr19698.t b/ld/testsuite/ld-elf/pr19698.t +new file mode 100644 +index 0000000..09d9125 +--- /dev/null ++++ b/ld/testsuite/ld-elf/pr19698.t +@@ -0,0 +1,11 @@ ++"foo@VERS.1" = foo; ++ ++VERSION { ++VERS.2 { ++ global: ++ foo; ++}; ++ ++VERS.1 { ++}; ++} +-- +1.7.1 + From 8ff7cbb9cef11181df46c626c298299321ba43c9 Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Sun, 12 Jun 2016 21:57:03 -0500 Subject: [PATCH 345/481] Run binutils/package.py through autopep8. --- var/spack/repos/builtin/packages/binutils/package.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/var/spack/repos/builtin/packages/binutils/package.py b/var/spack/repos/builtin/packages/binutils/package.py index 7ecb219df5..15e6f1ecc1 100644 --- a/var/spack/repos/builtin/packages/binutils/package.py +++ b/var/spack/repos/builtin/packages/binutils/package.py @@ -24,11 +24,12 @@ ############################################################################## from spack import * + class Binutils(Package): """GNU binutils, which contain the linker, assembler, objdump and others""" - homepage = "http://www.gnu.org/software/binutils/" - url="https://ftp.gnu.org/gnu/binutils/binutils-2.25.tar.bz2" + homepage = "http://www.gnu.org/software/binutils/" + url = "https://ftp.gnu.org/gnu/binutils/binutils-2.25.tar.bz2" version('2.26', '64146a0faa3b411ba774f47d41de239f') version('2.25', 'd9f3303f802a5b6b0bb73a335ab89d66') @@ -40,8 +41,10 @@ class Binutils(Package): depends_on('flex') depends_on('bison') - # Add a patch that creates binutils libiberty_pic.a which is preferred by OpenSpeedShop and cbtf-krell - variant('krellpatch', default=False, description="build with openspeedshop based patch.") + # Add a patch that creates binutils libiberty_pic.a which is preferred by + # OpenSpeedShop and cbtf-krell + variant('krellpatch', default=False, + description="build with openspeedshop based patch.") variant('gold', default=True, description="build the gold linker") patch('binutilskrell-2.24.patch', when='@2.24+krellpatch') From bdb9af63ed2128d7cad25f3788eb9ab7e7a43e1b Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Sun, 12 Jun 2016 21:58:41 -0500 Subject: [PATCH 346/481] Add version 9.5.3 and add readline as a dependency. --- var/spack/repos/builtin/packages/postgresql/package.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/postgresql/package.py b/var/spack/repos/builtin/packages/postgresql/package.py index 9362234881..a3a3bfa64a 100644 --- a/var/spack/repos/builtin/packages/postgresql/package.py +++ b/var/spack/repos/builtin/packages/postgresql/package.py @@ -34,8 +34,10 @@ class Postgresql(Package): url = "http://ftp.postgresql.org/pub/source/v9.3.4/postgresql-9.3.4.tar.bz2" version('9.3.4', 'd0a41f54c377b2d2fab4a003b0dac762') + version('9.5.3', '3f0c388566c688c82b01a0edf1e6b7a0') - depends_on("openssl") + depends_on('openssl') + depends_on('readline') def install(self, spec, prefix): configure("--prefix=%s" % prefix, From 4dcb15cf6d7ae2f6874972f1f9c339c056875500 Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Sun, 12 Jun 2016 22:00:49 -0500 Subject: [PATCH 347/481] Ran through autopep8 and cleaned up. --- .../repos/builtin/packages/postgresql/package.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/var/spack/repos/builtin/packages/postgresql/package.py b/var/spack/repos/builtin/packages/postgresql/package.py index a3a3bfa64a..caf3768362 100644 --- a/var/spack/repos/builtin/packages/postgresql/package.py +++ b/var/spack/repos/builtin/packages/postgresql/package.py @@ -24,12 +24,13 @@ ############################################################################## from spack import * + class Postgresql(Package): - """PostgreSQL is a powerful, open source object-relational - database system. It has more than 15 years of active - development and a proven architecture that has earned it a - strong reputation for reliability, data integrity, and - correctness.""" + """PostgreSQL is a powerful, open source object-relational database system. + It has more than 15 years of active development and a proven architecture + that has earned it a strong reputation for reliability, data integrity, and + correctness.""" + homepage = "http://www.postgresql.org/" url = "http://ftp.postgresql.org/pub/source/v9.3.4/postgresql-9.3.4.tar.bz2" From 7727f6fc7e0938db3773cd1aeeb44a18f4d691ab Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Sun, 12 Jun 2016 22:02:46 -0500 Subject: [PATCH 348/481] New package - libaio --- .../repos/builtin/packages/libaio/package.py | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 var/spack/repos/builtin/packages/libaio/package.py diff --git a/var/spack/repos/builtin/packages/libaio/package.py b/var/spack/repos/builtin/packages/libaio/package.py new file mode 100644 index 0000000000..735ced047b --- /dev/null +++ b/var/spack/repos/builtin/packages/libaio/package.py @@ -0,0 +1,43 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class Libaio(Package): + """This is the linux native Asynchronous I/O interface library.""" + + homepage = "https://git.fedorahosted.org/cgit/libaio.git" + url = "https://git.fedorahosted.org/cgit/libaio.git/snapshot/libaio-0.3.110-1.tar.gz" + + version('0.3.110-1', 'eb6b1b435afadb5b80c5dd80984249f6') + + def install(self, spec, prefix): + # libaio is not supported on OS X + if spec.satisfies('arch=darwin-x86_64'): + # create a dummy directory + mkdir(prefix.lib) + return + + make('prefix={0}'.format(prefix), 'install') From 5d3965c7a6276ca328a7ff86f5084bb51806dfd3 Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Sun, 12 Jun 2016 22:03:24 -0500 Subject: [PATCH 349/481] New package - unixODBC --- .../builtin/packages/unixODBC/package.py | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 var/spack/repos/builtin/packages/unixODBC/package.py diff --git a/var/spack/repos/builtin/packages/unixODBC/package.py b/var/spack/repos/builtin/packages/unixODBC/package.py new file mode 100644 index 0000000000..15de127b7e --- /dev/null +++ b/var/spack/repos/builtin/packages/unixODBC/package.py @@ -0,0 +1,42 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class Unixodbc(Package): + """ODBC is an open specification for providing application developers with + a predictable API with which to access Data Sources. Data Sources include + SQL Servers and any Data Source with an ODBC Driver.""" + + homepage = "http://www.unixodbc.org/" + url = "ftp://ftp.unixodbc.org/pub/unixODBC/unixODBC-2.3.4.tar.gz" + + version('2.3.4', 'bd25d261ca1808c947cb687e2034be81') + + def install(self, spec, prefix): + configure('--prefix={0}'.format(prefix)) + + make() + make('install') From b05a56564886bb54b147a83dea9d91ead04a2a0e Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Sun, 12 Jun 2016 22:04:09 -0500 Subject: [PATCH 350/481] New package - mariadb --- .../repos/builtin/packages/mariadb/package.py | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 var/spack/repos/builtin/packages/mariadb/package.py diff --git a/var/spack/repos/builtin/packages/mariadb/package.py b/var/spack/repos/builtin/packages/mariadb/package.py new file mode 100644 index 0000000000..d9df200d02 --- /dev/null +++ b/var/spack/repos/builtin/packages/mariadb/package.py @@ -0,0 +1,59 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class Mariadb(Package): + """MariaDB turns data into structured information in a wide array of + applications, ranging from banking to websites. It is an enhanced, drop-in + replacement for MySQL. MariaDB is used because it is fast, scalable and + robust, with a rich ecosystem of storage engines, plugins and many other + tools make it very versatile for a wide variety of use cases.""" + + homepage = "https://mariadb.org/about/" + url = "https://downloads.mariadb.org/f/mariadb-10.1.14/source/mariadb-10.1.14.tar.gz" + + version('10.1.14', '294925531e0fd2f0461e3894496a5adc') + version('5.5.49', '67b5a499a5f158b2a586e6e3bfb4f304') + + variant('nonblocking', default=True, description='Allow non blocking ' + 'operations in the mariadb client library.') + + depends_on('boost') + depends_on('cmake') + depends_on('jemalloc') + depends_on('libaio') + depends_on('libedit') + depends_on('libevent', when='+nonblocking') + depends_on('ncurses') + depends_on('zlib') + + def install(self, spec, prefix): + with working_dir('spack-build', create=True): + + cmake('..', *std_cmake_args) + + make() + make('install') From a7a640bfb800599b433835cd06aeac9851e866f3 Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Sun, 12 Jun 2016 22:04:37 -0500 Subject: [PATCH 351/481] New package - rDBI --- .../repos/builtin/packages/r-DBI/package.py | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 var/spack/repos/builtin/packages/r-DBI/package.py diff --git a/var/spack/repos/builtin/packages/r-DBI/package.py b/var/spack/repos/builtin/packages/r-DBI/package.py new file mode 100644 index 0000000000..a1e06d5100 --- /dev/null +++ b/var/spack/repos/builtin/packages/r-DBI/package.py @@ -0,0 +1,43 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class RDbi(Package): + """A database interface definition for communication between R and + relational database management systems. All classes in this package are + virtual and need to be extended by the various R/DBMS implementations.""" + + homepage = "https://github.com/rstats-db/DBI" + url = "https://cran.r-project.org/src/contrib/DBI_0.4-1.tar.gz" + list_url = "https://cran.r-project.org/src/contrib/Archive/DBI" + + version('0.4-1', 'c7ee8f1c5037c2284e99c62698d0f087') + + extends('R') + + def install(self, spec, prefix): + R('CMD', 'INSTALL', '--library={0}'.format(self.module.r_lib_dir), + self.stage.source_path) From bfb0985c38c480644a69eaac3262e122915d9884 Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Sun, 12 Jun 2016 22:05:10 -0500 Subject: [PATCH 352/481] New package - r-RMySQL --- .../builtin/packages/r-RMySQL/package.py | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 var/spack/repos/builtin/packages/r-RMySQL/package.py diff --git a/var/spack/repos/builtin/packages/r-RMySQL/package.py b/var/spack/repos/builtin/packages/r-RMySQL/package.py new file mode 100644 index 0000000000..774c166f8b --- /dev/null +++ b/var/spack/repos/builtin/packages/r-RMySQL/package.py @@ -0,0 +1,44 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class RRmysql(Package): + """Implements 'DBI' Interface to 'MySQL' and 'MariaDB' Databases.""" + + homepage = "https://github.com/rstats-db/rmysql" + url = "https://cran.r-project.org/src/contrib/RMySQL_0.10.9.tar.gz" + list_url = "https://cran.r-project.org/src/contrib/Archive/RMySQL" + + version('0.10.9', '3628200a1864ac3005cfd55cc7cde17a') + + extends('R') + + depends_on('r-DBI') + depends_on('mariadb') + + def install(self, spec, prefix): + R('CMD', 'INSTALL', '--library={0}'.format(self.module.r_lib_dir), + self.stage.source_path) From 77e8096a6fb1f7531d0654b10ce616c04cee9187 Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Sun, 12 Jun 2016 22:05:26 -0500 Subject: [PATCH 353/481] New package - r-RODBC --- .../repos/builtin/packages/r-RODBC/package.py | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 var/spack/repos/builtin/packages/r-RODBC/package.py diff --git a/var/spack/repos/builtin/packages/r-RODBC/package.py b/var/spack/repos/builtin/packages/r-RODBC/package.py new file mode 100644 index 0000000000..9cc9aebd9a --- /dev/null +++ b/var/spack/repos/builtin/packages/r-RODBC/package.py @@ -0,0 +1,43 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class RRodbc(Package): + """An ODBC database interface.""" + + homepage = "https://cran.rstudio.com/web/packages/RODBC/" + url = "https://cran.rstudio.com/src/contrib/RODBC_1.3-13.tar.gz" + list_url = "https://cran.rstudio.com/src/contrib/Archive/RODBC" + + version('1.3-13', 'c52ef9139c2ed85adc53ad6effa7d68e') + + extends('R') + + depends_on('unixODBC') + + def install(self, spec, prefix): + R('CMD', 'INSTALL', '--library={0}'.format(self.module.r_lib_dir), + self.stage.source_path) From b96b32e9ec8721c5acd3d1d99f7c074ac92debbc Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Sun, 12 Jun 2016 22:05:45 -0500 Subject: [PATCH 354/481] New package - r-RPostgreSQL --- .../builtin/packages/r-RPostgreSQL/package.py | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 var/spack/repos/builtin/packages/r-RPostgreSQL/package.py diff --git a/var/spack/repos/builtin/packages/r-RPostgreSQL/package.py b/var/spack/repos/builtin/packages/r-RPostgreSQL/package.py new file mode 100644 index 0000000000..83c9b85525 --- /dev/null +++ b/var/spack/repos/builtin/packages/r-RPostgreSQL/package.py @@ -0,0 +1,52 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class RRpostgresql(Package): + """Database interface and PostgreSQL driver for R This package provides a + Database Interface (DBI) compliant driver for R to access PostgreSQL + database systems. In order to build and install this package from source, + PostgreSQL itself must be present your system to provide PostgreSQL + functionality via its libraries and header files. These files are provided + as postgresql-devel package under some Linux distributions. On Microsoft + Windows system the attached libpq library source will be used. A wiki and + issue tracking system for the package are available at Google Code at + https://code.google.com/p/rpostgresql/.""" + + homepage = "https://code.google.com/p/rpostgresql/" + url = "https://cran.r-project.org/src/contrib/RPostgreSQL_0.4-1.tar.gz" + list_url = "https://cran.r-project.org/src/contrib/Archive/RPostgreSQL" + + version('0.4-1', 'e7b22e212afbb2cbb88bab937f93e55a') + + extends('R') + + depends_on('r-DBI') + depends_on('postgresql') + + def install(self, spec, prefix): + R('CMD', 'INSTALL', '--library={0}'.format(self.module.r_lib_dir), + self.stage.source_path) From d398ebe21f35684d8aec227904d8fd3155d4ea10 Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Sun, 12 Jun 2016 22:06:06 -0500 Subject: [PATCH 355/481] New package - r-RSQLite --- .../builtin/packages/r-RSQLite/package.py | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 var/spack/repos/builtin/packages/r-RSQLite/package.py diff --git a/var/spack/repos/builtin/packages/r-RSQLite/package.py b/var/spack/repos/builtin/packages/r-RSQLite/package.py new file mode 100644 index 0000000000..9602f6d927 --- /dev/null +++ b/var/spack/repos/builtin/packages/r-RSQLite/package.py @@ -0,0 +1,45 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class RRsqlite(Package): + """This package embeds the SQLite database engine in R and provides an + interface compliant with the DBI package. The source for the SQLite engine + (version 3.8.6) is included.""" + + homepage = "https://github.com/rstats-db/RSQLite" + url = "https://cran.r-project.org/src/contrib/RSQLite_1.0.0.tar.gz" + list_url = "https://cran.r-project.org/src/contrib/Archive/RSQLite" + + version('1.0.0', 'e6cbe2709612b687c13a10d30c7bad45') + + extends('R') + + depends_on('r-DBI') + + def install(self, spec, prefix): + R('CMD', 'INSTALL', '--library={0}'.format(self.module.r_lib_dir), + self.stage.source_path) From 52160bff187a7d33f6e6041417ced2968e9c13ea Mon Sep 17 00:00:00 2001 From: David Boehme Date: Mon, 13 Jun 2016 11:47:28 -0700 Subject: [PATCH 356/481] Use https for github --- var/spack/repos/builtin/packages/caliper/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/caliper/package.py b/var/spack/repos/builtin/packages/caliper/package.py index 4b8fe0d8af..a424c73859 100644 --- a/var/spack/repos/builtin/packages/caliper/package.py +++ b/var/spack/repos/builtin/packages/caliper/package.py @@ -34,7 +34,7 @@ class Caliper(Package): homepage = "https://github.com/LLNL/Caliper" url = "" - version('master', git='ssh://git@github.com:LLNL/Caliper.git') + version('master', git='https://github.com/LLNL/Caliper.git') variant('mpi', default=False, description='Enable MPI function wrappers.') From 427fa7fd0a798c2ae6842f4e7227437e7462e4d4 Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Mon, 13 Jun 2016 17:26:01 -0500 Subject: [PATCH 357/481] New package - rJava --- .../repos/builtin/packages/r-rJava/package.py | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 var/spack/repos/builtin/packages/r-rJava/package.py diff --git a/var/spack/repos/builtin/packages/r-rJava/package.py b/var/spack/repos/builtin/packages/r-rJava/package.py new file mode 100644 index 0000000000..47804bd7fc --- /dev/null +++ b/var/spack/repos/builtin/packages/r-rJava/package.py @@ -0,0 +1,44 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class RRjava(Package): + """Low-level interface to Java VM very much like .C/.Call and friends. + Allows creation of objects, calling methods and accessing fields.""" + + homepage = "http://www.rforge.net/rJava/" + url = "https://cran.r-project.org/src/contrib/rJava_0.9-8.tar.gz" + list_url = "https://cran.r-project.org/src/contrib/Archive/rJava" + + version('0.9-8', '51ae0d690ceed056ebe7c4be71fc6c7a') + + extends('R') + + depends_on('jdk') + + def install(self, spec, prefix): + R('CMD', 'INSTALL', '--library={0}'.format(self.module.r_lib_dir), + self.stage.source_path) From 450b082dafcc92ec38d344d74dc4945a05fd3d9f Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Mon, 13 Jun 2016 17:35:35 -0500 Subject: [PATCH 358/481] Do not need to depend on jdk here because R itself does. --- var/spack/repos/builtin/packages/r-rJava/package.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/r-rJava/package.py b/var/spack/repos/builtin/packages/r-rJava/package.py index 47804bd7fc..0ce0942602 100644 --- a/var/spack/repos/builtin/packages/r-rJava/package.py +++ b/var/spack/repos/builtin/packages/r-rJava/package.py @@ -37,8 +37,6 @@ class RRjava(Package): extends('R') - depends_on('jdk') - def install(self, spec, prefix): R('CMD', 'INSTALL', '--library={0}'.format(self.module.r_lib_dir), self.stage.source_path) From 3c234a7d3b09edbe6c7715ae9c542ef805bbf052 Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Mon, 13 Jun 2016 17:36:50 -0500 Subject: [PATCH 359/481] New package - r-XLConnectJars --- .../packages/r-XLConnectJars/package.py | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 var/spack/repos/builtin/packages/r-XLConnectJars/package.py diff --git a/var/spack/repos/builtin/packages/r-XLConnectJars/package.py b/var/spack/repos/builtin/packages/r-XLConnectJars/package.py new file mode 100644 index 0000000000..12f72e7468 --- /dev/null +++ b/var/spack/repos/builtin/packages/r-XLConnectJars/package.py @@ -0,0 +1,42 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class RXlconnectjars(Package): + """Provides external JAR dependencies for the XLConnect package.""" + + homepage = "http://miraisolutions.wordpress.com/" + url = "https://cran.r-project.org/src/contrib/XLConnectJars_0.2-9.tar.gz" + + version('0.2-9', 'e6d6b1acfede26acaa616ee421bd30fb') + + extends('R') + + depends_on('r-rJava') + + def install(self, spec, prefix): + R('CMD', 'INSTALL', '--library={0}'.format(self.module.r_lib_dir), + self.stage.source_path) From 5cd09af0662612ecdf7e28e1c6feabdc62afe85c Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Mon, 13 Jun 2016 17:48:44 -0500 Subject: [PATCH 360/481] New package - r-XLConnect --- .../builtin/packages/r-XLConnect/package.py | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 var/spack/repos/builtin/packages/r-XLConnect/package.py diff --git a/var/spack/repos/builtin/packages/r-XLConnect/package.py b/var/spack/repos/builtin/packages/r-XLConnect/package.py new file mode 100644 index 0000000000..919291a023 --- /dev/null +++ b/var/spack/repos/builtin/packages/r-XLConnect/package.py @@ -0,0 +1,45 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class RXlconnect(Package): + """Provides comprehensive functionality to read, write and format Excel + data.""" + + homepage = "http://miraisolutions.wordpress.com/" + url = "https://cran.r-project.org/src/contrib/XLConnect_0.2-11.tar.gz" + list_url = "https://cran.r-project.org/src/contrib/Archive/XLConnect" + + version('0.2-11', '9d1769a103cda05665df399cc335017d') + + extends('R') + + depends_on('r-XLConnectJars') + depends_on('r-rJava') + + def install(self, spec, prefix): + R('CMD', 'INSTALL', '--library={0}'.format(self.module.r_lib_dir), + self.stage.source_path) From 4767185082738f6f4d8e2c12586a910d337dce93 Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Mon, 13 Jun 2016 17:48:59 -0500 Subject: [PATCH 361/481] New package - r-xlsx --- .../repos/builtin/packages/r-xlsx/package.py | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 var/spack/repos/builtin/packages/r-xlsx/package.py diff --git a/var/spack/repos/builtin/packages/r-xlsx/package.py b/var/spack/repos/builtin/packages/r-xlsx/package.py new file mode 100644 index 0000000000..0aac6cdd1f --- /dev/null +++ b/var/spack/repos/builtin/packages/r-xlsx/package.py @@ -0,0 +1,45 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class RXlsx(Package): + """Provide R functions to read/write/format Excel 2007 and Excel + 97/2000/XP/2003 file formats.""" + + homepage = "http://code.google.com/p/rexcel/" + url = "https://cran.rstudio.com/src/contrib/xlsx_0.5.7.tar.gz" + list_url = "https://cran.rstudio.com/src/contrib/Archive/xlsx" + + version('0.5.7', '36b1b16f29c54b6089b1dae923180dd5') + + extends('R') + + depends_on('r-rJava') + depends_on('r-xlsxjars') + + def install(self, spec, prefix): + R('CMD', 'INSTALL', '--library={0}'.format(self.module.r_lib_dir), + self.stage.source_path) From 94d0f21823f2fa6338aa0be6ae8d90646aada875 Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Mon, 13 Jun 2016 17:50:06 -0500 Subject: [PATCH 362/481] New package - r-xlsxjars --- .../builtin/packages/r-xlsxjars/package.py | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 var/spack/repos/builtin/packages/r-xlsxjars/package.py diff --git a/var/spack/repos/builtin/packages/r-xlsxjars/package.py b/var/spack/repos/builtin/packages/r-xlsxjars/package.py new file mode 100644 index 0000000000..cff6e7427e --- /dev/null +++ b/var/spack/repos/builtin/packages/r-xlsxjars/package.py @@ -0,0 +1,44 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class RXlsxjars(Package): + """The xlsxjars package collects all the external jars required for the + xlxs package. This release corresponds to POI 3.10.1.""" + + homepage = "https://cran.rstudio.com/web/packages/xlsxjars/index.html" + url = "https://cran.rstudio.com/src/contrib/xlsxjars_0.6.1.tar.gz" + list_url = "https://cran.rstudio.com/src/contrib/Archive/xlsxjars" + + version('0.6.1', '5a1721d5733cb42f3a29e3f353e39166') + + extends('R') + + depends_on('r-rJava') + + def install(self, spec, prefix): + R('CMD', 'INSTALL', '--library={0}'.format(self.module.r_lib_dir), + self.stage.source_path) From 5ba5ef6c469845a7f0d2bdd43172aba110dff9e0 Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Mon, 13 Jun 2016 17:50:27 -0500 Subject: [PATCH 363/481] Add list_url for older versions. --- var/spack/repos/builtin/packages/r-XLConnectJars/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/r-XLConnectJars/package.py b/var/spack/repos/builtin/packages/r-XLConnectJars/package.py index 12f72e7468..bb4e39d5f3 100644 --- a/var/spack/repos/builtin/packages/r-XLConnectJars/package.py +++ b/var/spack/repos/builtin/packages/r-XLConnectJars/package.py @@ -30,6 +30,7 @@ class RXlconnectjars(Package): homepage = "http://miraisolutions.wordpress.com/" url = "https://cran.r-project.org/src/contrib/XLConnectJars_0.2-9.tar.gz" + list_url ="https://cran.r-project.org/src/contrib/Archive/XLConnectJars" version('0.2-9', 'e6d6b1acfede26acaa616ee421bd30fb') From 8861c99333c748037926dbdef51d4a7974d5f1cf Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Mon, 13 Jun 2016 22:50:07 -0500 Subject: [PATCH 364/481] New package - r-foreign --- .../builtin/packages/r-foreign/package.py | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 var/spack/repos/builtin/packages/r-foreign/package.py diff --git a/var/spack/repos/builtin/packages/r-foreign/package.py b/var/spack/repos/builtin/packages/r-foreign/package.py new file mode 100644 index 0000000000..df38fc5680 --- /dev/null +++ b/var/spack/repos/builtin/packages/r-foreign/package.py @@ -0,0 +1,43 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class RForeign(Package): + """Functions for reading and writing data stored by some versions of Epi + Info, Minitab, S, SAS, SPSS, Stata, Systat and Weka and for reading and + writing some dBase files.""" + + homepage = "https://cran.r-project.org/web/packages/foreign/index.html" + url = "https://cran.r-project.org/src/contrib/foreign_0.8-66.tar.gz" + list_url = "https://cran.r-project.org/src/contrib/Archive/foreign" + + version('0.8-66', 'ff12190f4631dca31e30ca786c2c8f62') + + extends('R') + + def install(self, spec, prefix): + R('CMD', 'INSTALL', '--library={0}'.format(self.module.r_lib_dir), + self.stage.source_path) From e226837d332ef33e195b039dba1dd090e8b56182 Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Mon, 13 Jun 2016 23:04:26 -0500 Subject: [PATCH 365/481] Fix flake8 error. --- var/spack/repos/builtin/packages/r-XLConnectJars/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/r-XLConnectJars/package.py b/var/spack/repos/builtin/packages/r-XLConnectJars/package.py index bb4e39d5f3..15d08cc5af 100644 --- a/var/spack/repos/builtin/packages/r-XLConnectJars/package.py +++ b/var/spack/repos/builtin/packages/r-XLConnectJars/package.py @@ -30,7 +30,7 @@ class RXlconnectjars(Package): homepage = "http://miraisolutions.wordpress.com/" url = "https://cran.r-project.org/src/contrib/XLConnectJars_0.2-9.tar.gz" - list_url ="https://cran.r-project.org/src/contrib/Archive/XLConnectJars" + list_url = "https://cran.r-project.org/src/contrib/Archive/XLConnectJars" version('0.2-9', 'e6d6b1acfede26acaa616ee421bd30fb') From e6ec3729260b666483b02e38501671bf32ec3af5 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Tue, 14 Jun 2016 09:59:48 -0400 Subject: [PATCH 366/481] Add packages c-blosc and hdf5-blosc c-blosc provides the Blosc compression filter, and hdf5-blosc provides a plugin for HDF5 to use that filter. --- .../repos/builtin/packages/c-blosc/package.py | 47 +++++++++++ .../builtin/packages/hdf5-blosc/package.py | 77 +++++++++++++++++++ 2 files changed, 124 insertions(+) create mode 100644 var/spack/repos/builtin/packages/c-blosc/package.py create mode 100644 var/spack/repos/builtin/packages/hdf5-blosc/package.py diff --git a/var/spack/repos/builtin/packages/c-blosc/package.py b/var/spack/repos/builtin/packages/c-blosc/package.py new file mode 100644 index 0000000000..16e8c89811 --- /dev/null +++ b/var/spack/repos/builtin/packages/c-blosc/package.py @@ -0,0 +1,47 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + +from spack import * + +class CBlosc(Package): + """Blosc, an extremely fast, multi-threaded, meta-compressor library""" + homepage = "http://www.blosc.org" + url = "https://github.com/Blosc/c-blosc/archive/v1.9.2.tar.gz" + + version('1.9.2', 'dd2d83069d74b36b8093f1c6b49defc5') + version('1.9.1', '7d708d3daadfacf984a87b71b1734ce2') + version('1.9.0', 'e4c1dc8e2c468e5cfa2bf05eeee5357a') + version('1.8.1', 'd73d5be01359cf271e9386c90dcf5b05') + version('1.8.0', '5b92ecb287695ba20cc33d30bf221c4f') + + depends_on("cmake") + depends_on("snappy") + depends_on("zlib") + + def install(self, spec, prefix): + cmake('.', *std_cmake_args) + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/hdf5-blosc/package.py b/var/spack/repos/builtin/packages/hdf5-blosc/package.py new file mode 100644 index 0000000000..897c079e4e --- /dev/null +++ b/var/spack/repos/builtin/packages/hdf5-blosc/package.py @@ -0,0 +1,77 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + +import os +import shutil +import sys + +from spack import * + +class Hdf5Blosc(Package): + """Blosc filter for HDF5""" + homepage = "https://github.com/Blosc/hdf5-blosc" + url = "https://github.com/Blosc/hdf5-blosc/archive/master.zip" + + version('master', '02c04acbf4bec66ec8a35bf157d1c9de') + + depends_on("c-blosc") + depends_on("hdf5") + depends_on("libtool") + + parallel = False + + def install(self, spec, prefix): + cmake('.', *std_cmake_args) + + make() + make("install") + if sys.platform == 'darwin': + fix_darwin_install_name(prefix.lib) + + # Build and install plugin manually + # The plugin requires at least HDF5 1.8.11: + if spec['hdf5'].satisfies('@1.8.11:'): + # The plugin does not yet work with Windows: + if sys.platform not in ('win32', 'cygwin'): + # cc = which('cc') + libtool = Executable(join_path(spec['libtool'].prefix.bin, + 'libtool')) + with working_dir('src'): + libtool('--mode=compile', '--tag=CC', + 'cc', '-g', '-O', '-c', 'blosc_plugin.c') + libtool('--mode=link', '--tag=CC', + 'cc', '-g', '-O', + '-rpath', prefix.lib, + '-o', 'libblosc_plugin.la', 'blosc_plugin.lo') + shlibext = 'so' if sys.platform!='darwin' else 'dylib' + shlib0 = 'libblosc_plugin.0.%s' % shlibext + shutil.copyfile(join_path('.libs', shlib0), + join_path(prefix.lib, shlib0)) + shlib = 'libblosc_plugin.%s' % shlibext + os.symlink(shlib0, join_path(prefix.lib, shlib)) + + def setup_dependent_environment(self, spack_env, run_env, dependent_spec): + spack_env.append_path('HDF5_PLUGIN_PATH', self.spec.prefix.lib) + run_env.append_path('HDF5_PLUGIN_PATH', self.spec.prefix.lib) From ca5180f7a24d08d49f046a4efa8ef5b48201211d Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Tue, 14 Jun 2016 17:03:00 -0400 Subject: [PATCH 367/481] Correct Blosc install on Darwin --- .../repos/builtin/packages/c-blosc/package.py | 4 + .../builtin/packages/hdf5-blosc/package.py | 177 +++++++++++++++--- 2 files changed, 153 insertions(+), 28 deletions(-) diff --git a/var/spack/repos/builtin/packages/c-blosc/package.py b/var/spack/repos/builtin/packages/c-blosc/package.py index 16e8c89811..dee332be14 100644 --- a/var/spack/repos/builtin/packages/c-blosc/package.py +++ b/var/spack/repos/builtin/packages/c-blosc/package.py @@ -23,6 +23,8 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +import sys + from spack import * class CBlosc(Package): @@ -45,3 +47,5 @@ def install(self, spec, prefix): make() make("install") + if sys.platform == 'darwin': + fix_darwin_install_name(prefix.lib) diff --git a/var/spack/repos/builtin/packages/hdf5-blosc/package.py b/var/spack/repos/builtin/packages/hdf5-blosc/package.py index 897c079e4e..680bc41c41 100644 --- a/var/spack/repos/builtin/packages/hdf5-blosc/package.py +++ b/var/spack/repos/builtin/packages/hdf5-blosc/package.py @@ -43,35 +43,156 @@ class Hdf5Blosc(Package): parallel = False def install(self, spec, prefix): - cmake('.', *std_cmake_args) + # The included cmake recipe doesn"t work for Darwin + # cmake(".", *std_cmake_args) + # + # make() + # make("install") + # if sys.platform == "darwin": + # fix_darwin_install_name(prefix.lib) - make() - make("install") - if sys.platform == 'darwin': - fix_darwin_install_name(prefix.lib) + libtool = Executable(join_path(spec["libtool"].prefix.bin, "libtool")) + if "+mpi" in spec["hdf5"]: + cc = "mpicc" + else: + cc = "cc" + shlibext = "so" if sys.platform!="darwin" else "dylib" + mkdirp(prefix.include) + mkdirp(prefix.lib) - # Build and install plugin manually - # The plugin requires at least HDF5 1.8.11: - if spec['hdf5'].satisfies('@1.8.11:'): - # The plugin does not yet work with Windows: - if sys.platform not in ('win32', 'cygwin'): - # cc = which('cc') - libtool = Executable(join_path(spec['libtool'].prefix.bin, - 'libtool')) - with working_dir('src'): - libtool('--mode=compile', '--tag=CC', - 'cc', '-g', '-O', '-c', 'blosc_plugin.c') - libtool('--mode=link', '--tag=CC', - 'cc', '-g', '-O', - '-rpath', prefix.lib, - '-o', 'libblosc_plugin.la', 'blosc_plugin.lo') - shlibext = 'so' if sys.platform!='darwin' else 'dylib' - shlib0 = 'libblosc_plugin.0.%s' % shlibext - shutil.copyfile(join_path('.libs', shlib0), - join_path(prefix.lib, shlib0)) - shlib = 'libblosc_plugin.%s' % shlibext - os.symlink(shlib0, join_path(prefix.lib, shlib)) + # Build and install filter + with working_dir("src"): + libtool("--mode=compile", "--tag=CC", + "cc", "-g", "-O", + "-c", "blosc_filter.c") + libtool("--mode=link", "--tag=CC", + "cc", "-g", "-O", + "-rpath", prefix.lib, + "-o", "libblosc_filter.la", + "blosc_filter.lo", + "-L%s" % spec["c-blosc"].prefix.lib, "-lblosc", + "-L%s" % spec["hdf5"].prefix.lib, "-lhdf5") + shlib0 = "libblosc_filter.0.%s" % shlibext + shlib = "libblosc_filter.%s" % shlibext + shutil.copyfile(join_path(".libs", shlib0), + join_path(prefix.lib, shlib0)) + os.symlink(shlib0, join_path(prefix.lib, shlib)) + + # Build and install plugin + # The plugin requires at least HDF5 1.8.11: + if spec["hdf5"].satisfies("@1.8.11:"): + libtool("--mode=compile", "--tag=CC", + "cc", "-g", "-O", + "-c", "blosc_plugin.c") + libtool("--mode=link", "--tag=CC", + "cc", "-g", "-O", + "-rpath", prefix.lib, + "-o", "libblosc_plugin.la", + "blosc_plugin.lo", + "-L%s" % prefix.lib, "-lblosc_filter", + "-L%s" % spec["c-blosc"].prefix.lib, "-lblosc", + "-L%s" % spec["hdf5"].prefix.lib, "-lhdf5") + shlib0 = "libblosc_plugin.0.%s" % shlibext + shlib = "libblosc_plugin.%s" % shlibext + shutil.copyfile(join_path(".libs", shlib0), + join_path(prefix.lib, shlib0)) + os.symlink(shlib0, join_path(prefix.lib, shlib)) + + self.check_install(spec) + def check_install(self, spec): + "Build and run a small program to test the installed HDF5 Blosc plugin" + print "Checking HDF5-Blosc plugin..." + checkdir = "spack-check" + with working_dir(checkdir, create=True): + source = r"""\ +#include +#include +#include +#include + +#define FILTER_BLOSC 32001 /* Blosc filter ID registered with the HDF group */ + +int main(int argc, char **argv) { + herr_t herr; + hid_t file = H5Fcreate("file.h5", H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + assert(file >= 0); + hsize_t dims[3] = {10, 10, 10}; + hid_t space = H5Screate_simple(3, dims, NULL); + assert(space >= 0); + hid_t create_proplist = H5Pcreate(H5P_DATASET_CREATE); + assert(create_proplist >= 0); + herr = H5Pset_chunk(create_proplist, 3, dims); + assert(herr >= 0); + herr = H5Pset_filter(create_proplist, FILTER_BLOSC, H5Z_FLAG_OPTIONAL, 0, + NULL); + assert(herr >= 0); + htri_t all_filters_avail = H5Pall_filters_avail(create_proplist); + assert(all_filters_avail > 0); + hid_t dataset = H5Dcreate(file, "dataset", H5T_NATIVE_DOUBLE, space, + H5P_DEFAULT, create_proplist, H5P_DEFAULT); + assert(dataset >= 0); + double data[10][10][10]; + for (int k=0; k<10; ++k) { + for (int j=0; j<10; ++j) { + for (int i=0; i<10; ++i) { + data[k][j][i] = 1.0 / (1.0 + i + j + k); + } + } + } + herr = H5Dwrite(dataset, H5T_NATIVE_DOUBLE, space, space, H5P_DEFAULT, + &data[0][0][0]); + assert(herr >= 0); + herr = H5Pclose(create_proplist); + assert(herr >= 0); + herr = H5Dclose(dataset); + assert(herr >= 0); + herr = H5Sclose(space); + assert(herr >= 0); + herr = H5Fclose(file); + assert(herr >= 0); + printf("Done.\n"); + return 0; +} +""" + expected = """\ +Done. +""" + with open("check.c", "w") as f: + f.write(source) + if "+mpi" in spec["hdf5"]: + cc = which("mpicc") + else: + cc = which("cc") + # TODO: Automate these path and library settings + cc("-c", "-I%s" % spec["hdf5"].prefix.include, "check.c") + cc("-o", "check", "check.o", + "-L%s" % spec["hdf5"].prefix.lib, "-lhdf5") + try: + check = Executable("./check") + output = check(return_output=True) + except: + output = "" + success = output == expected + if not success: + print "Produced output does not match expected output." + print "Expected output:" + print "-"*80 + print expected + print "-"*80 + print "Produced output:" + print "-"*80 + print output + print "-"*80 + print "Environment:" + env = which("env") + env() + raise RuntimeError("HDF5 Blosc plugin check failed") + shutil.rmtree(checkdir) + + def setup_environment(self, spack_env, run_env): + spack_env.append_path("HDF5_PLUGIN_PATH", self.spec.prefix.lib) + run_env.append_path("HDF5_PLUGIN_PATH", self.spec.prefix.lib) def setup_dependent_environment(self, spack_env, run_env, dependent_spec): - spack_env.append_path('HDF5_PLUGIN_PATH', self.spec.prefix.lib) - run_env.append_path('HDF5_PLUGIN_PATH', self.spec.prefix.lib) + spack_env.append_path("HDF5_PLUGIN_PATH", self.spec.prefix.lib) + run_env.append_path("HDF5_PLUGIN_PATH", self.spec.prefix.lib) From 05d7378da57f51a26830a61a88228fb8fc7d3108 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Tue, 14 Jun 2016 17:13:20 -0400 Subject: [PATCH 368/481] Install shared libraries correctly on Linux --- .../builtin/packages/hdf5-blosc/package.py | 28 ++++++++++++------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/var/spack/repos/builtin/packages/hdf5-blosc/package.py b/var/spack/repos/builtin/packages/hdf5-blosc/package.py index 680bc41c41..50f380083c 100644 --- a/var/spack/repos/builtin/packages/hdf5-blosc/package.py +++ b/var/spack/repos/builtin/packages/hdf5-blosc/package.py @@ -29,6 +29,21 @@ from spack import * +def _install_shlib(name, src, dst): + """Install a shared library from directory src to directory dst""" + if sys.platform == "darwin": + shlib0 = name + ".0.dylib" + shlib = name + ".dylib" + shutil.copyfile(join_path(src, shlib0), join_path(dst, shlib0)) + os.symlink(shlib0, join_path(dst, shlib)) + else: + shlib000 = name + ".so.0.0.0" + shlib0 = name + ".so.0" + shlib = name + ".dylib" + shutil.copyfile(join_path(src, shlib000), join_path(dst, shlib000)) + os.symlink(shlib000, join_path(dst, shlib0)) + os.symlink(shlib0, join_path(dst, shlib)) + class Hdf5Blosc(Package): """Blosc filter for HDF5""" homepage = "https://github.com/Blosc/hdf5-blosc" @@ -72,11 +87,7 @@ def install(self, spec, prefix): "blosc_filter.lo", "-L%s" % spec["c-blosc"].prefix.lib, "-lblosc", "-L%s" % spec["hdf5"].prefix.lib, "-lhdf5") - shlib0 = "libblosc_filter.0.%s" % shlibext - shlib = "libblosc_filter.%s" % shlibext - shutil.copyfile(join_path(".libs", shlib0), - join_path(prefix.lib, shlib0)) - os.symlink(shlib0, join_path(prefix.lib, shlib)) + _install_shlib("libblosc_filter", ".libs", prefix.lib) # Build and install plugin # The plugin requires at least HDF5 1.8.11: @@ -92,13 +103,10 @@ def install(self, spec, prefix): "-L%s" % prefix.lib, "-lblosc_filter", "-L%s" % spec["c-blosc"].prefix.lib, "-lblosc", "-L%s" % spec["hdf5"].prefix.lib, "-lhdf5") - shlib0 = "libblosc_plugin.0.%s" % shlibext - shlib = "libblosc_plugin.%s" % shlibext - shutil.copyfile(join_path(".libs", shlib0), - join_path(prefix.lib, shlib0)) - os.symlink(shlib0, join_path(prefix.lib, shlib)) + _install_shlib("libblosc_plugin", ".libs", prefix.lib) self.check_install(spec) + def check_install(self, spec): "Build and run a small program to test the installed HDF5 Blosc plugin" print "Checking HDF5-Blosc plugin..." From b59d4f243bc012f6673a53be75d197c16e5e7a06 Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 15 Jun 2016 13:16:29 +0200 Subject: [PATCH 369/481] openmpi, mvapich2, mpich : filtered out "--Wl,--enable-new-dtag" in wrappers, as it turns RPATH into RUNPATH Apart from that turned `os.path.join` into `join_path` and `os.environ` into `env` in the packages --- .../repos/builtin/packages/mpich/package.py | 24 ++++--- .../builtin/packages/mvapich2/package.py | 24 ++++--- .../repos/builtin/packages/openmpi/package.py | 63 +++++++++---------- 3 files changed, 58 insertions(+), 53 deletions(-) diff --git a/var/spack/repos/builtin/packages/mpich/package.py b/var/spack/repos/builtin/packages/mpich/package.py index 164d9db541..e7ae63ef70 100644 --- a/var/spack/repos/builtin/packages/mpich/package.py +++ b/var/spack/repos/builtin/packages/mpich/package.py @@ -23,7 +23,6 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## from spack import * -import os class Mpich(Package): @@ -102,18 +101,25 @@ def filter_compilers(self): be bound to whatever compiler they were built with. """ bin = self.prefix.bin - mpicc = os.path.join(bin, 'mpicc') - mpicxx = os.path.join(bin, 'mpicxx') - mpif77 = os.path.join(bin, 'mpif77') - mpif90 = os.path.join(bin, 'mpif90') + mpicc = join_path(bin, 'mpicc') + mpicxx = join_path(bin, 'mpicxx') + mpif77 = join_path(bin, 'mpif77') + mpif90 = join_path(bin, 'mpif90') - spack_cc = os.environ['CC'] - spack_cxx = os.environ['CXX'] - spack_f77 = os.environ['F77'] - spack_fc = os.environ['FC'] + spack_cc = env['CC'] + spack_cxx = env['CXX'] + spack_f77 = env['F77'] + spack_fc = env['FC'] + # Substitute Spack compile wrappers for the real + # underlying compiler kwargs = { 'ignore_absent' : True, 'backup' : False, 'string' : True } filter_file('CC="%s"' % spack_cc , 'CC="%s"' % self.compiler.cc, mpicc, **kwargs) filter_file('CXX="%s"'% spack_cxx, 'CXX="%s"' % self.compiler.cxx, mpicxx, **kwargs) filter_file('F77="%s"'% spack_f77, 'F77="%s"' % self.compiler.f77, mpif77, **kwargs) filter_file('FC="%s"' % spack_fc , 'FC="%s"' % self.compiler.fc, mpif90, **kwargs) + + # Remove this linking flag if present + # (it turns RPATH into RUNPATH) + for wrapper in (mpicc, mpicxx, mpif77, mpif90): + filter_file('-Wl,--enable-new-dtags', '', wrapper, **kwargs) diff --git a/var/spack/repos/builtin/packages/mvapich2/package.py b/var/spack/repos/builtin/packages/mvapich2/package.py index d1944023d1..131d430cf2 100644 --- a/var/spack/repos/builtin/packages/mvapich2/package.py +++ b/var/spack/repos/builtin/packages/mvapich2/package.py @@ -23,7 +23,6 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## from spack import * -import os class Mvapich2(Package): @@ -245,15 +244,15 @@ def filter_compilers(self): be bound to whatever compiler they were built with. """ bin = self.prefix.bin - mpicc = os.path.join(bin, 'mpicc') - mpicxx = os.path.join(bin, 'mpicxx') - mpif77 = os.path.join(bin, 'mpif77') - mpif90 = os.path.join(bin, 'mpif90') + mpicc = join_path(bin, 'mpicc') + mpicxx = join_path(bin, 'mpicxx') + mpif77 = join_path(bin, 'mpif77') + mpif90 = join_path(bin, 'mpif90') - spack_cc = os.environ['CC'] - spack_cxx = os.environ['CXX'] - spack_f77 = os.environ['F77'] - spack_fc = os.environ['FC'] + spack_cc = env['CC'] + spack_cxx = env['CXX'] + spack_f77 = env['F77'] + spack_fc = env['FC'] kwargs = { 'ignore_absent': True, @@ -261,6 +260,8 @@ def filter_compilers(self): 'string': True } + # Substitute Spack compile wrappers for the real + # underlying compiler filter_file('CC="%s"' % spack_cc, 'CC="%s"' % self.compiler.cc, mpicc, **kwargs) filter_file('CXX="%s"' % spack_cxx, @@ -269,3 +270,8 @@ def filter_compilers(self): 'F77="%s"' % self.compiler.f77, mpif77, **kwargs) filter_file('FC="%s"' % spack_fc, 'FC="%s"' % self.compiler.fc, mpif90, **kwargs) + + # Remove this linking flag if present + # (it turns RPATH into RUNPATH) + for wrapper in (mpicc, mpicxx, mpif77, mpif90): + filter_file('-Wl,--enable-new-dtags', '', wrapper, **kwargs) diff --git a/var/spack/repos/builtin/packages/openmpi/package.py b/var/spack/repos/builtin/packages/openmpi/package.py index 4e465e1784..e3d9a28b4a 100644 --- a/var/spack/repos/builtin/packages/openmpi/package.py +++ b/var/spack/repos/builtin/packages/openmpi/package.py @@ -173,40 +173,33 @@ def filter_compilers(self): be bound to whatever compiler they were built with. """ kwargs = {'ignore_absent': True, 'backup': False, 'string': False} - dir = os.path.join(self.prefix, 'share/openmpi/') + wrapper_basepath = join_path(self.prefix, 'share', 'openmpi') - cc_wrappers = ['mpicc-vt-wrapper-data.txt', 'mpicc-wrapper-data.txt', - 'ortecc-wrapper-data.txt', 'shmemcc-wrapper-data.txt'] + wrappers = [ + ('mpicc-vt-wrapper-data.txt', self.compiler.cc), + ('mpicc-wrapper-data.txt', self.compiler.cc), + ('ortecc-wrapper-data.txt', self.compiler.cc), + ('shmemcc-wrapper-data.txt', self.compiler.cc), + ('mpic++-vt-wrapper-data.txt', self.compiler.cxx), + ('mpic++-wrapper-data.txt', self.compiler.cxx), + ('ortec++-wrapper-data.txt', self.compiler.cxx), + ('mpifort-vt-wrapper-data.txt', self.compiler.fc), + ('mpifort-wrapper-data.txt', self.compiler.fc), + ('shmemfort-wrapper-data.txt', self.compiler.fc), + ('mpif90-vt-wrapper-data.txt', self.compiler.fc), + ('mpif90-wrapper-data.txt', self.compiler.fc), + ('mpif77-vt-wrapper-data.txt', self.compiler.f77), + ('mpif77-wrapper-data.txt', self.compiler.f77) + ] - cxx_wrappers = ['mpic++-vt-wrapper-data.txt', 'mpic++-wrapper-data.txt', - 'ortec++-wrapper-data.txt'] - - fc_wrappers = ['mpifort-vt-wrapper-data.txt', - 'mpifort-wrapper-data.txt', 'shmemfort-wrapper-data.txt'] - - for wrapper in cc_wrappers: - filter_file('compiler=.*', 'compiler=%s' % self.compiler.cc, - os.path.join(dir, wrapper), **kwargs) - - for wrapper in cxx_wrappers: - filter_file('compiler=.*', 'compiler=%s' % self.compiler.cxx, - os.path.join(dir, wrapper), **kwargs) - - for wrapper in fc_wrappers: - filter_file('compiler=.*', 'compiler=%s' % self.compiler.fc, - os.path.join(dir, wrapper), **kwargs) - - # These are symlinks in newer versions, so check that here - f77_wrappers = ['mpif77-vt-wrapper-data.txt', 'mpif77-wrapper-data.txt'] - f90_wrappers = ['mpif90-vt-wrapper-data.txt', 'mpif90-wrapper-data.txt'] - - for wrapper in f77_wrappers: - path = os.path.join(dir, wrapper) - if not os.path.islink(path): - filter_file('compiler=.*', 'compiler=%s' % self.compiler.f77, - path, **kwargs) - for wrapper in f90_wrappers: - path = os.path.join(dir, wrapper) - if not os.path.islink(path): - filter_file('compiler=.*', 'compiler=%s' % self.compiler.fc, - path, **kwargs) + for wrapper_name, compiler in wrappers: + wrapper = join_path(wrapper_basepath, wrapper_name) + if not os.path.islink(wrapper): + # Substitute Spack compile wrappers for the real + # underlying compiler + match = 'compiler=.*' + substitute = 'compiler={compiler}'.format(compiler=compiler) + filter_file(match, substitute, wrapper, **kwargs) + # Remove this linking flag if present + # (it turns RPATH into RUNPATH) + filter_file('-Wl,--enable-new-dtags', '', wrapper, **kwargs) From b09bee81584aa38e78596f3f7f8f4cc09a4418ca Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 15 Jun 2016 13:29:30 +0200 Subject: [PATCH 370/481] qa : fixed flake8 issues --- .../repos/builtin/packages/mpich/package.py | 32 +++++++++-------- .../repos/builtin/packages/openmpi/package.py | 35 +++++++++++-------- 2 files changed, 39 insertions(+), 28 deletions(-) diff --git a/var/spack/repos/builtin/packages/mpich/package.py b/var/spack/repos/builtin/packages/mpich/package.py index e7ae63ef70..511beafbbd 100644 --- a/var/spack/repos/builtin/packages/mpich/package.py +++ b/var/spack/repos/builtin/packages/mpich/package.py @@ -29,8 +29,8 @@ class Mpich(Package): """MPICH is a high performance and widely portable implementation of the Message Passing Interface (MPI) standard.""" homepage = "http://www.mpich.org" - url = "http://www.mpich.org/static/downloads/3.0.4/mpich-3.0.4.tar.gz" - list_url = "http://www.mpich.org/static/downloads/" + url = "http://www.mpich.org/static/downloads/3.0.4/mpich-3.0.4.tar.gz" + list_url = "http://www.mpich.org/static/downloads/" list_depth = 2 version('3.2', 'f414cfa77099cd1fa1a5ae4e22db508a') @@ -41,7 +41,8 @@ class Mpich(Package): version('3.1', '5643dd176499bfb7d25079aaff25f2ec') version('3.0.4', '9c5d5d4fe1e17dd12153f40bc5b6dbc0') - variant('verbs', default=False, description='Build support for OpenFabrics verbs.') + variant('verbs', default=False, + description='Build support for OpenFabrics verbs.') variant('pmi', default=True, description='Build with PMI support') variant('hydra', default=True, description='Build the hydra process manager') @@ -56,9 +57,9 @@ def setup_dependent_environment(self, spack_env, run_env, dependent_spec): spack_env.set('MPICH_FC', spack_fc) def setup_dependent_package(self, module, dep_spec): - self.spec.mpicc = join_path(self.prefix.bin, 'mpicc') + self.spec.mpicc = join_path(self.prefix.bin, 'mpicc') self.spec.mpicxx = join_path(self.prefix.bin, 'mpic++') - self.spec.mpifc = join_path(self.prefix.bin, 'mpif90') + self.spec.mpifc = join_path(self.prefix.bin, 'mpif90') self.spec.mpif77 = join_path(self.prefix.bin, 'mpif77') def install(self, spec, prefix): @@ -91,7 +92,6 @@ def install(self, spec, prefix): self.filter_compilers() - def filter_compilers(self): """Run after install to make the MPI compilers use the compilers that Spack built the package with. @@ -101,23 +101,27 @@ def filter_compilers(self): be bound to whatever compiler they were built with. """ bin = self.prefix.bin - mpicc = join_path(bin, 'mpicc') + mpicc = join_path(bin, 'mpicc') mpicxx = join_path(bin, 'mpicxx') mpif77 = join_path(bin, 'mpif77') mpif90 = join_path(bin, 'mpif90') - spack_cc = env['CC'] + spack_cc = env['CC'] spack_cxx = env['CXX'] spack_f77 = env['F77'] - spack_fc = env['FC'] + spack_fc = env['FC'] # Substitute Spack compile wrappers for the real # underlying compiler - kwargs = { 'ignore_absent' : True, 'backup' : False, 'string' : True } - filter_file('CC="%s"' % spack_cc , 'CC="%s"' % self.compiler.cc, mpicc, **kwargs) - filter_file('CXX="%s"'% spack_cxx, 'CXX="%s"' % self.compiler.cxx, mpicxx, **kwargs) - filter_file('F77="%s"'% spack_f77, 'F77="%s"' % self.compiler.f77, mpif77, **kwargs) - filter_file('FC="%s"' % spack_fc , 'FC="%s"' % self.compiler.fc, mpif90, **kwargs) + kwargs = {'ignore_absent': True, 'backup': False, 'string': True} + filter_file('CC="%s"' % spack_cc, + 'CC="%s"' % self.compiler.cc, mpicc, **kwargs) + filter_file('CXX="%s"' % spack_cxx, + 'CXX="%s"' % self.compiler.cxx, mpicxx, **kwargs) + filter_file('F77="%s"' % spack_f77, + 'F77="%s"' % self.compiler.f77, mpif77, **kwargs) + filter_file('FC="%s"' % spack_fc, + 'FC="%s"' % self.compiler.fc, mpif90, **kwargs) # Remove this linking flag if present # (it turns RPATH into RUNPATH) diff --git a/var/spack/repos/builtin/packages/openmpi/package.py b/var/spack/repos/builtin/packages/openmpi/package.py index e3d9a28b4a..ab71b4b503 100644 --- a/var/spack/repos/builtin/packages/openmpi/package.py +++ b/var/spack/repos/builtin/packages/openmpi/package.py @@ -72,20 +72,27 @@ class Openmpi(Package): patch('configure.patch', when="@1.10.0:1.10.1") variant('psm', default=False, description='Build support for the PSM library.') - variant('psm2', default=False, description='Build support for the Intel PSM2 library.') - variant('pmi', default=False, description='Build support for PMI-based launchers') - variant('verbs', default=_verbs_dir() is not None, description='Build support for OpenFabrics verbs.') + variant('psm2', default=False, + description='Build support for the Intel PSM2 library.') + variant('pmi', default=False, + description='Build support for PMI-based launchers') + variant('verbs', default=_verbs_dir() is not None, + description='Build support for OpenFabrics verbs.') variant('mxm', default=False, description='Build Mellanox Messaging support') - variant('thread_multiple', default=False, description='Enable MPI_THREAD_MULTIPLE support') + variant('thread_multiple', default=False, + description='Enable MPI_THREAD_MULTIPLE support') # TODO : variant support for alps, loadleveler is missing - variant('tm', default=False, description='Build TM (Torque, PBSPro, and compatible) support') - variant('slurm', default=False, description='Build SLURM scheduler component') + variant('tm', default=False, + description='Build TM (Torque, PBSPro, and compatible) support') + variant('slurm', default=False, + description='Build SLURM scheduler component') variant('sqlite3', default=False, description='Build sqlite3 support') - variant('vt', default=True, description='Build support for contributed package vt') + variant('vt', default=True, + description='Build support for contributed package vt') # TODO : support for CUDA is missing @@ -96,8 +103,7 @@ class Openmpi(Package): depends_on('sqlite', when='+sqlite3') def url_for_version(self, version): - return "http://www.open-mpi.org/software/ompi/v%s/downloads/openmpi-%s.tar.bz2" % (version.up_to(2), version) - + return "http://www.open-mpi.org/software/ompi/v%s/downloads/openmpi-%s.tar.bz2" % (version.up_to(2), version) # NOQA: ignore=E501 def setup_dependent_environment(self, spack_env, run_env, dependent_spec): spack_env.set('OMPI_CC', spack_cc) @@ -106,14 +112,15 @@ def setup_dependent_environment(self, spack_env, run_env, dependent_spec): spack_env.set('OMPI_F77', spack_f77) def setup_dependent_package(self, module, dep_spec): - self.spec.mpicc = join_path(self.prefix.bin, 'mpicc') + self.spec.mpicc = join_path(self.prefix.bin, 'mpicc') self.spec.mpicxx = join_path(self.prefix.bin, 'mpic++') - self.spec.mpifc = join_path(self.prefix.bin, 'mpif90') + self.spec.mpifc = join_path(self.prefix.bin, 'mpif90') self.spec.mpif77 = join_path(self.prefix.bin, 'mpif77') @property def verbs(self): - # Up through version 1.6, this option was previously named --with-openib + # Up through version 1.6, this option was previously named + # --with-openib if self.spec.satisfies('@:1.6'): return 'openib' # In version 1.7, it was renamed to be --with-verbs @@ -135,7 +142,7 @@ def install(self, spec, prefix): '--with-psm2' if '+psm2' in spec else '--without-psm2', '--with-mxm' if '+mxm' in spec else '--without-mxm', # Other options - '--enable-mpi-thread-multiple' if '+thread_multiple' in spec else '--disable-mpi-thread-multiple', + '--enable-mpi-thread-multiple' if '+thread_multiple' in spec else '--disable-mpi-thread-multiple', # NOQA: ignore=E501 '--with-pmi' if '+pmi' in spec else '--without-pmi', '--with-sqlite3' if '+sqlite3' in spec else '--without-sqlite3', '--enable-vt' if '+vt' in spec else '--disable-vt' @@ -153,7 +160,7 @@ def install(self, spec, prefix): # use this for LANL builds, but for LLNL builds, we need: # "--with-platform=contrib/platform/llnl/optimized" if self.version == ver("1.6.5") and '+lanl' in spec: - config_args.append("--with-platform=contrib/platform/lanl/tlcc2/optimized-nopanasas") + config_args.append("--with-platform=contrib/platform/lanl/tlcc2/optimized-nopanasas") # NOQA: ignore=E501 if not self.compiler.f77 and not self.compiler.fc: config_args.append("--enable-mpi-fortran=no") From a860a3d6145e6c35fd1b523b2f8166a0db1ed4b2 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Tue, 14 Jun 2016 15:54:58 +0200 Subject: [PATCH 371/481] superlu-dist: avoid harcoding blas/lapack/mpi; remove preferred version --- lib/spack/llnl/util/filesystem.py | 15 ++++++++++++++- .../builtin/packages/superlu-dist/package.py | 13 ++++++------- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index 6661a80f27..431289488d 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -28,7 +28,7 @@ 'FileFilter', 'change_sed_delimiter', 'is_exe', 'force_symlink', 'set_executable', 'copy_mode', 'unset_executable_mode', 'remove_dead_links', 'remove_linked_tree', 'find_library_path', - 'fix_darwin_install_name'] + 'fix_darwin_install_name','to_link_flags'] import os import glob @@ -424,6 +424,19 @@ def fix_darwin_install_name(path): break +def to_link_flags(library): + """Transforms a path to a into linking flags -L -l. + + Return: + A string of linking flags. + """ + dir = os.path.dirname(library) + # Asume libXYZ.suffix + name = os.path.basename(library)[3:].split(".")[0] + res = '-L%s -l%s' % (dir,name) + return res + + def find_library_path(libname, *paths): """Searches for a file called in each path. diff --git a/var/spack/repos/builtin/packages/superlu-dist/package.py b/var/spack/repos/builtin/packages/superlu-dist/package.py index 6c06b5497c..d24f03c230 100644 --- a/var/spack/repos/builtin/packages/superlu-dist/package.py +++ b/var/spack/repos/builtin/packages/superlu-dist/package.py @@ -31,8 +31,7 @@ class SuperluDist(Package): url = "http://crd-legacy.lbl.gov/~xiaoye/SuperLU/superlu_dist_4.1.tar.gz" version('5.0.0', '2b53baf1b0ddbd9fcf724992577f0670') - # default to version 4.3 since petsc and trilinos are not tested with 5.0. - version('4.3', 'ee66c84e37b4f7cc557771ccc3dc43ae', preferred=True) + version('4.3', 'ee66c84e37b4f7cc557771ccc3dc43ae') version('4.2', 'ae9fafae161f775fbac6eba11e530a65') version('4.1', '4edee38cc29f687bd0c8eb361096a455') version('4.0', 'c0b98b611df227ae050bc1635c6940e0') @@ -47,10 +46,10 @@ def install(self, spec, prefix): makefile_inc = [] makefile_inc.extend([ 'PLAT = _mac_x', - 'DSuperLUroot = %s' % self.stage.source_path, #self.stage.path, prefix + 'DSuperLUroot = %s' % self.stage.source_path, 'DSUPERLULIB = $(DSuperLUroot)/lib/libsuperlu_dist.a', 'BLASDEF = -DUSE_VENDOR_BLAS', - 'BLASLIB = -L%s -llapack %s -lblas' % (spec['lapack'].prefix.lib, spec['blas'].prefix.lib), # FIXME: avoid hardcoding blas/lapack lib names + 'BLASLIB = %s %s' % (to_link_flags(spec['lapack'].lapack_shared_lib),to_link_flags(spec['blas'].blas_shared_lib)), 'METISLIB = -L%s -lmetis' % spec['metis'].prefix.lib, 'PARMETISLIB = -L%s -lparmetis' % spec['parmetis'].prefix.lib, 'FLIBS =', @@ -58,12 +57,12 @@ def install(self, spec, prefix): 'ARCH = ar', 'ARCHFLAGS = cr', 'RANLIB = true', - 'CC = mpicc', # FIXME avoid hardcoding MPI compiler names + 'CC = %s' % spec['mpi'].mpicc, 'CFLAGS = -fPIC -std=c99 -O2 -I%s -I%s' %(spec['parmetis'].prefix.include, spec['metis'].prefix.include), 'NOOPTS = -fPIC -std=c99', - 'FORTRAN = mpif77', + 'FORTRAN = %s' % spec['mpi'].mpif77, 'F90FLAGS = -O2', - 'LOADER = mpif77', + 'LOADER = %s' % spec['mpi'].mpif77, 'LOADOPTS =', 'CDEFS = -DAdd_' ]) From 9ceb8fea78dfa19526eb258eaacf5c4c8373a093 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Tue, 14 Jun 2016 15:56:21 +0200 Subject: [PATCH 372/481] petsc: add 3.6.4 and 3.7.2; specify superlu-dist version dependency --- var/spack/repos/builtin/packages/petsc/package.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/petsc/package.py b/var/spack/repos/builtin/packages/petsc/package.py index 6456a1aabf..697dc56214 100644 --- a/var/spack/repos/builtin/packages/petsc/package.py +++ b/var/spack/repos/builtin/packages/petsc/package.py @@ -35,6 +35,8 @@ class Petsc(Package): homepage = "http://www.mcs.anl.gov/petsc/index.html" url = "http://ftp.mcs.anl.gov/pub/petsc/release-snapshots/petsc-3.5.3.tar.gz" + version('3.7.2', '50da49867ce7a49e7a0c1b37f4ec7b34') + version('3.6.4', '7632da2375a3df35b8891c9526dbdde7') version('3.6.3', '91dd3522de5a5ef039ff8f50800db606') version('3.5.3', 'd4fd2734661e89f18ac6014b5dd1ef2f') version('3.5.2', 'ad170802b3b058b5deb9cd1f968e7e13') @@ -72,7 +74,8 @@ class Petsc(Package): # Also PETSc prefer to build it without internal superlu, likely due to conflict in headers # see https://bitbucket.org/petsc/petsc/src/90564b43f6b05485163c147b464b5d6d28cde3ef/config/BuildSystem/config/packages/hypre.py depends_on('hypre~internal-superlu', when='+hypre+mpi~complex') - depends_on('superlu-dist', when='+superlu-dist+mpi') + depends_on('superlu-dist@:4.3', when='@:3.6.4+superlu-dist+mpi') + depends_on('superlu-dist@5.0.0:', when='@3.7:+superlu-dist+mpi') depends_on('mumps+mpi', when='+mumps+mpi') depends_on('scalapack', when='+mumps+mpi') From 5358ffbe5a9cbad3af1f5c37a9a239b0e7422bc9 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Tue, 14 Jun 2016 15:58:36 +0200 Subject: [PATCH 373/481] slepc: add 3.6.3 and 3.7.1; specify petsc version dependency --- var/spack/repos/builtin/packages/slepc/package.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/slepc/package.py b/var/spack/repos/builtin/packages/slepc/package.py index c148a579ec..2ce72ffa69 100644 --- a/var/spack/repos/builtin/packages/slepc/package.py +++ b/var/spack/repos/builtin/packages/slepc/package.py @@ -34,11 +34,14 @@ class Slepc(Package): homepage = "http://www.grycap.upv.es/slepc" url = "http://slepc.upv.es/download/download.php?filename=slepc-3.6.2.tar.gz" + version('3.7.1', '670216f263e3074b21e0623c01bc0f562fdc0bffcd7bd42dd5d8edbe73a532c2') + version('3.6.3', '384939d009546db37bc05ed81260c8b5ba451093bf891391d32eb7109ccff876') version('3.6.2', '2ab4311bed26ccf7771818665991b2ea3a9b15f97e29fd13911ab1293e8e65df') variant('arpack', default=False, description='Enables Arpack wrappers') - depends_on('petsc') + depends_on('petsc@3.7:', when='@3.7.1:') + depends_on('petsc@3.6.3:3.6.4', when='@3.6.2:3.6.3') depends_on('arpack-ng~mpi',when='+arpack^petsc~mpi') depends_on('arpack-ng+mpi',when='+arpack^petsc+mpi') From 4d40f86bd30e747a1fba518cab4a8acb760a61c7 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Tue, 14 Jun 2016 15:58:59 +0200 Subject: [PATCH 374/481] trilinos: add 12.6.2 and 12.6.3 --- var/spack/repos/builtin/packages/trilinos/package.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/var/spack/repos/builtin/packages/trilinos/package.py b/var/spack/repos/builtin/packages/trilinos/package.py index 6913d79dcc..cebcf1d360 100644 --- a/var/spack/repos/builtin/packages/trilinos/package.py +++ b/var/spack/repos/builtin/packages/trilinos/package.py @@ -45,6 +45,8 @@ class Trilinos(Package): homepage = "https://trilinos.org/" url = "http://trilinos.csbsju.edu/download/files/trilinos-12.2.1-Source.tar.gz" + version('12.6.3', '960f5f4d3f7c3da818e5a5fb4684559eff7e0c25f959ef576561b8a52f0e4d1e') + version('12.6.2', '0c076090508170ddee5efeed317745027f9418319720dc40a072e478775279f9') version('12.6.1', 'adcf2d3aab74cdda98f88fee19cd1442604199b0515ee3da4d80cbe8f37d00e4') version('12.4.2', '7c830f7f0f68b8ad324690603baf404e') version('12.2.1', '6161926ea247863c690e927687f83be9') From d04dc8440f61df77f79bd1fd3bb5cd799e83e4f0 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Tue, 14 Jun 2016 15:59:49 +0200 Subject: [PATCH 375/481] dealii: specify petsc/slepc version dependencies --- var/spack/repos/builtin/packages/dealii/package.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/dealii/package.py b/var/spack/repos/builtin/packages/dealii/package.py index 54b6426d36..d459976cfe 100644 --- a/var/spack/repos/builtin/packages/dealii/package.py +++ b/var/spack/repos/builtin/packages/dealii/package.py @@ -80,8 +80,8 @@ class Dealii(Package): depends_on("netcdf-cxx", when='+netcdf+mpi') depends_on("oce", when='+oce') depends_on("p4est", when='+p4est+mpi') - depends_on("petsc+mpi", when='+petsc+mpi') - depends_on("slepc", when='+slepc+petsc+mpi') + depends_on("petsc@:3.6.4+mpi", when='+petsc+mpi') # FIXME: update after 3.7 is supported upstream + depends_on("slepc@:3.6.3", when='+slepc+petsc+mpi') depends_on("trilinos", when='+trilinos+mpi') # developer dependnecies From d0d83cfe1e3f40867bd09b38240aaea5222c33eb Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Tue, 14 Jun 2016 16:00:37 +0200 Subject: [PATCH 376/481] slepc: build by default with Arpack --- var/spack/repos/builtin/packages/slepc/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/slepc/package.py b/var/spack/repos/builtin/packages/slepc/package.py index 2ce72ffa69..a14ede39c4 100644 --- a/var/spack/repos/builtin/packages/slepc/package.py +++ b/var/spack/repos/builtin/packages/slepc/package.py @@ -38,7 +38,7 @@ class Slepc(Package): version('3.6.3', '384939d009546db37bc05ed81260c8b5ba451093bf891391d32eb7109ccff876') version('3.6.2', '2ab4311bed26ccf7771818665991b2ea3a9b15f97e29fd13911ab1293e8e65df') - variant('arpack', default=False, description='Enables Arpack wrappers') + variant('arpack', default=True, description='Enables Arpack wrappers') depends_on('petsc@3.7:', when='@3.7.1:') depends_on('petsc@3.6.3:3.6.4', when='@3.6.2:3.6.3') From 36decbfa69e4eda472556ad6e813e14366443de2 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Wed, 15 Jun 2016 12:21:05 -0400 Subject: [PATCH 377/481] tty: do not break long words These "words" tend to be file paths which are best left unbroken. Fixes #1057. --- lib/spack/llnl/util/tty/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/spack/llnl/util/tty/__init__.py b/lib/spack/llnl/util/tty/__init__.py index c638b113fd..ee81e11a20 100644 --- a/lib/spack/llnl/util/tty/__init__.py +++ b/lib/spack/llnl/util/tty/__init__.py @@ -64,12 +64,14 @@ def info(message, *args, **kwargs): format = kwargs.get('format', '*b') stream = kwargs.get('stream', sys.stdout) wrap = kwargs.get('wrap', False) + break_long_words = kwargs.get('break_long_words', False) cprint("@%s{==>} %s" % (format, cescape(str(message))), stream=stream) for arg in args: if wrap: lines = textwrap.wrap( - str(arg), initial_indent=indent, subsequent_indent=indent) + str(arg), initial_indent=indent, subsequent_indent=indent, + break_long_words=break_long_words) for line in lines: stream.write(line + '\n') else: From 3d3a26cd9ab37ecbc936fe7b5937fb4f00d2594a Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Wed, 15 Jun 2016 18:31:10 +0200 Subject: [PATCH 378/481] flake8 fixes for filesystem.py --- lib/spack/llnl/util/filesystem.py | 65 +++++++++++++++++-------------- 1 file changed, 36 insertions(+), 29 deletions(-) diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index 431289488d..d72e8bae92 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -22,28 +22,28 @@ # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -__all__ = ['set_install_permissions', 'install', 'install_tree', 'traverse_tree', - 'expand_user', 'working_dir', 'touch', 'touchp', 'mkdirp', - 'force_remove', 'join_path', 'ancestor', 'can_access', 'filter_file', - 'FileFilter', 'change_sed_delimiter', 'is_exe', 'force_symlink', - 'set_executable', 'copy_mode', 'unset_executable_mode', - 'remove_dead_links', 'remove_linked_tree', 'find_library_path', - 'fix_darwin_install_name','to_link_flags'] - import os import glob -import sys import re import shutil import stat import errno import getpass from contextlib import contextmanager, closing -from tempfile import NamedTemporaryFile import subprocess import llnl.util.tty as tty -from spack.util.compression import ALLOWED_ARCHIVE_TYPES + +__all__ = ['set_install_permissions', 'install', 'install_tree', + 'traverse_tree', + 'expand_user', 'working_dir', 'touch', 'touchp', 'mkdirp', + 'force_remove', 'join_path', 'ancestor', 'can_access', + 'filter_file', + 'FileFilter', 'change_sed_delimiter', 'is_exe', 'force_symlink', + 'set_executable', 'copy_mode', 'unset_executable_mode', + 'remove_dead_links', 'remove_linked_tree', 'find_library_path', + 'fix_darwin_install_name', 'to_link_flags'] + def filter_file(regex, repl, *filenames, **kwargs): """Like sed, but uses python regular expressions. @@ -69,6 +69,7 @@ def filter_file(regex, repl, *filenames, **kwargs): # Allow strings to use \1, \2, etc. for replacement, like sed if not callable(repl): unescaped = repl.replace(r'\\', '\\') + def replace_groups_with_groupid(m): def groupid_to_group(x): return m.group(int(x.group(1))) @@ -157,9 +158,12 @@ def set_install_permissions(path): def copy_mode(src, dest): src_mode = os.stat(src).st_mode dest_mode = os.stat(dest).st_mode - if src_mode & stat.S_IXUSR: dest_mode |= stat.S_IXUSR - if src_mode & stat.S_IXGRP: dest_mode |= stat.S_IXGRP - if src_mode & stat.S_IXOTH: dest_mode |= stat.S_IXOTH + if src_mode & stat.S_IXUSR: + dest_mode |= stat.S_IXUSR + if src_mode & stat.S_IXGRP: + dest_mode |= stat.S_IXGRP + if src_mode & stat.S_IXOTH: + dest_mode |= stat.S_IXOTH os.chmod(dest, dest_mode) @@ -224,9 +228,10 @@ def force_remove(*paths): for path in paths: try: os.remove(path) - except OSError, e: + except OSError: pass + @contextmanager def working_dir(dirname, **kwargs): if kwargs.get('create', False): @@ -240,7 +245,7 @@ def working_dir(dirname, **kwargs): def touch(path): """Creates an empty file at the specified path.""" - with open(path, 'a') as file: + with open(path, 'a'): os.utime(path, None) @@ -253,7 +258,7 @@ def touchp(path): def force_symlink(src, dest): try: os.symlink(src, dest) - except OSError as e: + except OSError: os.remove(dest) os.symlink(src, dest) @@ -275,7 +280,7 @@ def ancestor(dir, n=1): def can_access(file_name): """True if we have read/write access to the file.""" - return os.access(file_name, os.R_OK|os.W_OK) + return os.access(file_name, os.R_OK | os.W_OK) def traverse_tree(source_root, dest_root, rel_path='', **kwargs): @@ -343,13 +348,14 @@ def traverse_tree(source_root, dest_root, rel_path='', **kwargs): # Treat as a directory if os.path.isdir(source_child) and ( - follow_links or not os.path.islink(source_child)): + follow_links or not os.path.islink(source_child)): # When follow_nonexisting isn't set, don't descend into dirs # in source that do not exist in dest if follow_nonexisting or os.path.exists(dest_child): - tuples = traverse_tree(source_root, dest_root, rel_child, **kwargs) - for t in tuples: yield t + tuples = traverse_tree(source_root, dest_root, rel_child, **kwargs) # NOQA: ignore=E501 + for t in tuples: + yield t # Treat as a file. elif not ignore(os.path.join(rel_path, f)): @@ -379,6 +385,7 @@ def remove_dead_links(root): if not os.path.exists(real_path): os.unlink(path) + def remove_linked_tree(path): """ Removes a directory and its contents. If the directory is a @@ -402,25 +409,25 @@ def fix_darwin_install_name(path): Fix install name of dynamic libraries on Darwin to have full path. There are two parts of this task: (i) use install_name('-id',...) to change install name of a single lib; - (ii) use install_name('-change',...) to change the cross linking between libs. - The function assumes that all libraries are in one folder and currently won't - follow subfolders. + (ii) use install_name('-change',...) to change the cross linking between + libs. The function assumes that all libraries are in one folder and + currently won't follow subfolders. Args: path: directory in which .dylib files are alocated """ - libs = glob.glob(join_path(path,"*.dylib")) + libs = glob.glob(join_path(path, "*.dylib")) for lib in libs: # fix install name first: - subprocess.Popen(["install_name_tool", "-id",lib,lib], stdout=subprocess.PIPE).communicate()[0] - long_deps = subprocess.Popen(["otool", "-L",lib], stdout=subprocess.PIPE).communicate()[0].split('\n') + subprocess.Popen(["install_name_tool", "-id", lib, lib], stdout=subprocess.PIPE).communicate()[0] # NOQA: ignore=E501 + long_deps = subprocess.Popen(["otool", "-L", lib], stdout=subprocess.PIPE).communicate()[0].split('\n') # NOQA: ignore=E501 deps = [dep.partition(' ')[0][1::] for dep in long_deps[2:-1]] # fix all dependencies: for dep in deps: for loc in libs: if dep == os.path.basename(loc): - subprocess.Popen(["install_name_tool", "-change",dep,loc,lib], stdout=subprocess.PIPE).communicate()[0] + subprocess.Popen(["install_name_tool", "-change", dep, loc, lib], stdout=subprocess.PIPE).communicate()[0] # NOQA: ignore=E501 break @@ -433,7 +440,7 @@ def to_link_flags(library): dir = os.path.dirname(library) # Asume libXYZ.suffix name = os.path.basename(library)[3:].split(".")[0] - res = '-L%s -l%s' % (dir,name) + res = '-L%s -l%s' % (dir, name) return res From 6d714e4425d91370548d0ff6f138959149a82386 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Wed, 15 Jun 2016 18:31:39 +0200 Subject: [PATCH 379/481] flake8 fixes for superlu-dist --- .../builtin/packages/superlu-dist/package.py | 36 +++++++++++-------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/var/spack/repos/builtin/packages/superlu-dist/package.py b/var/spack/repos/builtin/packages/superlu-dist/package.py index d24f03c230..e51d7224d9 100644 --- a/var/spack/repos/builtin/packages/superlu-dist/package.py +++ b/var/spack/repos/builtin/packages/superlu-dist/package.py @@ -25,8 +25,10 @@ from spack import * import glob + class SuperluDist(Package): - """A general purpose library for the direct solution of large, sparse, nonsymmetric systems of linear equations on high performance machines.""" + """A general purpose library for the direct solution of large, sparse, + nonsymmetric systems of linear equations on high performance machines.""" homepage = "http://crd-legacy.lbl.gov/~xiaoye/SuperLU/" url = "http://crd-legacy.lbl.gov/~xiaoye/SuperLU/superlu_dist_4.1.tar.gz" @@ -36,11 +38,11 @@ class SuperluDist(Package): version('4.1', '4edee38cc29f687bd0c8eb361096a455') version('4.0', 'c0b98b611df227ae050bc1635c6940e0') - depends_on ('mpi') - depends_on ('blas') - depends_on ('lapack') - depends_on ('parmetis') - depends_on ('metis@5:') + depends_on('mpi') + depends_on('blas') + depends_on('lapack') + depends_on('parmetis') + depends_on('metis@5:') def install(self, spec, prefix): makefile_inc = [] @@ -49,25 +51,28 @@ def install(self, spec, prefix): 'DSuperLUroot = %s' % self.stage.source_path, 'DSUPERLULIB = $(DSuperLUroot)/lib/libsuperlu_dist.a', 'BLASDEF = -DUSE_VENDOR_BLAS', - 'BLASLIB = %s %s' % (to_link_flags(spec['lapack'].lapack_shared_lib),to_link_flags(spec['blas'].blas_shared_lib)), + 'BLASLIB = %s %s' % + (to_link_flags(spec['lapack'].lapack_shared_lib), + to_link_flags(spec['blas'].blas_shared_lib)), 'METISLIB = -L%s -lmetis' % spec['metis'].prefix.lib, 'PARMETISLIB = -L%s -lparmetis' % spec['parmetis'].prefix.lib, 'FLIBS =', - 'LIBS = $(DSUPERLULIB) $(BLASLIB) $(PARMETISLIB) $(METISLIB)', + 'LIBS = $(DSUPERLULIB) $(BLASLIB) $(PARMETISLIB) $(METISLIB)', # NOQA: ignore=E501 'ARCH = ar', 'ARCHFLAGS = cr', 'RANLIB = true', 'CC = %s' % spec['mpi'].mpicc, - 'CFLAGS = -fPIC -std=c99 -O2 -I%s -I%s' %(spec['parmetis'].prefix.include, spec['metis'].prefix.include), + 'CFLAGS = -fPIC -std=c99 -O2 -I%s -I%s' % + (spec['parmetis'].prefix.include, + spec['metis'].prefix.include), 'NOOPTS = -fPIC -std=c99', 'FORTRAN = %s' % spec['mpi'].mpif77, 'F90FLAGS = -O2', 'LOADER = %s' % spec['mpi'].mpif77, 'LOADOPTS =', 'CDEFS = -DAdd_' - ]) + ]) - #with working_dir('src'): with open('make.inc', 'w') as fh: fh.write('\n'.join(makefile_inc)) @@ -82,9 +87,10 @@ def install(self, spec, prefix): mkdirp(headers_location) mkdirp(prefix.lib) - headers = glob.glob(join_path(self.stage.source_path, 'SRC','*.h')) + headers = glob.glob(join_path(self.stage.source_path, 'SRC', '*.h')) for h in headers: - install(h,headers_location) + install(h, headers_location) - superludist_lib = join_path(self.stage.source_path, 'lib/libsuperlu_dist.a') - install(superludist_lib,self.prefix.lib) + superludist_lib = join_path(self.stage.source_path, + 'lib/libsuperlu_dist.a') + install(superludist_lib, self.prefix.lib) From 6390acc660cfa1a402cd89ee9c2f7688ea1d6229 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Wed, 15 Jun 2016 18:32:42 +0200 Subject: [PATCH 380/481] flake8 fixes for petsc --- .../repos/builtin/packages/petsc/package.py | 45 +++++++++++-------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/var/spack/repos/builtin/packages/petsc/package.py b/var/spack/repos/builtin/packages/petsc/package.py index 697dc56214..8dd1d8d2b9 100644 --- a/var/spack/repos/builtin/packages/petsc/package.py +++ b/var/spack/repos/builtin/packages/petsc/package.py @@ -28,8 +28,9 @@ class Petsc(Package): """ - PETSc is a suite of data structures and routines for the scalable (parallel) solution of scientific applications - modeled by partial differential equations. + PETSc is a suite of data structures and routines for the scalable + (parallel) solution of scientific applications modeled by partial + differential equations. """ homepage = "http://www.mcs.anl.gov/petsc/index.html" @@ -71,8 +72,9 @@ class Petsc(Package): depends_on('hdf5+mpi', when='+hdf5+mpi') depends_on('parmetis', when='+metis+mpi') # Hypre does not support complex numbers. - # Also PETSc prefer to build it without internal superlu, likely due to conflict in headers - # see https://bitbucket.org/petsc/petsc/src/90564b43f6b05485163c147b464b5d6d28cde3ef/config/BuildSystem/config/packages/hypre.py + # Also PETSc prefer to build it without internal superlu, likely due to + # conflict in headers see + # https://bitbucket.org/petsc/petsc/src/90564b43f6b05485163c147b464b5d6d28cde3ef/config/BuildSystem/config/packages/hypre.py # NOQA: ignore=E501 depends_on('hypre~internal-superlu', when='+hypre+mpi~complex') depends_on('superlu-dist@:4.3', when='@:3.6.4+superlu-dist+mpi') depends_on('superlu-dist@5.0.0:', when='@3.7:+superlu-dist+mpi') @@ -83,17 +85,17 @@ def mpi_dependent_options(self): if '~mpi' in self.spec: compiler_opts = [ '--with-cc=%s' % os.environ['CC'], - '--with-cxx=%s' % (os.environ['CXX'] if self.compiler.cxx is not None else '0'), - '--with-fc=%s' % (os.environ['FC'] if self.compiler.fc is not None else '0'), + '--with-cxx=%s' % (os.environ['CXX'] if self.compiler.cxx is not None else '0'), # NOQA: ignore=E501 + '--with-fc=%s' % (os.environ['FC'] if self.compiler.fc is not None else '0'), # NOQA: ignore=E501 '--with-mpi=0' ] - error_message_fmt = '\t{library} support requires "+mpi" to be activated' + error_message_fmt = '\t{library} support requires "+mpi" to be activated' # NOQA: ignore=E501 - # If mpi is disabled (~mpi), it's an error to have any of these enabled. - # This generates a list of any such errors. + # If mpi is disabled (~mpi), it's an error to have any of these + # enabled. This generates a list of any such errors. errors = [error_message_fmt.format(library=x) - for x in ('hdf5', 'hypre', 'parmetis','mumps','superlu-dist') - if ('+'+x) in self.spec] + for x in ('hdf5', 'hypre', 'parmetis', 'mumps', 'superlu-dist') # NOQA: ignore=E501 + if ('+' + x) in self.spec] if errors: errors = ['incompatible variants given'] + errors raise RuntimeError('\n'.join(errors)) @@ -108,26 +110,31 @@ def install(self, spec, prefix): options = ['--with-ssl=0'] options.extend(self.mpi_dependent_options()) options.extend([ - '--with-precision=%s' % ('double' if '+double' in spec else 'single'), - '--with-scalar-type=%s' % ('complex' if '+complex' in spec else 'real'), + '--with-precision=%s' % ('double' if '+double' in spec else 'single'), # NOQA: ignore=E501 + '--with-scalar-type=%s' % ('complex' if '+complex' in spec else 'real'), # NOQA: ignore=E501 '--with-shared-libraries=%s' % ('1' if '+shared' in spec else '0'), '--with-debugging=%s' % ('1' if '+debug' in spec else '0'), '--with-blas-lapack-dir=%s' % spec['lapack'].prefix ]) # Activates library support if needed - for library in ('metis', 'boost', 'hdf5', 'hypre', 'parmetis','mumps','scalapack'): + for library in ('metis', 'boost', 'hdf5', 'hypre', 'parmetis', + 'mumps', 'scalapack'): options.append( - '--with-{library}={value}'.format(library=library, value=('1' if library in spec else '0')) + '--with-{library}={value}'.format(library=library, value=('1' if library in spec else '0')) # NOQA: ignore=E501 ) if library in spec: options.append( - '--with-{library}-dir={path}'.format(library=library, path=spec[library].prefix) + '--with-{library}-dir={path}'.format(library=library, path=spec[library].prefix) # NOQA: ignore=E501 ) - # PETSc does not pick up SuperluDist from the dir as they look for superlu_dist_4.1.a + # PETSc does not pick up SuperluDist from the dir as they look for + # superlu_dist_4.1.a if 'superlu-dist' in spec: options.extend([ - '--with-superlu_dist-include=%s' % spec['superlu-dist'].prefix.include, - '--with-superlu_dist-lib=%s' % join_path(spec['superlu-dist'].prefix.lib, 'libsuperlu_dist.a'), + '--with-superlu_dist-include=%s' % + spec['superlu-dist'].prefix.include, + '--with-superlu_dist-lib=%s' % + join_path(spec['superlu-dist'].prefix.lib, + 'libsuperlu_dist.a'), '--with-superlu_dist=1' ]) else: From e76083205f09cd19feae91c6779fc70f229af2b6 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Wed, 15 Jun 2016 18:33:08 +0200 Subject: [PATCH 381/481] flake8 fixes for slepc --- var/spack/repos/builtin/packages/slepc/package.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/var/spack/repos/builtin/packages/slepc/package.py b/var/spack/repos/builtin/packages/slepc/package.py index a14ede39c4..d3739bf6a6 100644 --- a/var/spack/repos/builtin/packages/slepc/package.py +++ b/var/spack/repos/builtin/packages/slepc/package.py @@ -42,8 +42,8 @@ class Slepc(Package): depends_on('petsc@3.7:', when='@3.7.1:') depends_on('petsc@3.6.3:3.6.4', when='@3.6.2:3.6.3') - depends_on('arpack-ng~mpi',when='+arpack^petsc~mpi') - depends_on('arpack-ng+mpi',when='+arpack^petsc+mpi') + depends_on('arpack-ng~mpi', when='+arpack^petsc~mpi') + depends_on('arpack-ng+mpi', when='+arpack^petsc+mpi') def install(self, spec, prefix): # set SLEPC_DIR for installation @@ -67,7 +67,7 @@ def install(self, spec, prefix): configure('--prefix=%s' % prefix, *options) make('MAKE_NP=%s' % make_jobs, parallel=False) - #FIXME: + # FIXME: # make('test') make('install') From 5fe531050c33f874882adcbe3a657628fe8cd517 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Wed, 15 Jun 2016 18:33:34 +0200 Subject: [PATCH 382/481] flake8 fixes for dealii --- .../repos/builtin/packages/dealii/package.py | 33 +++++++++---------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/var/spack/repos/builtin/packages/dealii/package.py b/var/spack/repos/builtin/packages/dealii/package.py index d459976cfe..23ec74abed 100644 --- a/var/spack/repos/builtin/packages/dealii/package.py +++ b/var/spack/repos/builtin/packages/dealii/package.py @@ -80,7 +80,7 @@ class Dealii(Package): depends_on("netcdf-cxx", when='+netcdf+mpi') depends_on("oce", when='+oce') depends_on("p4est", when='+p4est+mpi') - depends_on("petsc@:3.6.4+mpi", when='+petsc+mpi') # FIXME: update after 3.7 is supported upstream + depends_on("petsc@:3.6.4+mpi", when='+petsc+mpi') # FIXME: update after 3.7 is supported upstream. # NOQA: ignore=E501 depends_on("slepc@:3.6.3", when='+slepc+petsc+mpi') depends_on("trilinos", when='+trilinos+mpi') @@ -108,12 +108,11 @@ def install(self, spec, prefix): # of Spack's. Be more specific to avoid this. # Note that both lapack and blas are provided in -DLAPACK_XYZ. '-DLAPACK_FOUND=true', - '-DLAPACK_INCLUDE_DIRS=%s;%s' % - (spec['lapack'].prefix.include, - spec['blas'].prefix.include), - '-DLAPACK_LIBRARIES=%s;%s' % - (spec['lapack'].lapack_shared_lib, - spec['blas'].blas_shared_lib), + '-DLAPACK_INCLUDE_DIRS=%s;%s' % ( + spec['lapack'].prefix.include, spec['blas'].prefix.include), + '-DLAPACK_LIBRARIES=%s;%s' % ( + spec['lapack'].lapack_shared_lib, + spec['blas'].blas_shared_lib), '-DMUPARSER_DIR=%s' % spec['muparser'].prefix, '-DUMFPACK_DIR=%s' % spec['suite-sparse'].prefix, '-DTBB_DIR=%s' % spec['tbb'].prefix, @@ -168,14 +167,14 @@ def install(self, spec, prefix): if '+netcdf' in spec: options.extend([ '-DNETCDF_FOUND=true', - '-DNETCDF_LIBRARIES=%s;%s' % - (join_path(spec['netcdf-cxx'].prefix.lib, - 'libnetcdf_c++.%s' % dsuf), - join_path(spec['netcdf'].prefix.lib, - 'libnetcdf.%s' % dsuf)), - '-DNETCDF_INCLUDE_DIRS=%s;%s' % - (spec['netcdf-cxx'].prefix.include, - spec['netcdf'].prefix.include), + '-DNETCDF_LIBRARIES=%s;%s' % ( + join_path(spec['netcdf-cxx'].prefix.lib, + 'libnetcdf_c++.%s' % dsuf), + join_path(spec['netcdf'].prefix.lib, + 'libnetcdf.%s' % dsuf)), + '-DNETCDF_INCLUDE_DIRS=%s;%s' % ( + spec['netcdf-cxx'].prefix.include, + spec['netcdf'].prefix.include), ]) else: options.extend([ @@ -266,9 +265,9 @@ def install(self, spec, prefix): filter_file(r'(LA::SolverCG solver\(solver_control\);)', ('TrilinosWrappers::SolverDirect::AdditionalData data(false,"Amesos_Superludist"); TrilinosWrappers::SolverDirect solver(solver_control,data);'), 'step-40.cc') # NOQA: ignore=E501 filter_file(r'(LA::MPI::PreconditionAMG preconditioner;)', (''), 'step-40.cc') - filter_file(r'(LA::MPI::PreconditionAMG::AdditionalData data;)', + filter_file(r'(LA::MPI::PreconditionAMG::AdditionalData data;)', # NOQA: ignore=E501 (''), 'step-40.cc') - filter_file(r'(preconditioner.initialize\(system_matrix, data\);)', + filter_file(r'(preconditioner.initialize\(system_matrix, data\);)', # NOQA: ignore=E501 (''), 'step-40.cc') filter_file(r'(solver\.solve \(system_matrix, completely_distributed_solution, system_rhs,)', ('solver.solve (system_matrix, completely_distributed_solution, system_rhs);'), 'step-40.cc') # NOQA: ignore=E501 filter_file(r'(preconditioner\);)', (''), 'step-40.cc') From 0e04e65ae0108437e6bf5d3e2233cdbc775fd136 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Wed, 15 Jun 2016 18:44:59 +0200 Subject: [PATCH 383/481] trilinos: specify superlu-dist version dependency --- var/spack/repos/builtin/packages/trilinos/package.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/trilinos/package.py b/var/spack/repos/builtin/packages/trilinos/package.py index cebcf1d360..3a53ac5c01 100644 --- a/var/spack/repos/builtin/packages/trilinos/package.py +++ b/var/spack/repos/builtin/packages/trilinos/package.py @@ -91,7 +91,8 @@ class Trilinos(Package): # work at the end. But let's avoid all this by simply using shared libs depends_on('mumps@5.0:+mpi+shared', when='+mumps') depends_on('scalapack', when='+mumps') - depends_on('superlu-dist', when='+superlu-dist') + depends_on('superlu-dist@:4.3', when='@:12.6.1+superlu-dist') + depends_on('superlu-dist', when='@12.6.2:+superlu-dist') depends_on('hypre~internal-superlu', when='+hypre') depends_on('hdf5+mpi', when='+hdf5') depends_on('python', when='+python') From 716f12dd580cb6f78f085d94e5a45679ba6c8041 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Wed, 15 Jun 2016 12:48:40 -0400 Subject: [PATCH 384/481] reindex: preserve 'explicit' flags Look up in the old database whether the spec was explicit or not and preserve it into the new database. Fixes #1050. --- lib/spack/spack/database.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py index e768ddf5fe..59d577f0c7 100644 --- a/lib/spack/spack/database.py +++ b/lib/spack/spack/database.py @@ -309,7 +309,11 @@ def reindex(self, directory_layout): for spec in directory_layout.all_specs(): # Create a spec for each known package and add it. path = directory_layout.path_for_spec(spec) - self._add(spec, path, directory_layout) + old_info = old_data.get(spec.dag_hash()) + explicit = False + if old_info is not None: + explicit = old_info.explicit + self._add(spec, path, directory_layout, explicit=explicit) self._check_ref_counts() From 9d99042fed2abad1ccb22669bc1dfa298d495768 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Fri, 3 Jun 2016 15:12:10 -0500 Subject: [PATCH 385/481] Add Psi4 package --- .../repos/builtin/packages/boost/package.py | 2 +- .../repos/builtin/packages/psi4/package.py | 84 +++++++++++++++++++ 2 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 var/spack/repos/builtin/packages/psi4/package.py diff --git a/var/spack/repos/builtin/packages/boost/package.py b/var/spack/repos/builtin/packages/boost/package.py index 8a49672824..d833054e33 100644 --- a/var/spack/repos/builtin/packages/boost/package.py +++ b/var/spack/repos/builtin/packages/boost/package.py @@ -25,7 +25,6 @@ from spack import * import spack import sys - import os @@ -91,6 +90,7 @@ class Boost(Package): 'system', 'test', 'thread', + 'timer', 'wave']) # mpi/python are not installed by default because they pull in many diff --git a/var/spack/repos/builtin/packages/psi4/package.py b/var/spack/repos/builtin/packages/psi4/package.py new file mode 100644 index 0000000000..5c2ce369c9 --- /dev/null +++ b/var/spack/repos/builtin/packages/psi4/package.py @@ -0,0 +1,84 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class Psi4(Package): + """Psi4 is an open-source suite of ab initio quantum chemistry + programs designed for efficient, high-accuracy simulations of + a variety of molecular properties.""" + + homepage = "http://www.psicode.org/" + url = "https://github.com/psi4/psi4/archive/0.5.tar.gz" + + version('0.5', '53041b8a9be3958384171d0d22f9fdd0') + + variant('mpi', default=True, description='Enable MPI parallelization') + + # Required dependencies + depends_on('python@2.7:') + depends_on('py-numpy') + depends_on('blas') + depends_on('lapack') + depends_on('cmake@3.0:') + depends_on('boost@1.55.0:+chrono+filesystem~mpi+python+regex+serialization+system+timer+thread', when='~mpi') + depends_on('boost@1.55.0:+chrono+filesystem+mpi+python+regex+serialization+system+timer+thread', when='+mpi') + + # Optional dependencies + depends_on('mpi', when='+mpi') + # TODO: add packages for these + # depends_on('perl') + # depends_on('erd') + # depends_on('pcm-solver') + # depends_on('chemps2') + + def install(self, spec, prefix): + cmake_args = [ + '-DBLAS_TYPE={0}'.format(spec['blas'].name.upper()), + '-DBLAS_LIBRARIES={0}'.format(spec['blas'].blas_shared_lib), + '-DLAPACK_TYPE={0}'.format(spec['lapack'].name.upper()), + '-DLAPACK_LIBRARIES={0}'.format(spec['lapack'].lapack_shared_lib), + '-DBOOST_INCLUDEDIR={0}'.format(spec['boost'].prefix.include), + '-DBOOST_LIBRARYDIR={0}'.format(spec['boost'].prefix.lib), + '-DENABLE_CHEMPS2=OFF' + ] + + if '+mpi' in spec: + cmake_args.extend([ + '-DENABLE_MPI=ON' + #'-DMPI_C_COMPILER={0}'.format(spec['mpi'].mpicc) + #'-DMPI_C_INCLUDE_PATH={0}'.format(spec['mpi'].prefix.include), + ]) + else: + cmake_args.append('-DENABLE_MPI=OFF') + + cmake_args.extend(std_cmake_args) + + with working_dir('spack-build', create=True): + cmake('..', *cmake_args) + + make() + #ctest() + make('install') From deb4f919cf64717036ee0c8c1b6cb00d46b579aa Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Mon, 13 Jun 2016 16:14:16 -0500 Subject: [PATCH 386/481] Psi4 tweaks --- .../repos/builtin/packages/psi4/package.py | 22 ++++++------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/var/spack/repos/builtin/packages/psi4/package.py b/var/spack/repos/builtin/packages/psi4/package.py index 5c2ce369c9..e02dc65fd4 100644 --- a/var/spack/repos/builtin/packages/psi4/package.py +++ b/var/spack/repos/builtin/packages/psi4/package.py @@ -38,13 +38,13 @@ class Psi4(Package): variant('mpi', default=True, description='Enable MPI parallelization') # Required dependencies - depends_on('python@2.7:') - depends_on('py-numpy') depends_on('blas') depends_on('lapack') - depends_on('cmake@3.0:') - depends_on('boost@1.55.0:+chrono+filesystem~mpi+python+regex+serialization+system+timer+thread', when='~mpi') - depends_on('boost@1.55.0:+chrono+filesystem+mpi+python+regex+serialization+system+timer+thread', when='+mpi') + depends_on('boost+chrono+filesystem~mpi+python+regex+serialization+system+timer+thread', when='~mpi') + depends_on('boost+chrono+filesystem+mpi+python+regex+serialization+system+timer+thread', when='+mpi') + depends_on('python') + depends_on('cmake') + depends_on('py-numpy') # Optional dependencies depends_on('mpi', when='+mpi') @@ -62,23 +62,15 @@ def install(self, spec, prefix): '-DLAPACK_LIBRARIES={0}'.format(spec['lapack'].lapack_shared_lib), '-DBOOST_INCLUDEDIR={0}'.format(spec['boost'].prefix.include), '-DBOOST_LIBRARYDIR={0}'.format(spec['boost'].prefix.lib), + '-DENABLE_MPI={0}'.format('ON' if '+mpi' in spec else 'OFF'), '-DENABLE_CHEMPS2=OFF' ] - if '+mpi' in spec: - cmake_args.extend([ - '-DENABLE_MPI=ON' - #'-DMPI_C_COMPILER={0}'.format(spec['mpi'].mpicc) - #'-DMPI_C_INCLUDE_PATH={0}'.format(spec['mpi'].prefix.include), - ]) - else: - cmake_args.append('-DENABLE_MPI=OFF') - cmake_args.extend(std_cmake_args) with working_dir('spack-build', create=True): cmake('..', *cmake_args) make() - #ctest() + # ctest() make('install') From 9cf1257be4b6203f5a90ee5b410902daafa53d7b Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 14 Jun 2016 11:19:05 -0500 Subject: [PATCH 387/481] Filter compilers --- .../repos/builtin/packages/psi4/package.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/var/spack/repos/builtin/packages/psi4/package.py b/var/spack/repos/builtin/packages/psi4/package.py index e02dc65fd4..4bd47302ab 100644 --- a/var/spack/repos/builtin/packages/psi4/package.py +++ b/var/spack/repos/builtin/packages/psi4/package.py @@ -23,6 +23,7 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## from spack import * +import os class Psi4(Package): @@ -74,3 +75,26 @@ def install(self, spec, prefix): make() # ctest() make('install') + + self.filter_compilers() + + def filter_compilers(self): + """Run after install to tell the configuration files to + use the compilers that Spack built the package with. + + If this isn't done, they'll have PLUGIN_CXX set to + Spack's generic cxx. We want it to be bound to + whatever compiler it was built with.""" + + kwargs = {'ignore_absent': True, 'backup': False, 'string': True} + + cc_files = ['bin/psi4-config'] + cxx_files = ['bin/psi4-config', 'include/psi4/psiconfig.h'] + + for filename in cc_files: + filter_file(os.environ['CC'], self.compiler.cc, + os.path.join(self.prefix, filename), **kwargs) + + for filename in cxx_files: + filter_file(os.environ['CXX'], self.compiler.cxx, + os.path.join(self.prefix, filename), **kwargs) From 7b97ea248a355d0dd763facd808722dfb1632f14 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Wed, 15 Jun 2016 09:33:13 -0500 Subject: [PATCH 388/481] More template filtering --- .../repos/builtin/packages/psi4/package.py | 26 ++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/var/spack/repos/builtin/packages/psi4/package.py b/var/spack/repos/builtin/packages/psi4/package.py index 4bd47302ab..74f1554336 100644 --- a/var/spack/repos/builtin/packages/psi4/package.py +++ b/var/spack/repos/builtin/packages/psi4/package.py @@ -73,12 +73,12 @@ def install(self, spec, prefix): cmake('..', *cmake_args) make() - # ctest() + ctest() make('install') - self.filter_compilers() + self.filter_compilers(spec) - def filter_compilers(self): + def filter_compilers(self, spec): """Run after install to tell the configuration files to use the compilers that Spack built the package with. @@ -90,6 +90,7 @@ def filter_compilers(self): cc_files = ['bin/psi4-config'] cxx_files = ['bin/psi4-config', 'include/psi4/psiconfig.h'] + template = 'share/psi4/plugin/Makefile.template' for filename in cc_files: filter_file(os.environ['CC'], self.compiler.cc, @@ -98,3 +99,22 @@ def filter_compilers(self): for filename in cxx_files: filter_file(os.environ['CXX'], self.compiler.cxx, os.path.join(self.prefix, filename), **kwargs) + + # The binary still keeps track of the compiler used to install Psi4 + # and uses it when creating a plugin template + filter_file('@PLUGIN_CXX@', self.compiler.cxx, + os.path.join(self.prefix, template), **kwargs) + + # The binary links to the build include directory instead of the + # installation include directory: + # https://github.com/psi4/psi4/issues/410 + filter_file('@PLUGIN_INCLUDES@', '-I{0}'.format( + ' -I'.join( + spec['psi4'].prefix.include, + spec['boost'].prefix.include, + spec['lapack'].prefix.include, + spec['blas'].prefix.include, + spec['python'].prefix.include, + '/usr/include' + ) + ), os.path.join(self.prefix, template), **kwargs) From 44e188f1111ce4251a756d92cae7c5a7e0ed41fe Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Wed, 15 Jun 2016 13:55:51 -0500 Subject: [PATCH 389/481] Remove MPI support, fix INCLUDE filters --- .../repos/builtin/packages/psi4/package.py | 37 ++++++++----------- 1 file changed, 16 insertions(+), 21 deletions(-) diff --git a/var/spack/repos/builtin/packages/psi4/package.py b/var/spack/repos/builtin/packages/psi4/package.py index 74f1554336..fc77ba7412 100644 --- a/var/spack/repos/builtin/packages/psi4/package.py +++ b/var/spack/repos/builtin/packages/psi4/package.py @@ -36,19 +36,15 @@ class Psi4(Package): version('0.5', '53041b8a9be3958384171d0d22f9fdd0') - variant('mpi', default=True, description='Enable MPI parallelization') - # Required dependencies depends_on('blas') depends_on('lapack') - depends_on('boost+chrono+filesystem~mpi+python+regex+serialization+system+timer+thread', when='~mpi') - depends_on('boost+chrono+filesystem+mpi+python+regex+serialization+system+timer+thread', when='+mpi') + depends_on('boost+chrono+filesystem+python+regex+serialization+system+timer+thread') depends_on('python') depends_on('cmake') depends_on('py-numpy') # Optional dependencies - depends_on('mpi', when='+mpi') # TODO: add packages for these # depends_on('perl') # depends_on('erd') @@ -63,7 +59,6 @@ def install(self, spec, prefix): '-DLAPACK_LIBRARIES={0}'.format(spec['lapack'].lapack_shared_lib), '-DBOOST_INCLUDEDIR={0}'.format(spec['boost'].prefix.include), '-DBOOST_LIBRARYDIR={0}'.format(spec['boost'].prefix.lib), - '-DENABLE_MPI={0}'.format('ON' if '+mpi' in spec else 'OFF'), '-DENABLE_CHEMPS2=OFF' ] @@ -73,12 +68,11 @@ def install(self, spec, prefix): cmake('..', *cmake_args) make() - ctest() make('install') - self.filter_compilers(spec) + self.filter_compilers(spec, prefix) - def filter_compilers(self, spec): + def filter_compilers(self, spec, prefix): """Run after install to tell the configuration files to use the compilers that Spack built the package with. @@ -94,27 +88,28 @@ def filter_compilers(self, spec): for filename in cc_files: filter_file(os.environ['CC'], self.compiler.cc, - os.path.join(self.prefix, filename), **kwargs) + os.path.join(prefix, filename), **kwargs) for filename in cxx_files: filter_file(os.environ['CXX'], self.compiler.cxx, - os.path.join(self.prefix, filename), **kwargs) + os.path.join(prefix, filename), **kwargs) # The binary still keeps track of the compiler used to install Psi4 # and uses it when creating a plugin template filter_file('@PLUGIN_CXX@', self.compiler.cxx, - os.path.join(self.prefix, template), **kwargs) + os.path.join(prefix, template), **kwargs) # The binary links to the build include directory instead of the # installation include directory: # https://github.com/psi4/psi4/issues/410 filter_file('@PLUGIN_INCLUDES@', '-I{0}'.format( - ' -I'.join( - spec['psi4'].prefix.include, - spec['boost'].prefix.include, - spec['lapack'].prefix.include, - spec['blas'].prefix.include, - spec['python'].prefix.include, - '/usr/include' - ) - ), os.path.join(self.prefix, template), **kwargs) + ' -I'.join([ + os.path.join(spec['psi4'].prefix.include, 'psi4'), + os.path.join(spec['boost'].prefix.include, 'boost'), + os.path.join(spec['python'].prefix.include, 'python{0}'.format( + spec['python'].version.up_to(2))), + spec['lapack'].prefix.include, + spec['blas'].prefix.include, + '/usr/include' + ]) + ), os.path.join(prefix, template), **kwargs) From daa46105a71f99946b26e1fa9836f0bb27897ceb Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Wed, 15 Jun 2016 21:37:12 +0200 Subject: [PATCH 390/481] mpich based providers : shortened filter_compilers --- .../repos/builtin/packages/mpich/package.py | 17 ++++--------- .../builtin/packages/mvapich2/package.py | 24 ++++--------------- 2 files changed, 9 insertions(+), 32 deletions(-) diff --git a/var/spack/repos/builtin/packages/mpich/package.py b/var/spack/repos/builtin/packages/mpich/package.py index 511beafbbd..ea1844aea0 100644 --- a/var/spack/repos/builtin/packages/mpich/package.py +++ b/var/spack/repos/builtin/packages/mpich/package.py @@ -106,22 +106,13 @@ def filter_compilers(self): mpif77 = join_path(bin, 'mpif77') mpif90 = join_path(bin, 'mpif90') - spack_cc = env['CC'] - spack_cxx = env['CXX'] - spack_f77 = env['F77'] - spack_fc = env['FC'] - # Substitute Spack compile wrappers for the real # underlying compiler kwargs = {'ignore_absent': True, 'backup': False, 'string': True} - filter_file('CC="%s"' % spack_cc, - 'CC="%s"' % self.compiler.cc, mpicc, **kwargs) - filter_file('CXX="%s"' % spack_cxx, - 'CXX="%s"' % self.compiler.cxx, mpicxx, **kwargs) - filter_file('F77="%s"' % spack_f77, - 'F77="%s"' % self.compiler.f77, mpif77, **kwargs) - filter_file('FC="%s"' % spack_fc, - 'FC="%s"' % self.compiler.fc, mpif90, **kwargs) + filter_file(env['CC'], self.compiler.cc, mpicc, **kwargs) + filter_file(env['CXX'], self.compiler.cxx, mpicxx, **kwargs) + filter_file(env['F77'], self.compiler.f77, mpif77, **kwargs) + filter_file(env['FC'], self.compiler.fc, mpif90, **kwargs) # Remove this linking flag if present # (it turns RPATH into RUNPATH) diff --git a/var/spack/repos/builtin/packages/mvapich2/package.py b/var/spack/repos/builtin/packages/mvapich2/package.py index 131d430cf2..34bace10d6 100644 --- a/var/spack/repos/builtin/packages/mvapich2/package.py +++ b/var/spack/repos/builtin/packages/mvapich2/package.py @@ -249,27 +249,13 @@ def filter_compilers(self): mpif77 = join_path(bin, 'mpif77') mpif90 = join_path(bin, 'mpif90') - spack_cc = env['CC'] - spack_cxx = env['CXX'] - spack_f77 = env['F77'] - spack_fc = env['FC'] - - kwargs = { - 'ignore_absent': True, - 'backup': False, - 'string': True - } - # Substitute Spack compile wrappers for the real # underlying compiler - filter_file('CC="%s"' % spack_cc, - 'CC="%s"' % self.compiler.cc, mpicc, **kwargs) - filter_file('CXX="%s"' % spack_cxx, - 'CXX="%s"' % self.compiler.cxx, mpicxx, **kwargs) - filter_file('F77="%s"' % spack_f77, - 'F77="%s"' % self.compiler.f77, mpif77, **kwargs) - filter_file('FC="%s"' % spack_fc, - 'FC="%s"' % self.compiler.fc, mpif90, **kwargs) + kwargs = {'ignore_absent': True, 'backup': False, 'string': True} + filter_file(env['CC'], self.compiler.cc, mpicc, **kwargs) + filter_file(env['CXX'], self.compiler.cxx, mpicxx, **kwargs) + filter_file(env['F77'], self.compiler.f77, mpif77, **kwargs) + filter_file(env['FC'], self.compiler.fc, mpif90, **kwargs) # Remove this linking flag if present # (it turns RPATH into RUNPATH) From ea80c8d3c3a0690dcb357840cfb0db824f44fdc9 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Wed, 15 Jun 2016 14:38:53 -0500 Subject: [PATCH 391/481] Ignore E501 for depends_on and extends --- share/spack/qa/run-flake8 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/share/spack/qa/run-flake8 b/share/spack/qa/run-flake8 index 44eb0167fb..57534eb6e2 100755 --- a/share/spack/qa/run-flake8 +++ b/share/spack/qa/run-flake8 @@ -29,6 +29,8 @@ for file in $changed; do perl -i -pe 's/^(\s*url\s*=.*)$/\1 # NOQA: ignore=E501/' $file perl -i -pe 's/^(\s*version\(.*\).*)$/\1 # NOQA: ignore=E501/' $file perl -i -pe 's/^(\s*variant\(.*\).*)$/\1 # NOQA: ignore=E501/' $file + perl -i -pe 's/^(\s*depends_on\(.*\).*)$/\1 # NOQA: ignore=E501/' $file + perl -i -pe 's/^(\s*extends\(.*\).*)$/\1 # NOQA: ignore=E501/' $file # Exempt '@when' decorated functions from redefinition errors. perl -i -pe 's/^(\s*\@when\(.*\).*)$/\1 # NOQA: ignore=F811/' $file From a7ab064a9e2361bd92bd80826908efc6122075d5 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Wed, 15 Jun 2016 18:05:23 -0400 Subject: [PATCH 392/481] Update OpenMPI to 1.10.3 --- var/spack/repos/builtin/packages/openmpi/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/openmpi/package.py b/var/spack/repos/builtin/packages/openmpi/package.py index 0638628a6c..fe06afd9fb 100644 --- a/var/spack/repos/builtin/packages/openmpi/package.py +++ b/var/spack/repos/builtin/packages/openmpi/package.py @@ -61,6 +61,7 @@ class Openmpi(Package): list_url = "http://www.open-mpi.org/software/ompi/" list_depth = 3 + version('1.10.3', 'e2fe4513200e2aaa1500b762342c674b') version('1.10.2', 'b2f43d9635d2d52826e5ef9feb97fd4c') version('1.10.1', 'f0fcd77ed345b7eafb431968124ba16e') version('1.10.0', '280cf952de68369cebaca886c5ce0304') From 2fc9ac4036eb017164cc6ac6f19e50dab5008759 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 15 Jun 2016 16:39:39 -0700 Subject: [PATCH 393/481] Two minor fixes --- lib/spack/spack/compilers/__init__.py | 10 +++++----- lib/spack/spack/yaml_version_check.py | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index ea79f655b0..c76dd252d0 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -74,7 +74,7 @@ def _to_dict(compiler): d['spec'] = str(compiler.spec) d['paths'] = dict( (attr, getattr(compiler, attr, None)) for attr in _path_instance_vars ) d['operating_system'] = compiler.operating_system.to_dict() - d['modules'] = compiler.modules + d['modules'] = compiler.modules if compiler.modules else [] if not compiler.alias: yaml_text = yaml.dump( @@ -141,11 +141,11 @@ def remove_compiler_from_config(compiler_spec, scope=None): - scope: configuration scope to modify. """ compiler_config = get_compiler_config(scope) - matches = [(a,c) for (a,c) in compiler_config.items() if c['spec'] == compiler_spec] + matches = [(a,c) for (a,c) in compiler_config.items() if c['spec'] == compiler_spec] if len(matches) == 1: del compiler_config[matches[0][0]] else: - CompilerSpecInsufficientlySpecificError(compiler_spec) + CompilerSpecInsufficientlySpecificError(compiler_spec) spack.config.update_config('compilers', compiler_config, scope) @@ -234,7 +234,7 @@ def get_compilers(cspec): continue items = cmp alias = aka - + if not ('paths' in items and all(n in items['paths'] for n in _path_instance_vars)): raise InvalidCompilerConfigurationError(cspec) @@ -309,7 +309,7 @@ def all_os_classes(): this platform """ classes = [] - + platform = spack.architecture.sys_type() for os_class in platform.operating_sys.values(): classes.append(os_class) diff --git a/lib/spack/spack/yaml_version_check.py b/lib/spack/spack/yaml_version_check.py index 7e7bae4edf..2d3d78ed39 100644 --- a/lib/spack/spack/yaml_version_check.py +++ b/lib/spack/spack/yaml_version_check.py @@ -46,10 +46,10 @@ def check_compiler_yaml_version(): if data: compilers = data['compilers'].items() if len(compilers) > 0: - if 'operating_system' not in compilers[0][1]: + if (not isinstance(compilers, list)) or 'operating_system' not in compilers[0][1]: new_file = os.path.join(scope.path, '_old_compilers.yaml') - tty.warn('%s in out of date compilers format. ' + tty.warn('%s in out of date compilers format. ' 'Moved to %s. Spack automatically generate ' - 'a compilers config file ' + 'a compilers config file ' % (file_name, new_file)) os.rename(file_name, new_file) From 1724ff565877eb36267d6947c0d900609b1bd198 Mon Sep 17 00:00:00 2001 From: "Gregory L. Lee" Date: Wed, 15 Jun 2016 20:21:02 -0700 Subject: [PATCH 394/481] do not create shebang.bak file --- lib/spack/spack/hooks/sbang.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/lib/spack/spack/hooks/sbang.py b/lib/spack/spack/hooks/sbang.py index 83d67ea225..601c2420b5 100644 --- a/lib/spack/spack/hooks/sbang.py +++ b/lib/spack/spack/hooks/sbang.py @@ -34,6 +34,7 @@ # here, as it is the shortest I could find on a modern OS. shebang_limit = 127 + def shebang_too_long(path): """Detects whether a file has a shebang line that is too long.""" with open(path, 'r') as script: @@ -57,16 +58,10 @@ def filter_shebang(path): if original.startswith(new_sbang_line): return - backup = path + ".shebang.bak" - os.rename(path, backup) - with open(path, 'w') as new_file: new_file.write(new_sbang_line) new_file.write(original) - copy_mode(backup, path) - unset_executable_mode(backup) - tty.warn("Patched overly long shebang in %s" % path) From d4df4375d561dfd7d31069e2801a9405ab06be63 Mon Sep 17 00:00:00 2001 From: "Gregory L. Lee" Date: Wed, 15 Jun 2016 20:32:08 -0700 Subject: [PATCH 395/481] removed unused import per flake8 report --- lib/spack/spack/hooks/sbang.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/spack/spack/hooks/sbang.py b/lib/spack/spack/hooks/sbang.py index 601c2420b5..cb0ad42b14 100644 --- a/lib/spack/spack/hooks/sbang.py +++ b/lib/spack/spack/hooks/sbang.py @@ -24,7 +24,6 @@ ############################################################################## import os -from llnl.util.filesystem import * import llnl.util.tty as tty import spack From c90cc465f581514d18fc4fc95649f015cb178079 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 16 Jun 2016 00:54:59 -0700 Subject: [PATCH 396/481] Add `__contains__` to arch to satsify old packages. --- lib/spack/spack/architecture.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 1f25bc14ea..df0ab5c14d 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -195,7 +195,7 @@ def operating_system(self, name): name = self.back_os return self.operating_sys.get(name, None) - + @classmethod def detect(self): @@ -206,12 +206,15 @@ def detect(self): """ raise NotImplementedError() + def __repr__(self): return self.__str__() + def __str__(self): return self.name + def _cmp_key(self): t_keys = ''.join(str(t._cmp_key()) for t in sorted(self.targets.values())) @@ -338,7 +341,6 @@ def to_dict(self): d = {} d['name'] = self.name d['version'] = self.version - return d @key_ordering @@ -378,6 +380,11 @@ def __str__(self): else: return '' + + def __contains__(self, string): + return string in str(self) + + def _cmp_key(self): if isinstance(self.platform, Platform): platform = self.platform.name From f3d6818d5c09766946652266188c7210c56f027c Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 16 Jun 2016 00:55:39 -0700 Subject: [PATCH 397/481] be more tolerant when parsing new specs. --- lib/spack/spack/spec.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 7718adb72c..54219ec1b4 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -526,7 +526,12 @@ def _add_flag(self, name, value): """ valid_flags = FlagMap.valid_compiler_flags() if name == 'arch' or name == 'architecture': - platform, op_sys, target = value.split('-') + parts = value.split('-') + if len(parts) == 3: + platform, op_sys, target = parts + else: + platform, op_sys, target = None, None, value + assert(self.architecture.platform is None) assert(self.architecture.platform_os is None) assert(self.architecture.target is None) @@ -596,11 +601,13 @@ def _set_platform(self, value): def _set_os(self, value): """Called by the parser to set the architecture operating system""" - self.architecture.platform_os = self.architecture.platform.operating_system(value) + if self.architecture.platform: + self.architecture.platform_os = self.architecture.platform.operating_system(value) def _set_target(self, value): """Called by the parser to set the architecture target""" - self.architecture.target = self.architecture.platform.target(value) + if self.architecture.platform: + self.architecture.target = self.architecture.platform.target(value) def _add_dependency(self, spec): """Called by the parser to add another spec as a dependency.""" @@ -1063,7 +1070,7 @@ def feq(cfield, sfield): feq(replacement.architecture, spec.architecture) and feq(replacement.dependencies, spec.dependencies) and feq(replacement.variants, spec.variants) and - feq(replacement.external, spec.external) and + feq(replacement.external, spec.external) and feq(replacement.external_module, spec.external_module)): continue # Refine this spec to the candidate. This uses @@ -1468,7 +1475,7 @@ def constrain(self, other, deps=True): if self.architecture.target != other.architecture.target: raise UnsatisfiableArchitectureSpecError(self.architecture, other.architecture) - + changed = False if self.compiler is not None and other.compiler is not None: @@ -1619,7 +1626,7 @@ def satisfies(self, other, deps=True, strict=False): # Architecture satisfaction is currently just string equality. # If not strict, None means unconstrained. if self.architecture and other.architecture: - if ((self.architecture.platform and other.architecture.platform and self.architecture.platform != other.architecture.platform) or + if ((self.architecture.platform and other.architecture.platform and self.architecture.platform != other.architecture.platform) or (self.architecture.platform_os and other.architecture.platform_os and self.architecture.platform_os != other.architecture.platform_os) or (self.architecture.target and other.architecture.target and self.architecture.target != other.architecture.target)): return False From 40d25d90629f41114c2e0b86bb659f0f4dec7bb5 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 16 Jun 2016 00:56:53 -0700 Subject: [PATCH 398/481] Add libxcb dependencies. Fix up packages to get py-basemap and stat to build. --- .../builtin/packages/binutils/package.py | 3 +- .../repos/builtin/packages/boost/package.py | 2 +- .../packages/libpthread-stubs/package.py | 39 ++++++++++++++++ .../repos/builtin/packages/libxau/package.py | 44 +++++++++++++++++++ .../repos/builtin/packages/libxcb/package.py | 7 +-- .../repos/builtin/packages/openssl/package.py | 2 +- .../packages/xorg-util-macros/package.py | 38 ++++++++++++++++ .../repos/builtin/packages/xproto/package.py | 41 +++++++++++++++++ 8 files changed, 170 insertions(+), 6 deletions(-) create mode 100644 var/spack/repos/builtin/packages/libpthread-stubs/package.py create mode 100644 var/spack/repos/builtin/packages/libxau/package.py create mode 100644 var/spack/repos/builtin/packages/xorg-util-macros/package.py create mode 100644 var/spack/repos/builtin/packages/xproto/package.py diff --git a/var/spack/repos/builtin/packages/binutils/package.py b/var/spack/repos/builtin/packages/binutils/package.py index 9e4cc98ae6..5f305abb02 100644 --- a/var/spack/repos/builtin/packages/binutils/package.py +++ b/var/spack/repos/builtin/packages/binutils/package.py @@ -30,8 +30,9 @@ class Binutils(Package): url="https://ftp.gnu.org/gnu/binutils/binutils-2.25.tar.bz2" + # 2.26 is incompatible with py-pillow build for some reason. version('2.26', '64146a0faa3b411ba774f47d41de239f') - version('2.25', 'd9f3303f802a5b6b0bb73a335ab89d66') + version('2.25', 'd9f3303f802a5b6b0bb73a335ab89d66', preferred=True) version('2.24', 'e0f71a7b2ddab0f8612336ac81d9636b') version('2.23.2', '4f8fa651e35ef262edc01d60fb45702e') version('2.20.1', '2b9dc8f2b7dbd5ec5992c6e29de0b764') diff --git a/var/spack/repos/builtin/packages/boost/package.py b/var/spack/repos/builtin/packages/boost/package.py index 2f2965eb12..1f216a146a 100644 --- a/var/spack/repos/builtin/packages/boost/package.py +++ b/var/spack/repos/builtin/packages/boost/package.py @@ -128,7 +128,7 @@ def url_for_version(self, version): dots, underscores) def determine_toolset(self, spec): - if spec.satisfies("arch=darwin-x86_64"): + if spec.satisfies("platform=darwin"): return 'darwin' toolsets = {'g++': 'gcc', diff --git a/var/spack/repos/builtin/packages/libpthread-stubs/package.py b/var/spack/repos/builtin/packages/libpthread-stubs/package.py new file mode 100644 index 0000000000..4bcca43c24 --- /dev/null +++ b/var/spack/repos/builtin/packages/libpthread-stubs/package.py @@ -0,0 +1,39 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class LibpthreadStubs(Package): + """The libpthread-stubs package provides weak aliases for pthread + functions not provided in libc or otherwise available by + default. """ + homepage = "http://xcb.freedesktop.org/" + url = "http://xcb.freedesktop.org/dist/libpthread-stubs-0.1.tar.bz2" + + version('0.3', 'e8fa31b42e13f87e8f5a7a2b731db7ee') + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/libxau/package.py b/var/spack/repos/builtin/packages/libxau/package.py new file mode 100644 index 0000000000..55816ecdbd --- /dev/null +++ b/var/spack/repos/builtin/packages/libxau/package.py @@ -0,0 +1,44 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class Libxau(Package): + """The libXau package contains a library implementing the X11 + Authorization Protocol. This is useful for restricting client + access to the display.""" + homepage = "http://xcb.freedesktop.org/" + url = "http://ftp.x.org/pub/individual/lib/libXau-1.0.8.tar.bz2" + + version('1.0.8', '685f8abbffa6d145c0f930f00703b21b') + + depends_on('xproto') + + def install(self, spec, prefix): + # FIXME: Modify the configure line to suit your build system here. + configure('--prefix=%s' % prefix) + + # FIXME: Add logic to build and install here + make() + make("install") diff --git a/var/spack/repos/builtin/packages/libxcb/package.py b/var/spack/repos/builtin/packages/libxcb/package.py index 0f39bb0f1d..746d4567e2 100644 --- a/var/spack/repos/builtin/packages/libxcb/package.py +++ b/var/spack/repos/builtin/packages/libxcb/package.py @@ -35,18 +35,19 @@ class Libxcb(Package): version('1.11', '1698dd837d7e6e94d029dbe8b3a82deb') version('1.11.1', '118623c15a96b08622603a71d8789bf3') + depends_on("python") depends_on("xcb-proto") depends_on("pkg-config") - - # depends_on('pthread') # Ubuntu: apt-get install libpthread-stubs0-dev - # depends_on('xau') # Ubuntu: apt-get install libxau-dev + depends_on("libpthread-stubs") + depends_on('libxau') def patch(self): filter_file('typedef struct xcb_auth_info_t {', 'typedef struct {', 'src/xcb.h') def install(self, spec, prefix): + env['PKG_CONFIG_PATH'] = env['PKG_CONFIG_PATH'] + ':/usr/lib64/pkgconfig' configure("--prefix=%s" % prefix) make() diff --git a/var/spack/repos/builtin/packages/openssl/package.py b/var/spack/repos/builtin/packages/openssl/package.py index 34ab0703ad..377bffe723 100644 --- a/var/spack/repos/builtin/packages/openssl/package.py +++ b/var/spack/repos/builtin/packages/openssl/package.py @@ -100,7 +100,7 @@ def install(self, spec, prefix): # in the environment, then this will override what is set in the # Makefile, leading to build errors. env.pop('APPS', None) - if spec.satisfies("arch=darwin-x86_64") or spec.satisfies("arch=ppc64"): + if spec.satisfies("target=x86_64") or spec.satisfies("target=ppc64"): # This needs to be done for all 64-bit architectures (except Linux, # where it happens automatically?) env['KERNEL_BITS'] = '64' diff --git a/var/spack/repos/builtin/packages/xorg-util-macros/package.py b/var/spack/repos/builtin/packages/xorg-util-macros/package.py new file mode 100644 index 0000000000..963d93442f --- /dev/null +++ b/var/spack/repos/builtin/packages/xorg-util-macros/package.py @@ -0,0 +1,38 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class XorgUtilMacros(Package): + """The util-macros package contains the m4 macros used by all of the Xorg packages.""" + + homepage = "http://www.example.com" + url = "http://ftp.x.org/pub/individual/util/util-macros-1.19.0.tar.bz2" + + version('1.19.0', '1cf984125e75f8204938d998a8b6c1e1') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/xproto/package.py b/var/spack/repos/builtin/packages/xproto/package.py new file mode 100644 index 0000000000..7be6defb83 --- /dev/null +++ b/var/spack/repos/builtin/packages/xproto/package.py @@ -0,0 +1,41 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class Xproto(Package): + """The Xorg protocol headers provide the header files required to + build the system, and to allow other applications to build against + the installed X Window system.""" + homepage = "http://www.x.org/" + url = "https://www.x.org/archive//individual/proto/xproto-7.0.29.tar.gz" + + version('7.0.29', '16a78dd2c5ad73011105c96235f6a0af') + + depends_on("xorg-util-macros") + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix) + make() + make("install") From 76dfaa71a7f59d8c94c415819c57cb191a0e4a4a Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 16 Jun 2016 01:47:20 -0700 Subject: [PATCH 399/481] fix up packages for new architecture. --- var/spack/repos/builtin/packages/espresso/package.py | 2 +- .../repos/builtin/packages/libpciaccess/package.py | 2 +- var/spack/repos/builtin/packages/llvm/package.py | 12 ++++++------ var/spack/repos/builtin/packages/lua/package.py | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/var/spack/repos/builtin/packages/espresso/package.py b/var/spack/repos/builtin/packages/espresso/package.py index ef6a3ccc7b..63a5560137 100644 --- a/var/spack/repos/builtin/packages/espresso/package.py +++ b/var/spack/repos/builtin/packages/espresso/package.py @@ -87,7 +87,7 @@ def install(self, spec, prefix): configure(*options) make('all') - if spec.architecture.startswith('darwin'): + if spec.satisfies('platform=darwin'): mkdirp(prefix.bin) for filename in glob("bin/*.x"): install(filename, prefix.bin) diff --git a/var/spack/repos/builtin/packages/libpciaccess/package.py b/var/spack/repos/builtin/packages/libpciaccess/package.py index 42e8711a7d..91cef95cec 100644 --- a/var/spack/repos/builtin/packages/libpciaccess/package.py +++ b/var/spack/repos/builtin/packages/libpciaccess/package.py @@ -37,7 +37,7 @@ class Libpciaccess(Package): def install(self, spec, prefix): # libpciaccess does not support OS X - if spec.satisfies('arch=darwin-x86_64'): + if spec.satisfies('platform=darwin'): # create a dummy directory mkdir(prefix.lib) return diff --git a/var/spack/repos/builtin/packages/llvm/package.py b/var/spack/repos/builtin/packages/llvm/package.py index c32f66590a..7a6ea7401c 100644 --- a/var/spack/repos/builtin/packages/llvm/package.py +++ b/var/spack/repos/builtin/packages/llvm/package.py @@ -267,16 +267,16 @@ def install(self, spec, prefix): if '+all_targets' not in spec: # all is default on cmake targets = ['CppBackend', 'NVPTX', 'AMDGPU'] - if 'x86' in spec.architecture.lower(): + if 'x86' in spec.architecture.target.lower(): targets.append('X86') - elif 'arm' in spec.architecture.lower(): + elif 'arm' in spec.architecture.target.lower(): targets.append('ARM') - elif 'aarch64' in spec.architecture.lower(): + elif 'aarch64' in spec.architecture.target.lower(): targets.append('AArch64') - elif 'sparc' in spec.architecture.lower(): + elif 'sparc' in spec.architecture.target.lower(): targets.append('sparc') - elif ('ppc' in spec.architecture.lower() or - 'power' in spec.architecture.lower()): + elif ('ppc' in spec.architecture.target.lower() or + 'power' in spec.architecture.target.lower()): targets.append('PowerPC') cmake_args.append('-DLLVM_TARGETS_TO_BUILD:Bool=' + ';'.join(targets)) diff --git a/var/spack/repos/builtin/packages/lua/package.py b/var/spack/repos/builtin/packages/lua/package.py index 170f90516a..1424cf1a17 100644 --- a/var/spack/repos/builtin/packages/lua/package.py +++ b/var/spack/repos/builtin/packages/lua/package.py @@ -57,7 +57,7 @@ class Lua(Package): placement='luarocks') def install(self, spec, prefix): - if spec.satisfies("arch=darwin-i686") or spec.satisfies("arch=darwin-x86_64"): + if spec.satisfies("platform=darwin"): target = 'macosx' else: target = 'linux' From 88b671f8b1a82ec9d9365c296d77e251a821a66c Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 16 Jun 2016 02:06:19 -0700 Subject: [PATCH 400/481] Just call old os "unknown" --- lib/spack/spack/architecture.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index df0ab5c14d..4485bbe840 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -451,7 +451,7 @@ def arch_from_dict(d): if isinstance(d, basestring): # We have an old spec using a string for the architecture arch.platform = Platform('spack_compatibility') - arch.platform_os = OperatingSystem('pre_version', '1.0') + arch.platform_os = OperatingSystem('unknown', '') arch.target = Target(d) arch.os_string = None From 8d9ffcf5650adaca13e85b158022c8b8fed72caf Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 16 Jun 2016 11:22:26 +0200 Subject: [PATCH 401/481] espresso : updated to 5.4.0, added Parallel=False --- var/spack/repos/builtin/packages/espresso/package.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/var/spack/repos/builtin/packages/espresso/package.py b/var/spack/repos/builtin/packages/espresso/package.py index ef6a3ccc7b..7359978ce1 100644 --- a/var/spack/repos/builtin/packages/espresso/package.py +++ b/var/spack/repos/builtin/packages/espresso/package.py @@ -34,6 +34,11 @@ class Espresso(Package): homepage = 'http://quantum-espresso.org' url = 'http://www.qe-forge.org/gf/download/frsrelease/204/912/espresso-5.3.0.tar.gz' + version( + '5.4.0', + '8bb78181b39bd084ae5cb7a512c1cfe7', + url='http://www.qe-forge.org/gf/download/frsrelease/211/968/espresso-5.4.0.tar.gz' + ) version('5.3.0', '6848fcfaeb118587d6be36bd10b7f2c3') variant('mpi', default=True, description='Build Quantum-ESPRESSO with mpi support') @@ -49,6 +54,10 @@ class Espresso(Package): depends_on('fftw+mpi', when='+mpi') depends_on('scalapack', when='+scalapack+mpi') # TODO : + mpi needed to avoid false dependencies installation + # Spurious problems running in parallel the Makefile + # generated by qe configure + parallel = False + def check_variants(self, spec): error = 'you cannot ask for \'+{variant}\' when \'+mpi\' is not active' if '+scalapack' in spec and '~mpi' in spec: From a1cbb73f8b18f30b30e4ba68e54dfc859d6cd0fa Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 16 Jun 2016 11:24:18 +0200 Subject: [PATCH 402/481] qa : fixed flake8 issues --- .../repos/builtin/packages/espresso/package.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/var/spack/repos/builtin/packages/espresso/package.py b/var/spack/repos/builtin/packages/espresso/package.py index 7359978ce1..30966d8de9 100644 --- a/var/spack/repos/builtin/packages/espresso/package.py +++ b/var/spack/repos/builtin/packages/espresso/package.py @@ -26,10 +26,13 @@ import os + class Espresso(Package): """ - QE is an integrated suite of Open-Source computer codes for electronic-structure calculations and materials - modeling at the nanoscale. It is based on density-functional theory, plane waves, and pseudopotentials. + QE is an integrated suite of Open-Source computer codes for + electronic-structure calculations and materials modeling at + the nanoscale. It is based on density-functional theory, plane + waves, and pseudopotentials. """ homepage = 'http://quantum-espresso.org' url = 'http://www.qe-forge.org/gf/download/frsrelease/204/912/espresso-5.3.0.tar.gz' @@ -41,10 +44,10 @@ class Espresso(Package): ) version('5.3.0', '6848fcfaeb118587d6be36bd10b7f2c3') - variant('mpi', default=True, description='Build Quantum-ESPRESSO with mpi support') + variant('mpi', default=True, description='Builds with mpi support') variant('openmp', default=False, description='Enables openMP support') variant('scalapack', default=True, description='Enables scalapack support') - variant('elpa', default=True, description='Use elpa as an eigenvalue solver') + variant('elpa', default=True, description='Uses elpa as an eigenvalue solver') depends_on('blas') depends_on('lapack') @@ -52,7 +55,8 @@ class Espresso(Package): depends_on('mpi', when='+mpi') depends_on('fftw~mpi', when='~mpi') depends_on('fftw+mpi', when='+mpi') - depends_on('scalapack', when='+scalapack+mpi') # TODO : + mpi needed to avoid false dependencies installation + # TODO : + mpi needed to avoid false dependencies installation + depends_on('scalapack', when='+scalapack+mpi') # Spurious problems running in parallel the Makefile # generated by qe configure @@ -102,4 +106,3 @@ def install(self, spec, prefix): install(filename, prefix.bin) else: make('install') - From 17b868381f95fa324f7c6327c0977b303975de76 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 16 Jun 2016 02:55:33 -0700 Subject: [PATCH 403/481] Fixes #460: Do not show variants by default in spack find. This does two things: 1. By default `spack find` no longer shows variants. You have to supply `-v` to get that 2. This improves the `colify` implementation so that it no longer pads the rightmost column. This avoids the issue where if one spec was too long in the output, *all* specs would have space padding added to that width, and it would look like the output of `spack find` was double spaced. This no longer happens -- the one bad line wraps around and the other lines are now single-spaced when you use `-v` with boost. --- lib/spack/llnl/util/tty/colify.py | 9 +++++++-- lib/spack/spack/cmd/find.py | 19 +++++++++++++------ lib/spack/spack/cmd/uninstall.py | 13 ++++++++++--- 3 files changed, 30 insertions(+), 11 deletions(-) diff --git a/lib/spack/llnl/util/tty/colify.py b/lib/spack/llnl/util/tty/colify.py index 429ba45882..81a83691d7 100644 --- a/lib/spack/llnl/util/tty/colify.py +++ b/lib/spack/llnl/util/tty/colify.py @@ -198,8 +198,13 @@ def colify(elts, **options): for col in xrange(cols): elt = col * rows + row width = config.widths[col] + cextra(elts[elt]) - fmt = '%%-%ds' % width - output.write(fmt % elts[elt]) + if col < cols - 1: + fmt = '%%-%ds' % width + output.write(fmt % elts[elt]) + else: + # Don't pad the rightmost column (sapces can wrap on + # small teriminals if one line is overlong) + output.write(elts[elt]) output.write("\n") row += 1 diff --git a/lib/spack/spack/cmd/find.py b/lib/spack/spack/cmd/find.py index 9649bc7435..3ec671f93f 100644 --- a/lib/spack/spack/cmd/find.py +++ b/lib/spack/spack/cmd/find.py @@ -86,6 +86,11 @@ def setup_parser(subparser): action='store_true', dest='missing', help='Show missing dependencies as well as installed specs.') + subparser.add_argument( + '-v', '--variants', + action='store_true', + dest='variants', + help='Show variants in output (can be long)') subparser.add_argument('-M', '--only-missing', action='store_true', dest='only_missing', @@ -107,6 +112,8 @@ def display_specs(specs, **kwargs): mode = kwargs.get('mode', 'short') hashes = kwargs.get('long', False) namespace = kwargs.get('namespace', False) + flags = kwargs.get('show_flags', False) + variants = kwargs.get('variants', False) hlen = 7 if kwargs.get('very_long', False): @@ -114,10 +121,9 @@ def display_specs(specs, **kwargs): hlen = None nfmt = '.' if namespace else '_' - format_string = '$%s$@$+' % nfmt - flags = kwargs.get('show_flags', False) - if flags: - format_string = '$%s$@$%%+$+' % nfmt + ffmt = '$%+' if flags else '' + vfmt = '$+' if variants else '' + format_string = '$%s$@%s%s' % (nfmt, ffmt, vfmt) # Make a dict with specs keyed by architecture and compiler. index = index_by(specs, ('architecture', 'compiler')) @@ -163,7 +169,7 @@ def fmt(s): string = "" if hashes: string += gray_hash(s, hlen) + ' ' - string += s.format('$-%s$@$+' % nfmt, color=True) + string += s.format('$-%s$@%s' % (nfmt, vfmt), color=True) return string @@ -238,4 +244,5 @@ def find(parser, args): long=args.long, very_long=args.very_long, show_flags=args.show_flags, - namespace=args.namespace) + namespace=args.namespace, + variants=args.variants) diff --git a/lib/spack/spack/cmd/uninstall.py b/lib/spack/spack/cmd/uninstall.py index 9fdf3045b2..a6f08d09ed 100644 --- a/lib/spack/spack/cmd/uninstall.py +++ b/lib/spack/spack/cmd/uninstall.py @@ -39,6 +39,13 @@ b) use spack uninstall -a to uninstall ALL matching specs. """ +# Arguments for display_specs when we find ambiguity +display_args = { + 'long': True, + 'show_flags': True, + 'variants':True +} + def ask_for_confirmation(message): while True: @@ -92,7 +99,7 @@ def concretize_specs(specs, allow_multiple_matches=False, force=False): if not allow_multiple_matches and len(matching) > 1: tty.error("%s matches multiple packages:" % spec) print() - display_specs(matching, long=True, show_flags=True) + display_specs(matching, **display_args) print() has_errors = True @@ -172,7 +179,7 @@ def uninstall(parser, args): tty.error("Will not uninstall %s" % spec.format("$_$@$%@$#", color=True)) print('') print("The following packages depend on it:") - display_specs(lst, long=True) + display_specs(lst, **display_args) print('') has_error = True elif args.dependents: @@ -186,7 +193,7 @@ def uninstall(parser, args): if not args.yes_to_all: tty.msg("The following packages will be uninstalled : ") print('') - display_specs(uninstall_list, long=True, show_flags=True) + display_specs(uninstall_list, **display_args) print('') ask_for_confirmation('Do you want to proceed ? ') From 6b037706023c6cc138709d4cbca91d81d32cc4c4 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 16 Jun 2016 09:44:37 -0700 Subject: [PATCH 404/481] Added str method to mac_osx so that whenever we try to access darwins different operating systems we can grab the name correctly. This avoids having to worry whether we have a version attached to it or not. --- lib/spack/spack/operating_systems/mac_osx.py | 9 ++++----- lib/spack/spack/platforms/darwin.py | 8 ++++---- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/lib/spack/spack/operating_systems/mac_osx.py b/lib/spack/spack/operating_systems/mac_osx.py index a9de03d2cc..3cf9b4e12e 100644 --- a/lib/spack/spack/operating_systems/mac_osx.py +++ b/lib/spack/spack/operating_systems/mac_osx.py @@ -21,9 +21,8 @@ def __init__(self): "10.11": "elcapitan"} mac_ver = py_platform.mac_ver()[0][:-2] - try: - name = mac_releases[mac_ver] - except KeyError: - name = "mac_os" - + name = mac_releases.get(mac_ver, "mac_osx") super(MacOsx, self).__init__(name, mac_ver) + + def __str__(self): + return self.name diff --git a/lib/spack/spack/platforms/darwin.py b/lib/spack/spack/platforms/darwin.py index 7901f82d06..8c8af83d55 100644 --- a/lib/spack/spack/platforms/darwin.py +++ b/lib/spack/spack/platforms/darwin.py @@ -13,11 +13,11 @@ def __init__(self): self.add_target(self.default, Target(self.default)) mac_os = MacOsx() - self.default_os = mac_os.name - self.front_os = mac_os.name - self.back_os = mac_os.name + self.default_os = str(mac_os) + self.front_os = str(mac_os) + self.back_os = str(mac_os) - self.add_operating_system(mac_os.name, mac_os) + self.add_operating_system(str(mac_os), mac_os) @classmethod def detect(self): From a5aa159d9d0fc97cc9ae882e7325f19438ce597a Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 16 Jun 2016 19:23:16 +0200 Subject: [PATCH 405/481] flake8: ignore F405 --- .flake8 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.flake8 b/.flake8 index 12ae2efa80..286522bc48 100644 --- a/.flake8 +++ b/.flake8 @@ -19,5 +19,5 @@ # - F999: name name be undefined or undefined from star imports. # [flake8] -ignore = E221,E241,E731,F403,F821,F999 +ignore = E221,E241,E731,F403,F821,F999,F405 max-line-length = 79 From c7dfa1699a2898250e51bdcccc4eb568161a7c1c Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 16 Jun 2016 10:43:12 -0700 Subject: [PATCH 406/481] rename Mac OS X to macos --- .../{mac_osx.py => mac_os.py} | 21 ++++++++++--------- lib/spack/spack/platforms/darwin.py | 6 +++--- lib/spack/spack/test/operating_system.py | 13 +++++------- 3 files changed, 19 insertions(+), 21 deletions(-) rename lib/spack/spack/operating_systems/{mac_osx.py => mac_os.py} (53%) diff --git a/lib/spack/spack/operating_systems/mac_osx.py b/lib/spack/spack/operating_systems/mac_os.py similarity index 53% rename from lib/spack/spack/operating_systems/mac_osx.py rename to lib/spack/spack/operating_systems/mac_os.py index 3cf9b4e12e..f35b3ca577 100644 --- a/lib/spack/spack/operating_systems/mac_osx.py +++ b/lib/spack/spack/operating_systems/mac_os.py @@ -1,28 +1,29 @@ import platform as py_platform from spack.architecture import OperatingSystem -class MacOsx(OperatingSystem): - """ This class represents the MAC_OSX operating system. This will be auto - detected using the python platform.mac_ver. The MAC_OSX platform - will be represented using the major version operating system name, i.e - el capitan, yosemite...etc. +class MacOs(OperatingSystem): + """This class represents the macOS operating system. This will be + auto detected using the python platform.mac_ver. The macOS + platform will be represented using the major version operating + system name, i.e el capitan, yosemite...etc. """ - + def __init__(self): """ Autodetects the mac version from a dictionary. Goes back as far as 10.6 snowleopard. If the user has an older mac then the version will just be a generic mac_os. """ - mac_releases = {'10.6': "snowleopard", + mac_releases = {'10.6': "snowleopard", "10.7": "lion", "10.8": "mountainlion", "10.9": "mavericks", "10.10": "yosemite", - "10.11": "elcapitan"} + "10.11": "elcapitan", + "10.12": "sierra"} mac_ver = py_platform.mac_ver()[0][:-2] - name = mac_releases.get(mac_ver, "mac_osx") - super(MacOsx, self).__init__(name, mac_ver) + name = mac_releases.get(mac_ver, "macos") + super(MacOs, self).__init__(name, mac_ver) def __str__(self): return self.name diff --git a/lib/spack/spack/platforms/darwin.py b/lib/spack/spack/platforms/darwin.py index 8c8af83d55..d47dd640f9 100644 --- a/lib/spack/spack/platforms/darwin.py +++ b/lib/spack/spack/platforms/darwin.py @@ -1,6 +1,6 @@ import subprocess from spack.architecture import Platform, Target -from spack.operating_systems.mac_osx import MacOsx +from spack.operating_systems.mac_os import MacOs class Darwin(Platform): priority = 89 @@ -11,8 +11,8 @@ class Darwin(Platform): def __init__(self): super(Darwin, self).__init__('darwin') self.add_target(self.default, Target(self.default)) - mac_os = MacOsx() - + mac_os = MacOs() + self.default_os = str(mac_os) self.front_os = str(mac_os) self.back_os = str(mac_os) diff --git a/lib/spack/spack/test/operating_system.py b/lib/spack/spack/test/operating_system.py index 205408db3f..ed5f6ff8ad 100644 --- a/lib/spack/spack/test/operating_system.py +++ b/lib/spack/spack/test/operating_system.py @@ -10,7 +10,7 @@ from spack.platforms.linux import Linux from spack.platforms.darwin import Darwin from spack.operating_system.linux_distro import LinuxDistro -from spack.operating_system.mac_osx import MacOSX +from spack.operating_system.mac_os import MacOs from spack.operating_system.cnl import ComputeNodeLinux class TestOperatingSystem(unittest.TestCase): @@ -27,12 +27,12 @@ def setUp(self): def test_cray_front_end_operating_system(self): self.assertIsInstance(self.cray_operating_sys, LinuxDistro) - + def test_cray_front_end_compiler_strategy(self): self.assertEquals(self.cray_operating_sys.compiler_strategy, "PATH") def test_cray_back_end_operating_system(self): - self.assertIsInstance(self.cray_back_os,ComputeNodeLinux) + self.assertIsInstance(self.cray_back_os,ComputeNodeLinux) def test_cray_back_end_compiler_strategy(self): self.assertEquals(self.cray_back_os.compiler_strategy, "MODULES") @@ -43,16 +43,13 @@ def test_linux_operating_system(self): def test_linux_compiler_strategy(self): self.assertEquals(self.linux_operating_sys.compiler_strategy, "PATH") - + def test_cray_front_end_compiler_list(self): """ Operating systems will now be in charge of finding compilers. So, depending on which operating system you want to build for - or which operating system you are on, then you could detect + or which operating system you are on, then you could detect compilers in a certain way. Cray linux environment on the front end is just a regular linux distro whereas the Cray linux compute node is a stripped down version which modules are important """ self.assertEquals(True, False) - - - From 8867adf9428dedd42dfc92efece2a3f7498400e0 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 16 Jun 2016 10:43:36 -0700 Subject: [PATCH 407/481] simplify instance check. --- lib/spack/spack/architecture.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 4485bbe840..cbac7b41d6 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -424,8 +424,7 @@ def _operating_system_from_dict(os_name, platform=None): """ if not platform: platform = sys_type() - if isinstance(os_name, spack.util.spack_yaml.syaml_dict) or \ - isinstance(os_name, dict): + if isinstance(os_name, dict): name = os_name['name'] version = os_name['version'] return platform.operating_system(name+version) From f60ae61652397c614b2e6100eb17dcdc5a99164b Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 16 Jun 2016 10:44:15 -0700 Subject: [PATCH 408/481] remove dead code, cleanup. --- lib/spack/spack/test/architecture.py | 19 +++------- lib/spack/spack/test/spec_semantics.py | 49 +++++++++++--------------- 2 files changed, 26 insertions(+), 42 deletions(-) diff --git a/lib/spack/spack/test/architecture.py b/lib/spack/spack/test/architecture.py index 3d08269dee..a6847c5744 100644 --- a/lib/spack/spack/test/architecture.py +++ b/lib/spack/spack/test/architecture.py @@ -23,7 +23,7 @@ def setUp(self): def tearDown(self): super(ArchitectureTest, self).tearDown() - + def test_dict_functions_for_architecture(self): arch = Arch() arch.platform = spack.architecture.sys_type() @@ -46,14 +46,6 @@ def test_dict_functions_for_architecture(self): self.assertTrue( isinstance(new_arch.target, Target) ) -# def test_platform_class_and_compiler_strategies(self): -# a = CrayXc() -# t = a.operating_system('default_os') -# self.assertEquals(t.compiler_strategy, 'MODULES') -# b = Linux() -# s = b.operating_system('default_os') -# self.assertEquals(s.compiler_strategy, 'PATH') - def test_sys_type(self): output_platform_class = sys_type() my_arch_class = None @@ -101,12 +93,12 @@ def test_user_defaults(self): def test_user_input_combination(self): os_list = self.platform.operating_sys.keys() - target_list = self.platform.targets.keys() + target_list = self.platform.targets.keys() additional = ["fe", "be", "frontend", "backend"] os_list.extend(additional) - target_list.extend(additional) - + target_list.extend(additional) + combinations = itertools.product(os_list, target_list) results = [] for arch in combinations: @@ -117,5 +109,4 @@ def test_user_input_combination(self): results.append(spec.architecture.target == self.platform.target(t)) res = all(results) - self.assertTrue(res) - + self.assertTrue(res) diff --git a/lib/spack/spack/test/spec_semantics.py b/lib/spack/spack/test/spec_semantics.py index 0b2111bfe8..9876bfd5a8 100644 --- a/lib/spack/spack/test/spec_semantics.py +++ b/lib/spack/spack/test/spec_semantics.py @@ -108,7 +108,8 @@ def test_satisfies_namespace(self): def test_satisfies_namespaced_dep(self): - """Ensure spec from same or unspecified namespace satisfies namespace constraint.""" + """Ensure spec from same or unspecified namespace satisfies namespace + constraint.""" self.check_satisfies('mpileaks ^builtin.mock.mpich', '^mpich') self.check_satisfies('mpileaks ^builtin.mock.mpich', '^mpi') @@ -141,29 +142,17 @@ def test_satisfies_compiler_version(self): def test_satisfies_architecture(self): platform = spack.architecture.sys_type() - self.check_satisfies('foo platform=test target=frontend os=frontend', 'platform=test target=frontend os=frontend') - self.check_satisfies('foo platform=test target=backend os=backend', 'platform=test target=backend', 'platform=test os=backend') - self.check_satisfies('foo platform=test target=default_target os=default_os','platform=test target=default_target os=default_os') + self.check_satisfies( + 'foo platform=test target=frontend os=frontend', + 'platform=test target=frontend os=frontend') + self.check_satisfies( + 'foo platform=test target=backend os=backend', + 'platform=test target=backend', 'platform=test os=backend') + self.check_satisfies( + 'foo platform=test target=default_target os=default_os', + 'platform=test target=default_target os=default_os') -#ifdef NEW - #def test_satisfies_architecture(self): - # self.check_satisfies('foo arch=chaos_5_x86_64_ib', ' arch=chaos_5_x86_64_ib') - # self.check_satisfies('foo arch=bgqos_0', ' arch=bgqos_0') - - # self.check_unsatisfiable('foo arch=bgqos_0', ' arch=chaos_5_x86_64_ib') - # self.check_unsatisfiable('foo arch=chaos_5_x86_64_ib', ' arch=bgqos_0') -#els#e /* not NEW */ - #def test_satisfies_target(self): - # platform = spack.architecture.sys_type() - # targets = platform.targets.values() - # for target in targets: - # self.check_satisfies('foo='+target.name, '='+target.name) -#end#if /* not NEW */ - - # for i in range(1,len(targets)): - # self.check_unsatisfiable('foo='+targets[i-1].name, '='+targets[i].name) - def test_satisfies_dependencies(self): self.check_satisfies('mpileaks^mpich', '^mpich') self.check_satisfies('mpileaks^zmpi', '^zmpi') @@ -176,10 +165,14 @@ def test_satisfies_dependency_versions(self): self.check_satisfies('mpileaks^mpich@2.0', '^mpich@1:3') self.check_unsatisfiable('mpileaks^mpich@1.2', '^mpich@2.0') - self.check_satisfies('mpileaks^mpich@2.0^callpath@1.5', '^mpich@1:3^callpath@1.4:1.6') - self.check_unsatisfiable('mpileaks^mpich@4.0^callpath@1.5', '^mpich@1:3^callpath@1.4:1.6') - self.check_unsatisfiable('mpileaks^mpich@2.0^callpath@1.7', '^mpich@1:3^callpath@1.4:1.6') - self.check_unsatisfiable('mpileaks^mpich@4.0^callpath@1.7', '^mpich@1:3^callpath@1.4:1.6') + self.check_satisfies( + 'mpileaks^mpich@2.0^callpath@1.5', '^mpich@1:3^callpath@1.4:1.6') + self.check_unsatisfiable( + 'mpileaks^mpich@4.0^callpath@1.5', '^mpich@1:3^callpath@1.4:1.6') + self.check_unsatisfiable( + 'mpileaks^mpich@2.0^callpath@1.7', '^mpich@1:3^callpath@1.4:1.6') + self.check_unsatisfiable( + 'mpileaks^mpich@4.0^callpath@1.7', '^mpich@1:3^callpath@1.4:1.6') def test_satisfies_virtual_dependencies(self): @@ -390,7 +383,7 @@ def test_invalid_constraint(self): self.check_invalid_constraint('libelf debug=2', 'libelf debug=1') self.check_invalid_constraint('libelf cppflags="-O3"', 'libelf cppflags="-O2"') - self.check_invalid_constraint('libelf platform=test target=be os=be', + self.check_invalid_constraint('libelf platform=test target=be os=be', 'libelf target=fe os=fe') def test_constrain_changed(self): @@ -402,7 +395,7 @@ def test_constrain_changed(self): self.check_constrain_changed('libelf', '~debug') self.check_constrain_changed('libelf', 'debug=2') self.check_constrain_changed('libelf', 'cppflags="-O3"') - + platform = spack.architecture.sys_type() self.check_constrain_changed('libelf', 'target='+platform.target('default_target').name) self.check_constrain_changed('libelf', 'os='+platform.operating_system('default_os').name) From 005827ab234e4fdb8d8e298baa99573bb570e4c8 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 16 Jun 2016 12:12:29 -0700 Subject: [PATCH 409/481] Replace 2.6-incompatible skipIf with a conditional. - skipIf doesn't work with Python 2.6. --- lib/spack/spack/test/concretize.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index 67b20b4084..ab201f406a 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -253,10 +253,14 @@ def test_external_package(self): self.assertFalse('externalprereq' in spec) self.assertTrue(spec['externaltool'].compiler.satisfies('gcc')) - @unittest.skipIf(spack.architecture.sys_type().name == 'darwin' or - spack.architecture.sys_type().name == 'linux', - "No tcl modules on darwin/linux machines") + def test_external_package_module(self): + # No tcl modules on darwin/linux machines + # TODO: improved way to check for this. + if (spack.architecture.sys_type().name == 'darwin' or + spack.architecture.sys_type().name == 'linux'): + return + spec = Spec('externalmodule') spec.concretize() self.assertEqual(spec['externalmodule'].external_module, 'external-module') @@ -272,7 +276,7 @@ def test_nobuild_package(self): got_error = True self.assertTrue(got_error) - + def test_external_and_virtual(self): spec = Spec('externaltest') spec.concretize() From df84fe0b58522597c9c16e59ceeabb9c3163c1ea Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 16 Jun 2016 18:10:38 -0700 Subject: [PATCH 410/481] Added init_config parameter so we don't initialize the config when user runs spack compiler add. Should ONLY add what user specifies. Find compilers takes in paths parameter. --- lib/spack/spack/cmd/compiler.py | 6 ++-- lib/spack/spack/compilers/__init__.py | 44 +++++++++++++++------------ 2 files changed, 28 insertions(+), 22 deletions(-) diff --git a/lib/spack/spack/cmd/compiler.py b/lib/spack/spack/cmd/compiler.py index 030aa77c30..94d83512e1 100644 --- a/lib/spack/spack/cmd/compiler.py +++ b/lib/spack/spack/cmd/compiler.py @@ -77,10 +77,12 @@ def compiler_add(args): paths = get_path('PATH') compilers = [c for c in spack.compilers.find_compilers(*args.add_paths) - if c.spec not in spack.compilers.all_compilers(scope=args.scope)] + if c.spec not in spack.compilers.all_compilers(scope=args.scope, + init_config=False)] if compilers: - spack.compilers.add_compilers_to_config(compilers, scope=args.scope) + spack.compilers.add_compilers_to_config(compilers, scope=args.scope, + init_config=False) n = len(compilers) s = 's' if n > 1 else '' filename = spack.config.get_config_filename(args.scope, 'compilers') diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 4b546c2cbf..6de2d87b32 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -79,12 +79,10 @@ def _to_dict(compiler): if compiler.alias: d['alias'] = compiler.alias - return { - 'compiler': d - } + return {'compiler': d} -def get_compiler_config(scope=None): +def get_compiler_config(scope=None, init_config=True): """Return the compiler configuration for the specified architecture. """ def init_compiler_config(): @@ -100,7 +98,8 @@ def init_compiler_config(): # configured. Avoid updating automatically if there ARE site # compilers configured but no user ones. # if (isinstance(arch, basestring) or arch == my_arch) and arch not in config: - if not config: + empty_config = [] + if not config and init_config: if scope is None: # We know no compilers were configured in any scope. init_compiler_config() @@ -112,18 +111,20 @@ def init_compiler_config(): if not site_config: init_compiler_config() config = spack.config.get_config('compilers', scope=scope) - - return config + return config + else: + return empty_config -def add_compilers_to_config(compilers, scope=None): +def add_compilers_to_config(compilers, scope=None, init_config=True): """Add compilers to the config for the specified architecture. Arguments: - compilers: a list of Compiler objects. - scope: configuration scope to modify. """ - compiler_config = get_compiler_config(scope) + compiler_config = get_compiler_config(scope, init_config) + print compiler_config for compiler in compilers: compiler_config.append(_to_dict(compiler)) @@ -148,25 +149,23 @@ def remove_compiler_from_config(compiler_spec, scope=None): spack.config.update_config('compilers', compiler_config, scope) _cache_config_file = {} - -def all_compilers_config(scope=None): +def all_compilers_config(scope=None, init_config=True): """Return a set of specs for all the compiler versions currently available to build with. These are instances of CompilerSpec. """ # Get compilers for this architecture. global _cache_config_file #Create a cache of the config file so we don't load all the time. - if not _cache_config_file: - _cache_config_file = get_compiler_config(scope) + _cache_config_file = get_compiler_config(scope, init_config) return _cache_config_file - else: return _cache_config_file -def all_compilers(scope=None): + +def all_compilers(scope=None, init_config=True): # Return compiler specs from the merged config. return [spack.spec.CompilerSpec(s['compiler']['spec']) - for s in all_compilers_config(scope)] + for s in all_compilers_config(scope, init_config)] def default_compiler(): @@ -181,7 +180,13 @@ def default_compiler(): return sorted(versions)[-1] -def find_compilers(): +def find_compilers(*paths): + """ Call find compilers help and return the list it finds using + the operating system method (PATHS, MODULES) """ + return _find_compilers(*paths) + + +def _find_compilers(*paths): """Return a list of compilers found in the suppied paths. This invokes the find_compilers() method for each operating system associated with the host platform, and appends @@ -190,9 +195,8 @@ def find_compilers(): # Find compilers for each operating system class oss = all_os_classes() compiler_lists = [] - for os in oss: - compiler_lists.extend(os.find_compilers()) - + for o in oss: + compiler_lists.extend(o.find_compilers(*paths)) return compiler_lists def supported_compilers(): From 64acbbfcf03393eab99301f1e7843618c5547cf0 Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Sat, 28 May 2016 20:19:45 -0500 Subject: [PATCH 411/481] Clean up R package files - remove unused imports - apply pep8 - reformat text strings --- var/spack/repos/builtin/packages/R/package.py | 44 ++++++++----------- 1 file changed, 19 insertions(+), 25 deletions(-) diff --git a/var/spack/repos/builtin/packages/R/package.py b/var/spack/repos/builtin/packages/R/package.py index 0177fe1a2b..727cd38461 100644 --- a/var/spack/repos/builtin/packages/R/package.py +++ b/var/spack/repos/builtin/packages/R/package.py @@ -22,28 +22,22 @@ # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import functools -import glob -import inspect import os -import re -from contextlib import closing -import spack -from llnl.util.lang import match_predicate from spack import * from spack.util.environment import * class R(Package): - """ - R is 'GNU S', a freely available language and environment for statistical computing and graphics which provides a - wide variety of statistical and graphical techniques: linear and nonlinear modelling, statistical tests, time series - analysis, classification, clustering, etc. Please consult the R project homepage for further information. - """ + """R is 'GNU S', a freely available language and environment for + statistical computing and graphics which provides a wide variety of + statistical and graphical techniques: linear and nonlinear modelling, + statistical tests, time series analysis, classification, clustering, etc. + Please consult the R project homepage for further information.""" + homepage = "https://www.r-project.org" url = "http://cran.cnr.berkeley.edu/src/base/R-3/R-3.1.2.tar.gz" - + extendable = True version('3.2.3', '1ba3dac113efab69e706902810cc2970') @@ -53,7 +47,8 @@ class R(Package): version('3.1.3', '53a85b884925aa6b5811dfc361d73fc4') version('3.1.2', '3af29ec06704cbd08d4ba8d69250ae74') - variant('external-lapack', default=False, description='Links to externally installed BLAS/LAPACK') + variant('external-lapack', default=False, + description='Links to externally installed BLAS/LAPACK') # Virtual dependencies depends_on('blas', when='+external-lapack') @@ -106,25 +101,24 @@ def setup_dependent_environment(self, spack_env, run_env, extension_spec): r_libs_path = ':'.join(r_libs_path) spack_env.set('R_LIBS', r_libs_path) - # For run time environment set only the path for extension_spec and prepend it to R_LIBS + # For run time environment set only the path for extension_spec and + # prepend it to R_LIBS if extension_spec.package.extends(self.spec): - run_env.prepend_path('R_LIBS', os.path.join(extension_spec.prefix, self.r_lib_dir)) - + run_env.prepend_path('R_LIBS', os.path.join( + extension_spec.prefix, self.r_lib_dir)) def setup_dependent_package(self, module, ext_spec): - """ - Called before R modules' install() methods. - - In most cases, extensions will only need to have one line:: - - R('CMD', 'INSTALL', '--library=%s' % self.module.r_lib_dir, '%s' % self.stage.archive_file) - """ + """Called before R modules' install() methods. In most cases, + extensions will only need to have one line: + R('CMD', 'INSTALL', '--library=%s' % self.module.r_lib_dir, '%s' % + self.stage.archive_file)""" # R extension builds can have a global R executable function module.R = Executable(join_path(self.spec.prefix.bin, 'R')) # Add variable for library directry module.r_lib_dir = os.path.join(ext_spec.prefix, self.r_lib_dir) - # Make the site packages directory for extensions, if it does not exist already. + # Make the site packages directory for extensions, if it does not exist + # already. if ext_spec.package.is_extension: mkdirp(module.r_lib_dir) From 6649f9edc226742b4184b73106df5496ad66406a Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Sat, 28 May 2016 22:25:32 -0500 Subject: [PATCH 412/481] Add version 3.3.0 of R This commit adds the 3.3.0 version of R and adds dependencies on curl and pcre. --- var/spack/repos/builtin/packages/R/package.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/var/spack/repos/builtin/packages/R/package.py b/var/spack/repos/builtin/packages/R/package.py index 727cd38461..f4728bfd64 100644 --- a/var/spack/repos/builtin/packages/R/package.py +++ b/var/spack/repos/builtin/packages/R/package.py @@ -40,6 +40,7 @@ class R(Package): extendable = True + version('3.3.0', '5a7506c8813432d1621c9725e86baf7a') version('3.2.3', '1ba3dac113efab69e706902810cc2970') version('3.2.2', '57cef5c2e210a5454da1979562a10e5b') version('3.2.1', 'c2aac8b40f84e08e7f8c9068de9239a3') @@ -67,6 +68,8 @@ class R(Package): depends_on('freetype') depends_on('tcl') depends_on('tk') + depends_on('curl') + depends_on('pcre') def install(self, spec, prefix): rlibdir = join_path(prefix, 'rlib') From cb3505769b57e9f7c06f42faf8b88dfb40db7ec9 Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Sat, 28 May 2016 22:31:47 -0500 Subject: [PATCH 413/481] Clean up R extensions Run the existing R extension packages through autopep8 to clean up. --- var/spack/repos/builtin/packages/r-BiocGenerics/package.py | 6 ++++-- var/spack/repos/builtin/packages/r-abind/package.py | 6 ++++-- var/spack/repos/builtin/packages/r-filehash/package.py | 6 ++++-- var/spack/repos/builtin/packages/r-magic/package.py | 6 ++++-- 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/var/spack/repos/builtin/packages/r-BiocGenerics/package.py b/var/spack/repos/builtin/packages/r-BiocGenerics/package.py index 0d3b6a3e57..1d994cc2ef 100644 --- a/var/spack/repos/builtin/packages/r-BiocGenerics/package.py +++ b/var/spack/repos/builtin/packages/r-BiocGenerics/package.py @@ -24,15 +24,17 @@ ############################################################################## from spack import * + class RBiocgenerics(Package): """S4 generic functions needed by many Bioconductor packages.""" homepage = 'https://www.bioconductor.org/packages/release/bioc/html/BiocGenerics.html' - url = "https://www.bioconductor.org/packages/release/bioc/src/contrib/BiocGenerics_0.16.1.tar.gz" + url = "https://www.bioconductor.org/packages/release/bioc/src/contrib/BiocGenerics_0.16.1.tar.gz" version('0.16.1', 'c2148ffd86fc6f1f819c7f68eb2c744f', expand=False) extends('R') def install(self, spec, prefix): - R('CMD', 'INSTALL', '--library=%s' % self.module.r_lib_dir, '%s' % self.stage.archive_file) + R('CMD', 'INSTALL', '--library=%s' % + self.module.r_lib_dir, '%s' % self.stage.archive_file) diff --git a/var/spack/repos/builtin/packages/r-abind/package.py b/var/spack/repos/builtin/packages/r-abind/package.py index 34a1eee79f..f9ac684aa4 100644 --- a/var/spack/repos/builtin/packages/r-abind/package.py +++ b/var/spack/repos/builtin/packages/r-abind/package.py @@ -24,6 +24,7 @@ ############################################################################## from spack import * + class RAbind(Package): """Combine multidimensional arrays into a single array. This is a generalization of 'cbind' and 'rbind'. Works with vectors, matrices, and @@ -31,7 +32,7 @@ class RAbind(Package): 'afill' for manipulating, extracting and replacing data in arrays.""" homepage = "https://cran.r-project.org/" - url = "https://cran.r-project.org/src/contrib/abind_1.4-3.tar.gz" + url = "https://cran.r-project.org/src/contrib/abind_1.4-3.tar.gz" version('1.4-3', '10fcf80c677b991bf263d38be35a1fc5', expand=False) @@ -39,4 +40,5 @@ class RAbind(Package): def install(self, spec, prefix): - R('CMD', 'INSTALL', '--library=%s' % self.module.r_lib_dir, '%s' % self.stage.archive_file) + R('CMD', 'INSTALL', '--library=%s' % + self.module.r_lib_dir, '%s' % self.stage.archive_file) diff --git a/var/spack/repos/builtin/packages/r-filehash/package.py b/var/spack/repos/builtin/packages/r-filehash/package.py index fffae68019..e59ba18fa3 100644 --- a/var/spack/repos/builtin/packages/r-filehash/package.py +++ b/var/spack/repos/builtin/packages/r-filehash/package.py @@ -24,6 +24,7 @@ ############################################################################## from spack import * + class RFilehash(Package): """Implements a simple key-value style database where character string keys are associated with data values that are stored on the disk. A simple @@ -36,11 +37,12 @@ class RFilehash(Package): parties for use in the 'filehash' framework.""" homepage = 'https://cran.r-project.org/' - url = "https://cran.r-project.org/src/contrib/filehash_2.3.tar.gz" + url = "https://cran.r-project.org/src/contrib/filehash_2.3.tar.gz" version('2.3', '01fffafe09b148ccadc9814c103bdc2f', expand=False) extends('R') def install(self, spec, prefix): - R('CMD', 'INSTALL', '--library=%s' % self.module.r_lib_dir, '%s' % self.stage.archive_file) + R('CMD', 'INSTALL', '--library=%s' % + self.module.r_lib_dir, '%s' % self.stage.archive_file) diff --git a/var/spack/repos/builtin/packages/r-magic/package.py b/var/spack/repos/builtin/packages/r-magic/package.py index f86917ec0c..ec4b9b9e41 100644 --- a/var/spack/repos/builtin/packages/r-magic/package.py +++ b/var/spack/repos/builtin/packages/r-magic/package.py @@ -24,6 +24,7 @@ ############################################################################## from spack import * + class RMagic(Package): """A collection of efficient, vectorized algorithms for the creation and investigation of magic squares and hypercubes, including a variety of @@ -31,7 +32,7 @@ class RMagic(Package): arrays.""" homepage = "https://cran.r-project.org/" - url = "https://cran.r-project.org/src/contrib/magic_1.5-6.tar.gz" + url = "https://cran.r-project.org/src/contrib/magic_1.5-6.tar.gz" version('1.5-6', 'a68e5ced253b2196af842e1fc84fd029', expand=False) @@ -40,4 +41,5 @@ class RMagic(Package): depends_on('r-abind') def install(self, spec, prefix): - R('CMD', 'INSTALL', '--library=%s' % self.module.r_lib_dir, '%s' % self.stage.archive_file) + R('CMD', 'INSTALL', '--library=%s' % + self.module.r_lib_dir, '%s' % self.stage.archive_file) From fe256870b47d8b0381c918e54a45c46229506bad Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Sun, 29 May 2016 15:26:34 -0500 Subject: [PATCH 414/481] Add variant for UTF support to pcre This will build PCRE with UTF support by default. That seems to be reasonable for most environments. --- var/spack/repos/builtin/packages/pcre/package.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/pcre/package.py b/var/spack/repos/builtin/packages/pcre/package.py index 8e0f83110e..53b23a203d 100644 --- a/var/spack/repos/builtin/packages/pcre/package.py +++ b/var/spack/repos/builtin/packages/pcre/package.py @@ -29,6 +29,7 @@ class Pcre(Package): """The PCRE package contains Perl Compatible Regular Expression libraries. These are useful for implementing regular expression pattern matching using the same syntax and semantics as Perl 5.""" + homepage = "http://www.pcre.org""" url = "ftp://ftp.csx.cam.ac.uk/pub/software/programming/pcre/pcre-8.36.tar.bz2" @@ -37,7 +38,14 @@ class Pcre(Package): patch("intel.patch") + variant('utf', default=True, + description='Enable support for UTF-8/16/32, incompatible with EBCDIC.') + def install(self, spec, prefix): - configure("--prefix=%s" % prefix) + configure_args = ['--prefix=%s' % prefix] + if '+utf' in spec: + configure_args.append('--enable-utf') + + configure(*configure_args) make() make("install") From 8704a4ed35b5868a62cde613d85c3eeae27e5a5e Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Sun, 29 May 2016 16:06:03 -0500 Subject: [PATCH 415/481] Use configure_args rather than options Replaced 'options' with 'configure_args'. That seems like a better name and is consistent with the packaging guide. --- var/spack/repos/builtin/packages/R/package.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/var/spack/repos/builtin/packages/R/package.py b/var/spack/repos/builtin/packages/R/package.py index f4728bfd64..af520082ec 100644 --- a/var/spack/repos/builtin/packages/R/package.py +++ b/var/spack/repos/builtin/packages/R/package.py @@ -73,15 +73,15 @@ class R(Package): def install(self, spec, prefix): rlibdir = join_path(prefix, 'rlib') - options = ['--prefix=%s' % prefix, - '--libdir=%s' % rlibdir, - '--enable-R-shlib', - '--enable-BLAS-shlib', - '--enable-R-framework=no'] + configure_args = ['--prefix=%s' % prefix, + '--libdir=%s' % rlibdir, + '--enable-R-shlib', + '--enable-BLAS-shlib', + '--enable-R-framework=no'] if '+external-lapack' in spec: - options.extend(['--with-blas', '--with-lapack']) + configure_args.extend(['--with-blas', '--with-lapack']) - configure(*options) + configure(*configure_args) make() make('install') @@ -111,7 +111,7 @@ def setup_dependent_environment(self, spack_env, run_env, extension_spec): extension_spec.prefix, self.r_lib_dir)) def setup_dependent_package(self, module, ext_spec): - """Called before R modules' install() methods. In most cases, + """Called before R modules' install() methods. In most cases, extensions will only need to have one line: R('CMD', 'INSTALL', '--library=%s' % self.module.r_lib_dir, '%s' % self.stage.archive_file)""" From 37dc13edaf92c9fa45c0b82c736f5b40ed8172f7 Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Sun, 29 May 2016 17:03:38 -0500 Subject: [PATCH 416/481] Update version of BiocGenerics The BioConductor repositories only keep the most recent version so this has to be bumped. --- var/spack/repos/builtin/packages/r-BiocGenerics/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/r-BiocGenerics/package.py b/var/spack/repos/builtin/packages/r-BiocGenerics/package.py index 1d994cc2ef..16a8027e3c 100644 --- a/var/spack/repos/builtin/packages/r-BiocGenerics/package.py +++ b/var/spack/repos/builtin/packages/r-BiocGenerics/package.py @@ -31,7 +31,7 @@ class RBiocgenerics(Package): homepage = 'https://www.bioconductor.org/packages/release/bioc/html/BiocGenerics.html' url = "https://www.bioconductor.org/packages/release/bioc/src/contrib/BiocGenerics_0.16.1.tar.gz" - version('0.16.1', 'c2148ffd86fc6f1f819c7f68eb2c744f', expand=False) + version('0.18.0', 'baaff00eb2c2b15396fed2f7f43e634c', expand=False) extends('R') From 0e64c25ffc09d764eaeb11ffb0db93bb8baaf34a Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Sun, 29 May 2016 17:40:26 -0500 Subject: [PATCH 417/481] Fix flake8 errors Use Bioconductor provided Short URL for homepage in BiocGenerics. Wrap description text for utf variant in pcre. --- var/spack/repos/builtin/packages/pcre/package.py | 3 ++- var/spack/repos/builtin/packages/r-BiocGenerics/package.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/var/spack/repos/builtin/packages/pcre/package.py b/var/spack/repos/builtin/packages/pcre/package.py index 53b23a203d..a43729a1ac 100644 --- a/var/spack/repos/builtin/packages/pcre/package.py +++ b/var/spack/repos/builtin/packages/pcre/package.py @@ -39,7 +39,8 @@ class Pcre(Package): patch("intel.patch") variant('utf', default=True, - description='Enable support for UTF-8/16/32, incompatible with EBCDIC.') + description='Enable support for UTF-8/16/32, ' + 'incompatible with EBCDIC.') def install(self, spec, prefix): configure_args = ['--prefix=%s' % prefix] diff --git a/var/spack/repos/builtin/packages/r-BiocGenerics/package.py b/var/spack/repos/builtin/packages/r-BiocGenerics/package.py index 16a8027e3c..d509bc3d84 100644 --- a/var/spack/repos/builtin/packages/r-BiocGenerics/package.py +++ b/var/spack/repos/builtin/packages/r-BiocGenerics/package.py @@ -28,8 +28,8 @@ class RBiocgenerics(Package): """S4 generic functions needed by many Bioconductor packages.""" - homepage = 'https://www.bioconductor.org/packages/release/bioc/html/BiocGenerics.html' - url = "https://www.bioconductor.org/packages/release/bioc/src/contrib/BiocGenerics_0.16.1.tar.gz" + homepage = 'https://bioconductor.org/packages/BiocGenerics/' + url = 'https://www.bioconductor.org/packages/release/bioc/src/contrib/BiocGenerics_0.18.0.tar.gz' version('0.18.0', 'baaff00eb2c2b15396fed2f7f43e634c', expand=False) From 7213b123245895ab3fe588c7b5656b68f648b99d Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Sun, 5 Jun 2016 20:12:11 -0500 Subject: [PATCH 418/481] Tweak how R packages are built. Add bzip2 as a dependency to base R. Switch from installing the R packages from tar files to installing from the extracted directory. This is more in line with a standard install and also allows installing from VCS. Use the Bioconductor version for BiocGenerics and future Bioconductor packages. Suffix the package with 'bioc-$version' to indicate which Bioconductor release the package is a part of. This is actually more important than the actual version attached to individual packages although those are subject to change within the Bioconductor branch. --- var/spack/repos/builtin/packages/R/package.py | 1 + .../repos/builtin/packages/r-BiocGenerics/package.py | 11 +++++++---- var/spack/repos/builtin/packages/r-abind/package.py | 4 ++-- .../repos/builtin/packages/r-filehash/package.py | 4 ++-- var/spack/repos/builtin/packages/r-magic/package.py | 4 ++-- 5 files changed, 14 insertions(+), 10 deletions(-) diff --git a/var/spack/repos/builtin/packages/R/package.py b/var/spack/repos/builtin/packages/R/package.py index af520082ec..74dd95e5c2 100644 --- a/var/spack/repos/builtin/packages/R/package.py +++ b/var/spack/repos/builtin/packages/R/package.py @@ -61,6 +61,7 @@ class R(Package): depends_on('icu') depends_on('glib') depends_on('zlib') + depends_on('bzip2') depends_on('libtiff') depends_on('jpeg') depends_on('cairo') diff --git a/var/spack/repos/builtin/packages/r-BiocGenerics/package.py b/var/spack/repos/builtin/packages/r-BiocGenerics/package.py index d509bc3d84..ffd578a42c 100644 --- a/var/spack/repos/builtin/packages/r-BiocGenerics/package.py +++ b/var/spack/repos/builtin/packages/r-BiocGenerics/package.py @@ -29,12 +29,15 @@ class RBiocgenerics(Package): """S4 generic functions needed by many Bioconductor packages.""" homepage = 'https://bioconductor.org/packages/BiocGenerics/' - url = 'https://www.bioconductor.org/packages/release/bioc/src/contrib/BiocGenerics_0.18.0.tar.gz' - - version('0.18.0', 'baaff00eb2c2b15396fed2f7f43e634c', expand=False) + version('bioc-3.3', + git='https://github.com/Bioconductor-mirror/BiocGenerics.git', + branch='release-3.3') + version('bioc-3.2', + git='https://github.com/Bioconductor-mirror/BiocGenerics.git', + branch='release-3.2') extends('R') def install(self, spec, prefix): R('CMD', 'INSTALL', '--library=%s' % - self.module.r_lib_dir, '%s' % self.stage.archive_file) + self.module.r_lib_dir, '%s' % self.stage.source_path) diff --git a/var/spack/repos/builtin/packages/r-abind/package.py b/var/spack/repos/builtin/packages/r-abind/package.py index f9ac684aa4..b67a06d9b5 100644 --- a/var/spack/repos/builtin/packages/r-abind/package.py +++ b/var/spack/repos/builtin/packages/r-abind/package.py @@ -34,11 +34,11 @@ class RAbind(Package): homepage = "https://cran.r-project.org/" url = "https://cran.r-project.org/src/contrib/abind_1.4-3.tar.gz" - version('1.4-3', '10fcf80c677b991bf263d38be35a1fc5', expand=False) + version('1.4-3', '10fcf80c677b991bf263d38be35a1fc5') extends('R') def install(self, spec, prefix): R('CMD', 'INSTALL', '--library=%s' % - self.module.r_lib_dir, '%s' % self.stage.archive_file) + self.module.r_lib_dir, '%s' % self.stage.source_path) diff --git a/var/spack/repos/builtin/packages/r-filehash/package.py b/var/spack/repos/builtin/packages/r-filehash/package.py index e59ba18fa3..924066e063 100644 --- a/var/spack/repos/builtin/packages/r-filehash/package.py +++ b/var/spack/repos/builtin/packages/r-filehash/package.py @@ -39,10 +39,10 @@ class RFilehash(Package): homepage = 'https://cran.r-project.org/' url = "https://cran.r-project.org/src/contrib/filehash_2.3.tar.gz" - version('2.3', '01fffafe09b148ccadc9814c103bdc2f', expand=False) + version('2.3', '01fffafe09b148ccadc9814c103bdc2f') extends('R') def install(self, spec, prefix): R('CMD', 'INSTALL', '--library=%s' % - self.module.r_lib_dir, '%s' % self.stage.archive_file) + self.module.r_lib_dir, '%s' % self.stage.source_path) diff --git a/var/spack/repos/builtin/packages/r-magic/package.py b/var/spack/repos/builtin/packages/r-magic/package.py index ec4b9b9e41..31dab3b8af 100644 --- a/var/spack/repos/builtin/packages/r-magic/package.py +++ b/var/spack/repos/builtin/packages/r-magic/package.py @@ -34,7 +34,7 @@ class RMagic(Package): homepage = "https://cran.r-project.org/" url = "https://cran.r-project.org/src/contrib/magic_1.5-6.tar.gz" - version('1.5-6', 'a68e5ced253b2196af842e1fc84fd029', expand=False) + version('1.5-6', 'a68e5ced253b2196af842e1fc84fd029') extends('R') @@ -42,4 +42,4 @@ class RMagic(Package): def install(self, spec, prefix): R('CMD', 'INSTALL', '--library=%s' % - self.module.r_lib_dir, '%s' % self.stage.archive_file) + self.module.r_lib_dir, '%s' % self.stage.source_path) From 7da405e630b6b0e65f4ec8d551a4ed988f32ac74 Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Mon, 6 Jun 2016 17:06:50 -0500 Subject: [PATCH 419/481] Add validator for bioconductor packages This commit adds a validator function to make sure the versions of R and bioconductor are in sync. --- .../builtin/packages/r-BiocGenerics/package.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/var/spack/repos/builtin/packages/r-BiocGenerics/package.py b/var/spack/repos/builtin/packages/r-BiocGenerics/package.py index ffd578a42c..5a6ce72a4b 100644 --- a/var/spack/repos/builtin/packages/r-BiocGenerics/package.py +++ b/var/spack/repos/builtin/packages/r-BiocGenerics/package.py @@ -38,6 +38,19 @@ class RBiocgenerics(Package): extends('R') + def validate(self, spec): + """ + Checks that the version of R is appropriate for the Bioconductor + version. + """ + if spec.satisfies('@bioc-3.3'): + if not spec.satisfies('^R@3.3.0:3.3.9'): + raise InstallError('Must use R-3.3 for Bioconductor-3.3') + elif spec.satisfies('@bioc-3.2'): + if not spec.satisfies('^R@3.2.0:3.2.9'): + raise InstallError('Must use R-3.2 for Bioconductor-3.2') + def install(self, spec, prefix): + self.validate(spec) R('CMD', 'INSTALL', '--library=%s' % self.module.r_lib_dir, '%s' % self.stage.source_path) From 4850b9d4d1446f96732e27053df50c16629ec448 Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Mon, 6 Jun 2016 17:30:35 -0500 Subject: [PATCH 420/481] Make install comment consistent with actual call. --- var/spack/repos/builtin/packages/R/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/R/package.py b/var/spack/repos/builtin/packages/R/package.py index 74dd95e5c2..c7d8f89906 100644 --- a/var/spack/repos/builtin/packages/R/package.py +++ b/var/spack/repos/builtin/packages/R/package.py @@ -115,7 +115,7 @@ def setup_dependent_package(self, module, ext_spec): """Called before R modules' install() methods. In most cases, extensions will only need to have one line: R('CMD', 'INSTALL', '--library=%s' % self.module.r_lib_dir, '%s' % - self.stage.archive_file)""" + self.stage.source_path)""" # R extension builds can have a global R executable function module.R = Executable(join_path(self.spec.prefix.bin, 'R')) From a813f03955397140104b7d926f0fddf925e672bc Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Sat, 11 Jun 2016 16:18:54 -0500 Subject: [PATCH 421/481] Add list_url for old versions. --- var/spack/repos/builtin/packages/r-abind/package.py | 8 ++++---- var/spack/repos/builtin/packages/r-filehash/package.py | 7 ++++--- var/spack/repos/builtin/packages/r-magic/package.py | 7 ++++--- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/var/spack/repos/builtin/packages/r-abind/package.py b/var/spack/repos/builtin/packages/r-abind/package.py index b67a06d9b5..96f623a096 100644 --- a/var/spack/repos/builtin/packages/r-abind/package.py +++ b/var/spack/repos/builtin/packages/r-abind/package.py @@ -32,13 +32,13 @@ class RAbind(Package): 'afill' for manipulating, extracting and replacing data in arrays.""" homepage = "https://cran.r-project.org/" - url = "https://cran.r-project.org/src/contrib/abind_1.4-3.tar.gz" + url = "https://cran.r-project.org/src/contrib/abind_1.4-3.tar.gz" + list_url = "https://cran.r-project.org/src/contrib/Archive/abind" version('1.4-3', '10fcf80c677b991bf263d38be35a1fc5') extends('R') def install(self, spec, prefix): - - R('CMD', 'INSTALL', '--library=%s' % - self.module.r_lib_dir, '%s' % self.stage.source_path) + R('CMD', 'INSTALL', '--library={0}'.format(self.module.r_lib_dir), + self.stage.source_path) diff --git a/var/spack/repos/builtin/packages/r-filehash/package.py b/var/spack/repos/builtin/packages/r-filehash/package.py index 924066e063..a09b458eaf 100644 --- a/var/spack/repos/builtin/packages/r-filehash/package.py +++ b/var/spack/repos/builtin/packages/r-filehash/package.py @@ -37,12 +37,13 @@ class RFilehash(Package): parties for use in the 'filehash' framework.""" homepage = 'https://cran.r-project.org/' - url = "https://cran.r-project.org/src/contrib/filehash_2.3.tar.gz" + url = "https://cran.r-project.org/src/contrib/filehash_2.3.tar.gz" + list_url = "https://cran.r-project.org/src/contrib/Archive/filehash" version('2.3', '01fffafe09b148ccadc9814c103bdc2f') extends('R') def install(self, spec, prefix): - R('CMD', 'INSTALL', '--library=%s' % - self.module.r_lib_dir, '%s' % self.stage.source_path) + R('CMD', 'INSTALL', '--library={0}'.format(self.module.r_lib_dir), + self.stage.source_path) diff --git a/var/spack/repos/builtin/packages/r-magic/package.py b/var/spack/repos/builtin/packages/r-magic/package.py index 31dab3b8af..f6ba97740b 100644 --- a/var/spack/repos/builtin/packages/r-magic/package.py +++ b/var/spack/repos/builtin/packages/r-magic/package.py @@ -32,7 +32,8 @@ class RMagic(Package): arrays.""" homepage = "https://cran.r-project.org/" - url = "https://cran.r-project.org/src/contrib/magic_1.5-6.tar.gz" + url = "https://cran.r-project.org/src/contrib/magic_1.5-6.tar.gz" + list_url = "https://cran.r-project.org/src/contrib/Archive/magic" version('1.5-6', 'a68e5ced253b2196af842e1fc84fd029') @@ -41,5 +42,5 @@ class RMagic(Package): depends_on('r-abind') def install(self, spec, prefix): - R('CMD', 'INSTALL', '--library=%s' % - self.module.r_lib_dir, '%s' % self.stage.source_path) + R('CMD', 'INSTALL', '--library={0}'.format(self.module.r_lib_dir), + self.stage.source_path) From 8970785d33e07105f47bca390d846127d7e5ce7b Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Mon, 13 Jun 2016 17:22:44 -0500 Subject: [PATCH 422/481] Add 'jdk' as a dependency. Add jdk as a dependency so that R has support for java and allow packages that need java to work. --- var/spack/repos/builtin/packages/R/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/R/package.py b/var/spack/repos/builtin/packages/R/package.py index c7d8f89906..001dde5329 100644 --- a/var/spack/repos/builtin/packages/R/package.py +++ b/var/spack/repos/builtin/packages/R/package.py @@ -71,6 +71,7 @@ class R(Package): depends_on('tk') depends_on('curl') depends_on('pcre') + depends_on('jdk') def install(self, spec, prefix): rlibdir = join_path(prefix, 'rlib') From a660c60eb1997317fe0e99054a99cc1cea1d3a56 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Fri, 10 Jun 2016 09:24:52 -0500 Subject: [PATCH 423/481] Add py-sncosmo package --- .../builtin/packages/py-astropy/package.py | 41 ++++++++------ .../builtin/packages/py-emcee/package.py | 41 ++++++++++++++ .../repos/builtin/packages/py-h5py/package.py | 31 +++++++---- .../builtin/packages/py-iminuit/package.py | 54 +++++++++++++++++++ .../builtin/packages/py-nestle/package.py | 44 +++++++++++++++ .../builtin/packages/py-sncosmo/package.py | 51 ++++++++++++++++++ 6 files changed, 237 insertions(+), 25 deletions(-) create mode 100644 var/spack/repos/builtin/packages/py-emcee/package.py create mode 100644 var/spack/repos/builtin/packages/py-iminuit/package.py create mode 100644 var/spack/repos/builtin/packages/py-nestle/package.py create mode 100644 var/spack/repos/builtin/packages/py-sncosmo/package.py diff --git a/var/spack/repos/builtin/packages/py-astropy/package.py b/var/spack/repos/builtin/packages/py-astropy/package.py index 86875bbcae..09d1688f4b 100644 --- a/var/spack/repos/builtin/packages/py-astropy/package.py +++ b/var/spack/repos/builtin/packages/py-astropy/package.py @@ -24,29 +24,40 @@ ############################################################################## from spack import * -class PyAstropy(Package): - """ - The Astropy Project is a community effort to develop a single core - package for Astronomy in Python and foster interoperability between - Python astronomy packages. - """ - homepage = 'http://www.astropy.org/' +class PyAstropy(Package): + """The Astropy Project is a community effort to develop a single core + package for Astronomy in Python and foster interoperability between + Python astronomy packages.""" + + homepage = 'http://www.astropy.org/' + url = 'https://pypi.python.org/packages/source/a/astropy/astropy-1.1.2.tar.gz' + + version('1.1.2', 'cbe32023b5b1177d1e2498a0d00cda51') version('1.1.post1', 'b52919f657a37d45cc45f5cb0f58c44d') - def url_for_version(self, version): - return 'https://pypi.python.org/packages/source/a/astropy/astropy-{0}.tar.gz'.format(version) - - extends('python') + # Required dependencies + extends('python@2.6.5:2.7,3.3:') + depends_on('py-numpy@1.6.0:') + # Optional dependencies depends_on('cfitsio') depends_on('expat') depends_on('py-h5py') - depends_on('py-numpy') + depends_on('py-pyyaml') depends_on('py-scipy') + depends_on('py-matplotlib') + depends_on('py-pytz') + depends_on('py-scikit-image') + depends_on('py-pandas') + + # TODO: Add packages + # depends_on('py-beautiful-soup') + # depends_on('py-xmllint') + # depends_on('py-wcsaxes') + # depends_on('py-objgraph') def install(self, spec, prefix): python('setup.py', 'build', '--use-system-cfitsio', - '--use-system-expat') - python('setup.py', 'install', '--prefix=' + prefix) - + '--use-system-expat') + python('setup.py', 'install', '--prefix={0}'.format(prefix)) diff --git a/var/spack/repos/builtin/packages/py-emcee/package.py b/var/spack/repos/builtin/packages/py-emcee/package.py new file mode 100644 index 0000000000..d57ef4bd76 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-emcee/package.py @@ -0,0 +1,41 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class PyEmcee(Package): + """emcee is an MIT licensed pure-Python implementation of Goodman & Weare's + Affine Invariant Markov chain Monte Carlo (MCMC) Ensemble sampler.""" + + homepage = "http://dan.iel.fm/emcee/current/" + url = "https://pypi.python.org/packages/source/e/emcee/emcee-2.1.0.tar.gz" + + version('2.1.0', 'c6b6fad05c824d40671d4a4fc58dfff7') + + extends('python') + depends_on('py-numpy') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix={0}'.format(prefix)) diff --git a/var/spack/repos/builtin/packages/py-h5py/package.py b/var/spack/repos/builtin/packages/py-h5py/package.py index a8eb027ba5..a92fc3c9c7 100644 --- a/var/spack/repos/builtin/packages/py-h5py/package.py +++ b/var/spack/repos/builtin/packages/py-h5py/package.py @@ -23,23 +23,34 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## from spack import * -import re + class PyH5py(Package): - """The h5py package provides both a high- and low-level interface to the HDF5 library from Python.""" + """The h5py package provides both a high- and low-level interface to the + HDF5 library from Python.""" + homepage = "https://pypi.python.org/pypi/h5py" url = "https://pypi.python.org/packages/source/h/h5py/h5py-2.4.0.tar.gz" - version('2.4.0', '80c9a94ae31f84885cc2ebe1323d6758') + version('2.6.0', 'ec476211bd1de3f5ac150544189b0bf4') version('2.5.0', '6e4301b5ad5da0d51b0a1e5ac19e3b74') + version('2.4.0', '80c9a94ae31f84885cc2ebe1323d6758') - extends('python', ignore=lambda f: re.match(r'bin/cy*', f)) - depends_on('hdf5') - depends_on('py-numpy') + variant('mpi', default=False, description='Build with MPI support') + + extends('python', ignore=r'bin/cy.*') + depends_on('hdf5@1.8.4:+mpi', when='+mpi') + depends_on('hdf5@1.8.4:~mpi', when='~mpi') + depends_on('py-numpy@1.6.1:') depends_on('py-cython') - depends_on('py-six') - depends_on('py-pkgconfig') + depends_on('mpi', when='+mpi') def install(self, spec, prefix): - python('setup.py', 'configure', '--hdf5=%s' % spec['hdf5'].prefix) - python('setup.py', 'install', '--prefix=%s' % prefix) + python('setup.py', 'configure', + '--hdf5={0}'.format(spec['hdf5'].prefix)) + + if '+mpi' in spec: + env['CC'] = spec['mpi'].mpicc + python('setup.py', 'configure', '--mpi') + + python('setup.py', 'install', '--prefix={0}'.format(prefix)) diff --git a/var/spack/repos/builtin/packages/py-iminuit/package.py b/var/spack/repos/builtin/packages/py-iminuit/package.py new file mode 100644 index 0000000000..5a42268a73 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-iminuit/package.py @@ -0,0 +1,54 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class PyIminuit(Package): + """Interactive IPython-Friendly Minimizer based on SEAL Minuit2.""" + + homepage = "https://pypi.python.org/pypi/iminuit" + url = "https://pypi.python.org/packages/source/i/iminuit/iminuit-1.2.tar.gz" + + version('1.2', '4701ec472cae42015e26251703e6e984') + + # Required dependencies + extends('python@2.7:2.8,3.4:') + depends_on('py-setuptools') + + # Optional dependencies + depends_on('py-numpy') + depends_on('py-matplotlib') + + def install(self, spec, prefix): + site_packages = '{0}/lib/python{1}/site-packages'.format( + prefix, spec['python'].version.up_to(2)) + + # site-packages directory must already exist + mkdirp(site_packages) + + # PYTHONPATH must include site-packages directory + env['PYTHONPATH'] += ':{0}'.format(site_packages) + + python('setup.py', 'install', '--prefix={0}'.format(prefix)) diff --git a/var/spack/repos/builtin/packages/py-nestle/package.py b/var/spack/repos/builtin/packages/py-nestle/package.py new file mode 100644 index 0000000000..38fa0561db --- /dev/null +++ b/var/spack/repos/builtin/packages/py-nestle/package.py @@ -0,0 +1,44 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class PyNestle(Package): + """Nested sampling algorithms for evaluating Bayesian evidence.""" + + homepage = "http://kbarbary.github.io/nestle/" + url = "https://pypi.python.org/packages/source/n/nestle/nestle-0.1.1.tar.gz" + + version('0.1.1', '4875c0f9a0a8e263c1d7f5fa6ce604c5') + + # Required dependencies + extends('python') + depends_on('py-numpy@1.6:') + + # Optional dependencies + depends_on('py-scipy') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix={0}'.format(prefix)) diff --git a/var/spack/repos/builtin/packages/py-sncosmo/package.py b/var/spack/repos/builtin/packages/py-sncosmo/package.py new file mode 100644 index 0000000000..d40623ffa1 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-sncosmo/package.py @@ -0,0 +1,51 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class PySncosmo(Package): + """SNCosmo is a Python library for high-level supernova cosmology + analysis.""" + + homepage = "http://sncosmo.readthedocs.io/" + url = "https://pypi.python.org/packages/source/s/sncosmo/sncosmo-1.2.0.tar.gz" + + version('1.2.0', '028e6d1dc84ab1c17d2f3b6378b2cb1e') + + # Required dependencies + # py-sncosmo binaries are duplicates of those from py-astropy + extends('python@2.6:2.7,3.3:', ignore=r'bin/*') + depends_on('py-numpy') + depends_on('py-scipy') + depends_on('py-astropy') + + # Recommended dependencies + depends_on('py-matplotlib') + depends_on('py-iminuit') + depends_on('py-emcee') + depends_on('py-nestle') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix={0}'.format(prefix)) From 4eeaff175cc6c9fd15a7d2455e1e0663ad147aca Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Fri, 10 Jun 2016 14:29:57 -0500 Subject: [PATCH 424/481] Allow packages to build --- .../repos/builtin/packages/py-astropy/package.py | 3 ++- var/spack/repos/builtin/packages/py-h5py/package.py | 4 +++- .../repos/builtin/packages/py-iminuit/package.py | 12 ++---------- .../repos/builtin/packages/py-sncosmo/package.py | 3 ++- 4 files changed, 9 insertions(+), 13 deletions(-) diff --git a/var/spack/repos/builtin/packages/py-astropy/package.py b/var/spack/repos/builtin/packages/py-astropy/package.py index 09d1688f4b..bac794e551 100644 --- a/var/spack/repos/builtin/packages/py-astropy/package.py +++ b/var/spack/repos/builtin/packages/py-astropy/package.py @@ -60,4 +60,5 @@ class PyAstropy(Package): def install(self, spec, prefix): python('setup.py', 'build', '--use-system-cfitsio', '--use-system-expat') - python('setup.py', 'install', '--prefix={0}'.format(prefix)) + python('setup.py', 'install', '--single-version-externally-managed', + '--root=/', '--prefix={0}'.format(prefix)) diff --git a/var/spack/repos/builtin/packages/py-h5py/package.py b/var/spack/repos/builtin/packages/py-h5py/package.py index a92fc3c9c7..1b8675ecf2 100644 --- a/var/spack/repos/builtin/packages/py-h5py/package.py +++ b/var/spack/repos/builtin/packages/py-h5py/package.py @@ -39,6 +39,7 @@ class PyH5py(Package): variant('mpi', default=False, description='Build with MPI support') extends('python', ignore=r'bin/cy.*') + depends_on('hdf5@1.8.4:+mpi', when='+mpi') depends_on('hdf5@1.8.4:~mpi', when='~mpi') depends_on('py-numpy@1.6.1:') @@ -53,4 +54,5 @@ def install(self, spec, prefix): env['CC'] = spec['mpi'].mpicc python('setup.py', 'configure', '--mpi') - python('setup.py', 'install', '--prefix={0}'.format(prefix)) + python('setup.py', 'install', '--single-version-externally-managed', + '--root=/', '--prefix={0}'.format(prefix)) diff --git a/var/spack/repos/builtin/packages/py-iminuit/package.py b/var/spack/repos/builtin/packages/py-iminuit/package.py index 5a42268a73..557cf19c19 100644 --- a/var/spack/repos/builtin/packages/py-iminuit/package.py +++ b/var/spack/repos/builtin/packages/py-iminuit/package.py @@ -42,13 +42,5 @@ class PyIminuit(Package): depends_on('py-matplotlib') def install(self, spec, prefix): - site_packages = '{0}/lib/python{1}/site-packages'.format( - prefix, spec['python'].version.up_to(2)) - - # site-packages directory must already exist - mkdirp(site_packages) - - # PYTHONPATH must include site-packages directory - env['PYTHONPATH'] += ':{0}'.format(site_packages) - - python('setup.py', 'install', '--prefix={0}'.format(prefix)) + python('setup.py', 'install', '--single-version-externally-managed', + '--root=/', '--prefix={0}'.format(prefix)) diff --git a/var/spack/repos/builtin/packages/py-sncosmo/package.py b/var/spack/repos/builtin/packages/py-sncosmo/package.py index d40623ffa1..58c4625a4f 100644 --- a/var/spack/repos/builtin/packages/py-sncosmo/package.py +++ b/var/spack/repos/builtin/packages/py-sncosmo/package.py @@ -48,4 +48,5 @@ class PySncosmo(Package): depends_on('py-nestle') def install(self, spec, prefix): - python('setup.py', 'install', '--prefix={0}'.format(prefix)) + python('setup.py', 'install', '--single-version-externally-managed', + '--root=/', '--prefix={0}'.format(prefix)) From b2aa78d6ab569f5dac65b665c2f9b2962b72fdbb Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Fri, 10 Jun 2016 17:08:50 -0500 Subject: [PATCH 425/481] Remove version constraints --- var/spack/repos/builtin/packages/py-astropy/package.py | 7 +++---- var/spack/repos/builtin/packages/py-h5py/package.py | 3 +-- var/spack/repos/builtin/packages/py-iminuit/package.py | 5 ++--- var/spack/repos/builtin/packages/py-nestle/package.py | 2 +- var/spack/repos/builtin/packages/py-sncosmo/package.py | 5 ++--- 5 files changed, 9 insertions(+), 13 deletions(-) diff --git a/var/spack/repos/builtin/packages/py-astropy/package.py b/var/spack/repos/builtin/packages/py-astropy/package.py index bac794e551..e3776caf70 100644 --- a/var/spack/repos/builtin/packages/py-astropy/package.py +++ b/var/spack/repos/builtin/packages/py-astropy/package.py @@ -37,8 +37,8 @@ class PyAstropy(Package): version('1.1.post1', 'b52919f657a37d45cc45f5cb0f58c44d') # Required dependencies - extends('python@2.6.5:2.7,3.3:') - depends_on('py-numpy@1.6.0:') + extends('python') + depends_on('py-numpy') # Optional dependencies depends_on('cfitsio') @@ -60,5 +60,4 @@ class PyAstropy(Package): def install(self, spec, prefix): python('setup.py', 'build', '--use-system-cfitsio', '--use-system-expat') - python('setup.py', 'install', '--single-version-externally-managed', - '--root=/', '--prefix={0}'.format(prefix)) + python('setup.py', 'install', '--prefix={0}'.format(prefix)) diff --git a/var/spack/repos/builtin/packages/py-h5py/package.py b/var/spack/repos/builtin/packages/py-h5py/package.py index 1b8675ecf2..1379513197 100644 --- a/var/spack/repos/builtin/packages/py-h5py/package.py +++ b/var/spack/repos/builtin/packages/py-h5py/package.py @@ -54,5 +54,4 @@ def install(self, spec, prefix): env['CC'] = spec['mpi'].mpicc python('setup.py', 'configure', '--mpi') - python('setup.py', 'install', '--single-version-externally-managed', - '--root=/', '--prefix={0}'.format(prefix)) + python('setup.py', 'install', '--prefix={0}'.format(prefix)) diff --git a/var/spack/repos/builtin/packages/py-iminuit/package.py b/var/spack/repos/builtin/packages/py-iminuit/package.py index 557cf19c19..ffc5a5bd14 100644 --- a/var/spack/repos/builtin/packages/py-iminuit/package.py +++ b/var/spack/repos/builtin/packages/py-iminuit/package.py @@ -34,7 +34,7 @@ class PyIminuit(Package): version('1.2', '4701ec472cae42015e26251703e6e984') # Required dependencies - extends('python@2.7:2.8,3.4:') + extends('python') depends_on('py-setuptools') # Optional dependencies @@ -42,5 +42,4 @@ class PyIminuit(Package): depends_on('py-matplotlib') def install(self, spec, prefix): - python('setup.py', 'install', '--single-version-externally-managed', - '--root=/', '--prefix={0}'.format(prefix)) + python('setup.py', 'install', '--prefix={0}'.format(prefix)) diff --git a/var/spack/repos/builtin/packages/py-nestle/package.py b/var/spack/repos/builtin/packages/py-nestle/package.py index 38fa0561db..16506e3eca 100644 --- a/var/spack/repos/builtin/packages/py-nestle/package.py +++ b/var/spack/repos/builtin/packages/py-nestle/package.py @@ -35,7 +35,7 @@ class PyNestle(Package): # Required dependencies extends('python') - depends_on('py-numpy@1.6:') + depends_on('py-numpy') # Optional dependencies depends_on('py-scipy') diff --git a/var/spack/repos/builtin/packages/py-sncosmo/package.py b/var/spack/repos/builtin/packages/py-sncosmo/package.py index 58c4625a4f..8976413ab9 100644 --- a/var/spack/repos/builtin/packages/py-sncosmo/package.py +++ b/var/spack/repos/builtin/packages/py-sncosmo/package.py @@ -36,7 +36,7 @@ class PySncosmo(Package): # Required dependencies # py-sncosmo binaries are duplicates of those from py-astropy - extends('python@2.6:2.7,3.3:', ignore=r'bin/*') + extends('python', ignore=r'bin/*') depends_on('py-numpy') depends_on('py-scipy') depends_on('py-astropy') @@ -48,5 +48,4 @@ class PySncosmo(Package): depends_on('py-nestle') def install(self, spec, prefix): - python('setup.py', 'install', '--single-version-externally-managed', - '--root=/', '--prefix={0}'.format(prefix)) + python('setup.py', 'install', '--prefix={0}'.format(prefix)) From 24f65c502e54cc8767d47bafa827c7434c81bd07 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Mon, 13 Jun 2016 12:31:18 -0500 Subject: [PATCH 426/481] Add additional Astropy dependencies --- .../builtin/packages/py-astropy/package.py | 12 +++--- .../packages/py-beautifulsoup4/package.py | 41 ++++++++++++++++++ .../builtin/packages/py-wcsaxes/package.py | 43 +++++++++++++++++++ 3 files changed, 89 insertions(+), 7 deletions(-) create mode 100644 var/spack/repos/builtin/packages/py-beautifulsoup4/package.py create mode 100644 var/spack/repos/builtin/packages/py-wcsaxes/package.py diff --git a/var/spack/repos/builtin/packages/py-astropy/package.py b/var/spack/repos/builtin/packages/py-astropy/package.py index e3776caf70..a9962777dc 100644 --- a/var/spack/repos/builtin/packages/py-astropy/package.py +++ b/var/spack/repos/builtin/packages/py-astropy/package.py @@ -41,21 +41,19 @@ class PyAstropy(Package): depends_on('py-numpy') # Optional dependencies - depends_on('cfitsio') - depends_on('expat') depends_on('py-h5py') + depends_on('py-beautifulsoup4') depends_on('py-pyyaml') depends_on('py-scipy') + depends_on('libxml2') depends_on('py-matplotlib') depends_on('py-pytz') depends_on('py-scikit-image') depends_on('py-pandas') - # TODO: Add packages - # depends_on('py-beautiful-soup') - # depends_on('py-xmllint') - # depends_on('py-wcsaxes') - # depends_on('py-objgraph') + # System dependencies + depends_on('cfitsio') + depends_on('expat') def install(self, spec, prefix): python('setup.py', 'build', '--use-system-cfitsio', diff --git a/var/spack/repos/builtin/packages/py-beautifulsoup4/package.py b/var/spack/repos/builtin/packages/py-beautifulsoup4/package.py new file mode 100644 index 0000000000..64368fe70d --- /dev/null +++ b/var/spack/repos/builtin/packages/py-beautifulsoup4/package.py @@ -0,0 +1,41 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class PyBeautifulsoup4(Package): + """Beautiful Soup is a Python library for pulling data out of HTML and + XML files. It works with your favorite parser to provide idiomatic ways + of navigating, searching, and modifying the parse tree.""" + + homepage = "https://www.crummy.com/software/BeautifulSoup" + url = "https://pypi.python.org/packages/source/b/beautifulsoup4/beautifulsoup4-4.4.1.tar.gz" + + version('4.4.1', '8fbd9a7cac0704645fa20d1419036815') + + extends('python') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix={0}'.format(prefix)) diff --git a/var/spack/repos/builtin/packages/py-wcsaxes/package.py b/var/spack/repos/builtin/packages/py-wcsaxes/package.py new file mode 100644 index 0000000000..b0adbe3658 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-wcsaxes/package.py @@ -0,0 +1,43 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class PyWcsaxes(Package): + """WCSAxes is a framework for making plots of Astronomical data + in Matplotlib.""" + + homepage = "http://wcsaxes.readthedocs.io/en/latest/index.html" + url = "https://github.com/astrofrog/wcsaxes/archive/v0.8.tar.gz" + + version('0.8', 'de1c60fdae4c330bf5ddb9f1ab5ab920') + + extends('python', ignore=r'bin/pbr') + depends_on('py-numpy') + depends_on('py-matplotlib') + depends_on('py-astropy') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix={0}'.format(prefix)) From 80d444624b433d1059bed6580c24fb87db331016 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Wed, 15 Jun 2016 16:42:50 -0500 Subject: [PATCH 427/481] Fix activation of sites.py for sncosmo --- var/spack/repos/builtin/packages/py-h5py/package.py | 13 ++++++++++--- .../repos/builtin/packages/py-iminuit/package.py | 1 + .../repos/builtin/packages/py-sncosmo/package.py | 2 +- var/spack/repos/builtin/packages/python/package.py | 7 +++---- 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/var/spack/repos/builtin/packages/py-h5py/package.py b/var/spack/repos/builtin/packages/py-h5py/package.py index 1379513197..0180b658f5 100644 --- a/var/spack/repos/builtin/packages/py-h5py/package.py +++ b/var/spack/repos/builtin/packages/py-h5py/package.py @@ -38,14 +38,21 @@ class PyH5py(Package): variant('mpi', default=False, description='Build with MPI support') - extends('python', ignore=r'bin/cy.*') + extends('python') + # Build dependencies + depends_on('py-cython@0.19:') + depends_on('pkg-config') depends_on('hdf5@1.8.4:+mpi', when='+mpi') depends_on('hdf5@1.8.4:~mpi', when='~mpi') - depends_on('py-numpy@1.6.1:') - depends_on('py-cython') depends_on('mpi', when='+mpi') + # Build and runtime dependencies + depends_on('py-numpy@1.6.1:') + + # Runtime dependencies + depends_on('py-six') + def install(self, spec, prefix): python('setup.py', 'configure', '--hdf5={0}'.format(spec['hdf5'].prefix)) diff --git a/var/spack/repos/builtin/packages/py-iminuit/package.py b/var/spack/repos/builtin/packages/py-iminuit/package.py index ffc5a5bd14..47751a702d 100644 --- a/var/spack/repos/builtin/packages/py-iminuit/package.py +++ b/var/spack/repos/builtin/packages/py-iminuit/package.py @@ -40,6 +40,7 @@ class PyIminuit(Package): # Optional dependencies depends_on('py-numpy') depends_on('py-matplotlib') + depends_on('py-cython') def install(self, spec, prefix): python('setup.py', 'install', '--prefix={0}'.format(prefix)) diff --git a/var/spack/repos/builtin/packages/py-sncosmo/package.py b/var/spack/repos/builtin/packages/py-sncosmo/package.py index 8976413ab9..82ae2a2e69 100644 --- a/var/spack/repos/builtin/packages/py-sncosmo/package.py +++ b/var/spack/repos/builtin/packages/py-sncosmo/package.py @@ -36,7 +36,7 @@ class PySncosmo(Package): # Required dependencies # py-sncosmo binaries are duplicates of those from py-astropy - extends('python', ignore=r'bin/*') + extends('python', ignore=r'bin/.*') depends_on('py-numpy') depends_on('py-scipy') depends_on('py-astropy') diff --git a/var/spack/repos/builtin/packages/python/package.py b/var/spack/repos/builtin/packages/python/package.py index 86c12498e1..d6d5fac4b1 100644 --- a/var/spack/repos/builtin/packages/python/package.py +++ b/var/spack/repos/builtin/packages/python/package.py @@ -167,14 +167,13 @@ def python_ignore(self, ext_pkg, args): ignore_arg = args.get('ignore', lambda f: False) # Always ignore easy-install.pth, as it needs to be merged. - patterns = [r'easy-install\.pth$'] + patterns = [r'site-packages/easy-install\.pth$'] # Ignore pieces of setuptools installed by other packages. if ext_pkg.name != 'py-setuptools': - patterns.append(r'/site[^/]*\.pyc?$') - patterns.append(r'setuptools\.pth') patterns.append(r'bin/easy_install[^/]*$') - patterns.append(r'setuptools.*egg$') + patterns.append(r'site-packages/setuptools\.pth$') + patterns.append(r'site-packages/site[^/]*\.pyc?$') if ext_pkg.name != 'py-numpy': patterns.append(r'bin/f2py$') From fe5eec5230dd86e2ed247a178a54230f631c1c60 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 17 Jun 2016 09:16:03 -0700 Subject: [PATCH 428/481] 'find and 'add' are options for adding compilers via command line --- lib/spack/spack/cmd/compiler.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/spack/spack/cmd/compiler.py b/lib/spack/spack/cmd/compiler.py index 94d83512e1..3a496492ff 100644 --- a/lib/spack/spack/cmd/compiler.py +++ b/lib/spack/spack/cmd/compiler.py @@ -69,7 +69,7 @@ def setup_parser(subparser): help="Configuration scope to read from.") -def compiler_add(args): +def compiler_find(args): """Search either $PATH or a list of paths OR MODULES for compilers and add them to Spack's configuration.""" paths = args.add_paths @@ -79,7 +79,6 @@ def compiler_add(args): compilers = [c for c in spack.compilers.find_compilers(*args.add_paths) if c.spec not in spack.compilers.all_compilers(scope=args.scope, init_config=False)] - if compilers: spack.compilers.add_compilers_to_config(compilers, scope=args.scope, init_config=False) @@ -139,9 +138,10 @@ def compiler_list(args): def compiler(parser, args): - action = { 'add' : compiler_add, - 'remove' : compiler_remove, - 'rm' : compiler_remove, - 'info' : compiler_info, - 'list' : compiler_list } + action = {'add' : compiler_find, + 'find' : compiler_find, + 'remove' : compiler_remove, + 'rm' : compiler_remove, + 'info' : compiler_info, + 'list' : compiler_list } action[args.compiler_command](args) From 448f8c43d44dfe8dc3b38e5a637fc560535a10d0 Mon Sep 17 00:00:00 2001 From: "Robert D. French" Date: Tue, 7 Jun 2016 11:42:15 -0400 Subject: [PATCH 429/481] Added working Ferret spec Fixed ferret spec. It builds now Made ferret spec more flake8-y Fixed homepage and info text about Ferret Removed vim swap file Reverted changes to netcdf-fortran spec and added netcdf~mpi dependency to ferret spec to compensate for it Fixed flake8 issues! --- .../repos/builtin/packages/ferret/package.py | 103 ++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 var/spack/repos/builtin/packages/ferret/package.py diff --git a/var/spack/repos/builtin/packages/ferret/package.py b/var/spack/repos/builtin/packages/ferret/package.py new file mode 100644 index 0000000000..15ddfcee16 --- /dev/null +++ b/var/spack/repos/builtin/packages/ferret/package.py @@ -0,0 +1,103 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + +from spack import * +import os + + +class Ferret(Package): + """Ferret is an interactive computer visualization and analysis environment + designed to meet the needs of oceanographers and meteorologists + analyzing large and complex gridded data sets.""" + homepage = "http://ferret.noaa.gov/Ferret/" + url = "ftp://ftp.pmel.noaa.gov/ferret/pub/source/fer_source.tar.gz" + + version('6.96', '51722027c864369f41bab5751dfff8cc', + url="ftp://ftp.pmel.noaa.gov/ferret/pub/source/fer_source.tar.gz") + + depends_on("hdf5~mpi~fortran") + depends_on("netcdf~mpi") + depends_on("netcdf-fortran") + depends_on("readline") + depends_on("zlib") + + def patch(self): + hdf5_prefix = self.spec['hdf5'].prefix + netcdff_prefix = self.spec['netcdf-fortran'].prefix + readline_prefix = self.spec['readline'].prefix + libz_prefix = self.spec['zlib'].prefix + + filter_file(r'^BUILDTYPE.+', + 'BUILDTYPE = x86_64-linux', + 'FERRET/site_specific.mk') + filter_file(r'^INSTALL_FER_DIR.+', + 'INSTALL_FER_DIR = %s' % self.spec.prefix, + 'FERRET/site_specific.mk') + filter_file(r'^HDF5_DIR.+', + 'HDF5_DIR = %s' % hdf5_prefix, + 'FERRET/site_specific.mk') + filter_file(r'^NETCDF4_DIR.+', + 'NETCDF4_DIR = %s' % netcdff_prefix, + 'FERRET/site_specific.mk') + filter_file(r'^READLINE_DIR.+', + 'READLINE_DIR = %s' % readline_prefix, + 'FERRET/site_specific.mk') + filter_file(r'^LIBZ_DIR.+', + 'LIBZ_DIR = %s' % libz_prefix, + 'FERRET/site_specific.mk') + filter_file(r'^JAVA_HOME.+', + ' ', + 'FERRET/site_specific.mk') + filter_file(r'-lm', + '-lgfortran -lm', + 'FERRET/platform_specific.mk.x86_64-linux') + + def install(self, spec, prefix): + hdf5_prefix = spec['hdf5'].prefix + netcdff_prefix = spec['netcdf-fortran'].prefix + netcdf_prefix = spec['netcdf'].prefix + libz_prefix = spec['zlib'].prefix + ln = which('ln') + ln('-sf', + hdf5_prefix + '/lib', + hdf5_prefix + '/lib64') + ln('-sf', + netcdff_prefix + '/lib', + netcdff_prefix + '/lib64') + ln('-sf', + netcdf_prefix + '/lib/libnetcdf.a', + netcdff_prefix + '/lib/libnetcdf.a') + ln('-sf', + netcdf_prefix + '/lib/libnetcdf.la', + netcdff_prefix + '/lib/libnetcdf.la') + ln('-sf', + libz_prefix + '/lib', + libz_prefix + '/lib64') + os.environ['LDFLAGS'] = '-lquadmath' + with working_dir('FERRET', create=False): + os.environ['LD_X11'] = '-L/usr/lib/X11 -lX11' + os.environ['HOSTTYPE'] = 'x86_64-linux' + make(parallel=False) + make("install") From 536fa73d5f1e0a676d3db68097bbdbab8ba424cd Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Fri, 17 Jun 2016 13:33:09 -0500 Subject: [PATCH 430/481] Python flake8 overhaul --- .../repos/builtin/packages/python/package.py | 188 +++++++++++------- 1 file changed, 113 insertions(+), 75 deletions(-) diff --git a/var/spack/repos/builtin/packages/python/package.py b/var/spack/repos/builtin/packages/python/package.py index d6d5fac4b1..b6a62bf742 100644 --- a/var/spack/repos/builtin/packages/python/package.py +++ b/var/spack/repos/builtin/packages/python/package.py @@ -22,9 +22,6 @@ # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import functools -import glob -import inspect import os import re from contextlib import closing @@ -37,11 +34,10 @@ class Python(Package): """The Python programming language.""" + homepage = "http://www.python.org" url = "http://www.python.org/ftp/python/2.7.8/Python-2.7.8.tgz" - extendable = True - version('3.5.1', 'be78e48cdfc1a7ad90efff146dce6cfe') version('3.5.0', 'a56c0c0b45d75a0ec9c6dee933c41c36') version('2.7.11', '6b6076ec9e93f05dd63e47eb9c15728b', preferred=True) @@ -49,6 +45,8 @@ class Python(Package): version('2.7.9', '5eebcaa0030dc4061156d3429657fb83') version('2.7.8', 'd4bca0159acb0b44a781292b5231936f') + extendable = True + depends_on("openssl") depends_on("bzip2") depends_on("readline") @@ -64,39 +62,63 @@ def install(self, spec, prefix): # Rest of install is pretty standard except setup.py needs to # be able to read the CPPFLAGS and LDFLAGS as it scans for the # library and headers to build - configure_args= [ - "--prefix=%s" % prefix, - "--with-threads", - "--enable-shared", - "CPPFLAGS=-I%s/include -I%s/include -I%s/include -I%s/include -I%s/include -I%s/include" % ( - spec['openssl'].prefix, spec['bzip2'].prefix, - spec['readline'].prefix, spec['ncurses'].prefix, - spec['sqlite'].prefix, spec['zlib'].prefix), - "LDFLAGS=-L%s/lib -L%s/lib -L%s/lib -L%s/lib -L%s/lib -L%s/lib" % ( - spec['openssl'].prefix, spec['bzip2'].prefix, - spec['readline'].prefix, spec['ncurses'].prefix, - spec['sqlite'].prefix, spec['zlib'].prefix) - ] + cppflags = ' -I'.join([ + spec['openssl'].prefix.include, spec['bzip2'].prefix.include, + spec['readline'].prefix.include, spec['ncurses'].prefix.include, + spec['sqlite'].prefix.include, spec['zlib'].prefix.include + ]) + + ldflags = ' -L'.join([ + spec['openssl'].prefix.lib, spec['bzip2'].prefix.lib, + spec['readline'].prefix.lib, spec['ncurses'].prefix.lib, + spec['sqlite'].prefix.lib, spec['zlib'].prefix.lib + ]) + + config_args = [ + "--prefix={0}".format(prefix), + "--with-threads", + "--enable-shared", + "CPPFLAGS=-I{0}".format(cppflags), + "LDFLAGS=-L{0}".format(ldflags) + ] + if spec.satisfies('@3:'): - configure_args.append('--without-ensurepip') - configure(*configure_args) + config_args.append('--without-ensurepip') + + configure(*config_args) + make() make("install") - # Modify compiler paths in configuration files. This is necessary for - # building site packages outside of spack - filter_file(r'([/s]=?)([\S=]*)/lib/spack/env(/[^\s/]*)?/(\S*)(\s)', - (r'\4\5'), - join_path(prefix.lib, 'python%d.%d' % self.version[:2], '_sysconfigdata.py')) + self.filter_compilers(spec, prefix) - python3_version = '' + def filter_compilers(self, spec, prefix): + """Run after install to tell the configuration files and Makefiles + to use the compilers that Spack built the package with. + + If this isn't done, they'll have CC and CXX set to Spack's generic + cc and c++. We want them to be bound to whatever compiler + they were built with.""" + + kwargs = {'ignore_absent': True, 'backup': False, 'string': True} + + dirname = join_path(prefix.lib, + 'python{0}'.format(self.version.up_to(2))) + + config = 'config' if spec.satisfies('@3:'): - python3_version = '-%d.%dm' % self.version[:2] - makefile_filepath = join_path(prefix.lib, 'python%d.%d' % self.version[:2], 'config%s' % python3_version, 'Makefile') - filter_file(r'([/s]=?)([\S=]*)/lib/spack/env(/[^\s/]*)?/(\S*)(\s)', - (r'\4\5'), - makefile_filepath) + config = 'config-{0}m'.format(self.version.up_to(2)) + files = [ + '_sysconfigdata.py', + join_path(config, 'Makefile') + ] + + for filename in files: + filter_file(env['CC'], self.compiler.cc, + join_path(dirname, filename), **kwargs) + filter_file(env['CXX'], self.compiler.cxx, + join_path(dirname, filename), **kwargs) # ======================================================================== # Set up environment to make install easy for python extensions. @@ -104,57 +126,59 @@ def install(self, spec, prefix): @property def python_lib_dir(self): - return os.path.join('lib', 'python%d.%d' % self.version[:2]) - + return join_path('lib', 'python{0}'.format(self.version.up_to(2))) @property def python_include_dir(self): - return os.path.join('include', 'python%d.%d' % self.version[:2]) - + return join_path('include', 'python{0}'.format(self.version.up_to(2))) @property def site_packages_dir(self): - return os.path.join(self.python_lib_dir, 'site-packages') - + return join_path(self.python_lib_dir, 'site-packages') def setup_dependent_environment(self, spack_env, run_env, extension_spec): - # TODO: do this only for actual extensions. + """Set PYTHONPATH to include site-packages dir for the + extension and any other python extensions it depends on.""" - # Set PYTHONPATH to include site-packages dir for the - # extension and any other python extensions it depends on. python_paths = [] for d in extension_spec.traverse(): if d.package.extends(self.spec): - python_paths.append(os.path.join(d.prefix, self.site_packages_dir)) + python_paths.append(join_path(d.prefix, + self.site_packages_dir)) pythonpath = ':'.join(python_paths) spack_env.set('PYTHONPATH', pythonpath) - # For run time environment set only the path for extension_spec and prepend it to PYTHONPATH + # For run time environment set only the path for + # extension_spec and prepend it to PYTHONPATH if extension_spec.package.extends(self.spec): - run_env.prepend_path('PYTHONPATH', os.path.join(extension_spec.prefix, self.site_packages_dir)) - + run_env.prepend_path('PYTHONPATH', join_path( + extension_spec.prefix, self.site_packages_dir)) def setup_dependent_package(self, module, ext_spec): - """ - Called before python modules' install() methods. + """Called before python modules' install() methods. In most cases, extensions will only need to have one line:: - python('setup.py', 'install', '--prefix=%s' % prefix) - """ + python('setup.py', 'install', '--prefix={0}'.format(prefix))""" + # Python extension builds can have a global python executable function - if self.version >= Version("3.0.0") and self.version < Version("4.0.0"): - module.python = Executable(join_path(self.spec.prefix.bin, 'python3')) + if Version("3.0.0") <= self.version < Version("4.0.0"): + module.python = Executable(join_path(self.spec.prefix.bin, + 'python3')) else: - module.python = Executable(join_path(self.spec.prefix.bin, 'python')) + module.python = Executable(join_path(self.spec.prefix.bin, + 'python')) # Add variables for lib/pythonX.Y and lib/pythonX.Y/site-packages dirs. - module.python_lib_dir = os.path.join(ext_spec.prefix, self.python_lib_dir) - module.python_include_dir = os.path.join(ext_spec.prefix, self.python_include_dir) - module.site_packages_dir = os.path.join(ext_spec.prefix, self.site_packages_dir) + module.python_lib_dir = join_path(ext_spec.prefix, + self.python_lib_dir) + module.python_include_dir = join_path(ext_spec.prefix, + self.python_include_dir) + module.site_packages_dir = join_path(ext_spec.prefix, + self.site_packages_dir) - # Make the site packages directory for extensions, if it does not exist already. + # Make the site packages directory for extensions if ext_spec.package.is_extension: mkdirp(module.site_packages_dir) @@ -170,21 +194,25 @@ def python_ignore(self, ext_pkg, args): patterns = [r'site-packages/easy-install\.pth$'] # Ignore pieces of setuptools installed by other packages. + # Must include directory name or it will remove all site*.py files. if ext_pkg.name != 'py-setuptools': - patterns.append(r'bin/easy_install[^/]*$') - patterns.append(r'site-packages/setuptools\.pth$') - patterns.append(r'site-packages/site[^/]*\.pyc?$') + patterns.extend([ + r'bin/easy_install[^/]*$', + r'site-packages/setuptools[^/]*\.egg$', + r'site-packages/setuptools\.pth$', + r'site-packages/site[^/]*\.pyc?$', + r'site-packages/__pycache__/site[^/]*\.pyc?$' + ]) if ext_pkg.name != 'py-numpy': patterns.append(r'bin/f2py$') return match_predicate(ignore_arg, patterns) - def write_easy_install_pth(self, exts): paths = [] for ext in sorted(exts.values()): - ext_site_packages = os.path.join(ext.prefix, self.site_packages_dir) - easy_pth = "%s/easy-install.pth" % ext_site_packages + ext_site_packages = join_path(ext.prefix, self.site_packages_dir) + easy_pth = join_path(ext_site_packages, "easy-install.pth") if not os.path.isfile(easy_pth): continue @@ -194,15 +222,18 @@ def write_easy_install_pth(self, exts): line = line.rstrip() # Skip lines matching these criteria - if not line: continue - if re.search(r'^(import|#)', line): continue - if (ext.name != 'py-setuptools' and - re.search(r'setuptools.*egg$', line)): continue + if not line: + continue + if re.search(r'^(import|#)', line): + continue + if ((ext.name != 'py-setuptools' and + re.search(r'setuptools.*egg$', line))): + continue paths.append(line) - site_packages = os.path.join(self.prefix, self.site_packages_dir) - main_pth = "%s/easy-install.pth" % site_packages + site_packages = join_path(self.prefix, self.site_packages_dir) + main_pth = join_path(site_packages, "easy-install.pth") if not paths: if os.path.isfile(main_pth): @@ -210,15 +241,22 @@ def write_easy_install_pth(self, exts): else: with closing(open(main_pth, 'w')) as f: - f.write("import sys; sys.__plen = len(sys.path)\n") + f.write(""" +import sys +sys.__plen = len(sys.path) +""") for path in paths: - f.write("%s\n" % path) - f.write("import sys; new=sys.path[sys.__plen:]; del sys.path[sys.__plen:]; " - "p=getattr(sys,'__egginsert',0); sys.path[p:p]=new; sys.__egginsert = p+len(new)\n") - + f.write("{0}\n".format(path)) + f.write(""" +new = sys.path[sys.__plen:] +del sys.path[sys.__plen:] +p = getattr(sys, '__egginsert', 0) +sys.path[p:p] = new +sys.__egginsert = p + len(new) +""") def activate(self, ext_pkg, **args): - ignore=self.python_ignore(ext_pkg, args) + ignore = self.python_ignore(ext_pkg, args) args.update(ignore=ignore) super(Python, self).activate(ext_pkg, **args) @@ -227,12 +265,12 @@ def activate(self, ext_pkg, **args): exts[ext_pkg.name] = ext_pkg.spec self.write_easy_install_pth(exts) - def deactivate(self, ext_pkg, **args): args.update(ignore=self.python_ignore(ext_pkg, args)) super(Python, self).deactivate(ext_pkg, **args) exts = spack.install_layout.extension_map(self.spec) - if ext_pkg.name in exts: # Make deactivate idempotent. + # Make deactivate idempotent + if ext_pkg.name in exts: del exts[ext_pkg.name] self.write_easy_install_pth(exts) From a4ac99877a512249fa7d126f48716d590cfa80a9 Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Fri, 17 Jun 2016 12:00:35 -0700 Subject: [PATCH 431/481] cmd/list: adding description search and glob by default --- lib/spack/spack/cmd/list.py | 37 +++++++++++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/lib/spack/spack/cmd/list.py b/lib/spack/spack/cmd/list.py index 1e3699cee0..5406283b05 100644 --- a/lib/spack/spack/cmd/list.py +++ b/lib/spack/spack/cmd/list.py @@ -28,7 +28,7 @@ from llnl.util.tty.colify import colify import spack -import fnmatch +import fnmatch, re description ="List available spack packages" @@ -39,20 +39,41 @@ def setup_parser(subparser): subparser.add_argument( '-i', '--insensitive', action='store_true', default=False, help='Filtering will be case insensitive.') + subparser.add_argument( + '-s', '--search_description', action='store_true', default=False, + help='Filtering will also search the description for a match.') def list(parser, args): # Start with all package names. - pkgs = spack.repo.all_package_names() + pkgs = set(spack.repo.all_package_names()) # filter if a filter arg was provided if args.filter: - def match(p, f): - if args.insensitive: - p = p.lower() - f = f.lower() - return fnmatch.fnmatchcase(p, f) - pkgs = [p for p in pkgs if any(match(p, f) for f in args.filter)] + filters = [] + for f in args.filter: + if '*' not in f and '?' not in f: + filters.append('*' + f + '*') + else: + filters.append(f) + + res = [re.compile(fnmatch.translate(f), + flags=re.I if args.insensitive else 0) + for f in filters] + + if args.search_description: + def match(p, f): + if f.match(p): + return True + + pkg = spack.repo.get(p) + if pkg.__doc__: + return f.match(pkg.__doc__) + return False + else: + def match(p, f): + return f.match(p) + pkgs = [p for p in pkgs if any(match(p, f) for f in res)] # sort before displaying. sorted_packages = sorted(pkgs, key=lambda s:s.lower()) From 0743ef4d0ba6632992fee3657f0d89200d4e481b Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Fri, 17 Jun 2016 13:32:28 -0700 Subject: [PATCH 432/481] fixing flake8 issues --- lib/spack/spack/cmd/list.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/cmd/list.py b/lib/spack/spack/cmd/list.py index 5406283b05..d1c19b45c6 100644 --- a/lib/spack/spack/cmd/list.py +++ b/lib/spack/spack/cmd/list.py @@ -28,9 +28,11 @@ from llnl.util.tty.colify import colify import spack -import fnmatch, re +import fnmatch +import re + +description = "List available spack packages" -description ="List available spack packages" def setup_parser(subparser): subparser.add_argument( @@ -76,10 +78,10 @@ def match(p, f): pkgs = [p for p in pkgs if any(match(p, f) for f in res)] # sort before displaying. - sorted_packages = sorted(pkgs, key=lambda s:s.lower()) + sorted_packages = sorted(pkgs, key=lambda s: s.lower()) # Print all the package names in columns - indent=0 + indent = 0 if sys.stdout.isatty(): tty.msg("%d packages." % len(sorted_packages)) colify(sorted_packages, indent=indent) From 8770f2a0ea14c7145e87bfd08ca7149ae3838127 Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Fri, 17 Jun 2016 13:42:27 -0700 Subject: [PATCH 433/481] fixing last flake8 issue --- lib/spack/spack/cmd/list.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/lib/spack/spack/cmd/list.py b/lib/spack/spack/cmd/list.py index d1c19b45c6..228ab76780 100644 --- a/lib/spack/spack/cmd/list.py +++ b/lib/spack/spack/cmd/list.py @@ -52,16 +52,14 @@ def list(parser, args): # filter if a filter arg was provided if args.filter: - filters = [] + res = [] for f in args.filter: if '*' not in f and '?' not in f: - filters.append('*' + f + '*') + r = fnmatch.translate('*' + f + '*') else: - filters.append(f) - - res = [re.compile(fnmatch.translate(f), - flags=re.I if args.insensitive else 0) - for f in filters] + r = fnmatch.translate(f) + rc = re.compile(r, flags=re.I if args.insensitive else 0) + res.append(rc) if args.search_description: def match(p, f): From 652a5eb5924415bd7d8c9d568f33f7dab211d998 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Fri, 17 Jun 2016 18:09:03 -0500 Subject: [PATCH 434/481] Allow py-pillow to link to Spack-built libraries --- .../builtin/packages/py-pillow/package.py | 134 +++++++++++------- 1 file changed, 85 insertions(+), 49 deletions(-) diff --git a/var/spack/repos/builtin/packages/py-pillow/package.py b/var/spack/repos/builtin/packages/py-pillow/package.py index cdbc57e0a5..5784b9f181 100644 --- a/var/spack/repos/builtin/packages/py-pillow/package.py +++ b/var/spack/repos/builtin/packages/py-pillow/package.py @@ -23,77 +23,113 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## from spack import * -import os + class PyPillow(Package): - """Pillow is the friendly PIL fork by Alex Clark and Contributors. PIL is the Python Imaging Library by Fredrik Lundh and Contributors. The Python Imaging Library (PIL) adds image processing capabilities to your Python interpreter. This library supports many file formats, and provides powerful image processing and graphics capabilities.""" + """Pillow is a fork of the Python Imaging Library (PIL). It adds image + processing capabilities to your Python interpreter. This library supports + many file formats, and provides powerful image processing and graphics + capabilities.""" - homepage = "https://python-pillow.github.io/" + homepage = "https://python-pillow.org/" url = "https://pypi.python.org/packages/source/P/Pillow/Pillow-3.0.0.tar.gz" + version('3.2.0', '7cfd093c11205d9e2ebe3c51dfcad510') version('3.0.0', 'fc8ac44e93da09678eac7e30c9b7377d') - provides('PIL') # These defaults correspond to Pillow defaults - variant('jpeg', default=True, description='Provide JPEG functionality') - variant('zlib', default=True, description='Access to compressed PNGs') - variant('tiff', default=False, description='Access to TIFF files') + variant('jpeg', default=True, description='Provide JPEG functionality') + variant('zlib', default=True, description='Access to compressed PNGs') + variant('tiff', default=False, description='Access to TIFF files') variant('freetype', default=False, description='Font related services') - variant('tk', default=False, description='Support for tkinter bitmap and photo images') - variant('lcms', default=False, description='Color management') + variant('lcms', default=False, description='Color management') + variant('tk', default=False, description='Support for tkinter bitmap and photo images') + variant('jpeg2000', default=False, description='Provide JPEG 2000 functionality') # Spack does not (yet) support these modes of building - # variant('webp', default=False, description='') - # variant('webpmux', default=False, description='') - # variant('jpeg2000', default=False, description='') + # variant('webp', default=False, description='Provide the WebP format') + # variant('webpmux', default=False, description='WebP metadata, relies on WebP support') + # variant('imagequant', default=False, description='Provide improved color quantization') + provides('PIL') + + # Required dependencies extends('python') - depends_on('binutils') depends_on('py-setuptools') - depends_on('jpeg', when='+jpeg') # BUG: It will use the system libjpeg anyway + # Recommended dependencies + depends_on('jpeg', when='+jpeg') depends_on('zlib', when='+zlib') - depends_on('tiff', when='+tiff') - depends_on('freetype', when='+freetype') - depends_on('lcms', when='+lcms') - depends_on('tcl', when='+tk') - depends_on('tk', when='+tk') - def install(self, spec, prefix): - libpath=[] + # Optional dependencies + depends_on('libtiff', when='+tiff') + depends_on('freetype', when='+freetype') + depends_on('lcms', when='+lcms') + depends_on('tcl', when='+tk') + depends_on('tk', when='+tk') + depends_on('openjpeg', when='+jpeg2000') + + # Spack does not (yet) support these modes of building + # depends_on('webp', when='+webp') + # depends_on('webpmux', when='+webpmux') + # depends_on('imagequant', when='+imagequant') + + def patch(self): + """Patch setup.py to provide lib and include directories + for dependencies.""" + + spec = self.spec + setup = FileFilter('setup.py') if '+jpeg' in spec: - libpath.append(join_path(spec['jpeg'].prefix, 'lib')) + setup.filter('JPEG_ROOT = None', + 'JPEG_ROOT = ("{0}", "{1}")'.format( + spec['jpeg'].prefix.lib, + spec['jpeg'].prefix.include)) if '+zlib' in spec: - libpath.append(join_path(spec['zlib'].prefix, 'lib')) + setup.filter('ZLIB_ROOT = None', + 'ZLIB_ROOT = ("{0}", "{1}")'.format( + spec['zlib'].prefix.lib, + spec['zlib'].prefix.include)) if '+tiff' in spec: - libpath.append(join_path(spec['tiff'].prefix, 'lib')) + setup.filter('TIFF_ROOT = None', + 'TIFF_ROOT = ("{0}", "{1}")'.format( + spec['libtiff'].prefix.lib, + spec['libtiff'].prefix.include)) if '+freetype' in spec: - libpath.append(join_path(spec['freetype'].prefix, 'lib')) + setup.filter('FREETYPE_ROOT = None', + 'FREETYPE_ROOT = ("{0}", "{1}")'.format( + spec['freetype'].prefix.lib, + spec['freetype'].prefix.include)) if '+lcms' in spec: - libpath.append(join_path(spec['lcms'].prefix, 'lib')) + setup.filter('LCMS_ROOT = None', + 'LCMS_ROOT = ("{0}", "{1}")'.format( + spec['lcms'].prefix.lib, + spec['lcms'].prefix.include)) + if '+tk' in spec: + setup.filter('TCL_ROOT = None', + 'TCL_ROOT = ("{0}", "{1}")'.format( + spec['tcl'].prefix.lib, + spec['tcl'].prefix.include)) + if '+jpeg2000' in spec: + setup.filter('JPEG2K_ROOT = None', + 'JPEG2K_ROOT = ("{0}", "{1}")'.format( + spec['openjpeg'].prefix.lib, + spec['openjpeg'].prefix.include)) - # This has not been tested, and likely needs some other treatment. - #if '+tk' in spec: - # libpath.append(join_path(spec['tcl'].prefix, 'lib')) - # libpath.append(join_path(spec['tk'].prefix, 'lib')) + def install(self, spec, prefix): + build_args = [ + '--{0}-jpeg'.format('enable' if '+jpeg' in spec else 'disable'), + '--{0}-zlib'.format('enable' if '+zlib' in spec else 'disable'), + '--{0}-tiff'.format('enable' if '+tiff' in spec else 'disable'), + '--{0}-freetype'.format( + 'enable' if '+freetype' in spec else 'disable'), + '--{0}-lcms'.format('enable' if '+lcms' in spec else 'disable'), + '--{0}-tk'.format('enable' if '+tk' in spec else 'disable'), + '--{0}-tcl'.format('enable' if '+tk' in spec else 'disable'), + '--{0}-jpeg2000'.format( + 'enable' if '+jpeg2000' in spec else 'disable') + ] - # -------- Building - cmd = ['build_ext', - '--%s-jpeg' % ('enable' if '+jpeg' in spec else 'disable'), - '--%s-zlib' % ('enable' if '+zlib' in spec else 'disable'), - '--%s-tiff' % ('enable' if '+tiff' in spec else 'disable'), - '--%s-freetype' % ('enable' if '+freetype' in spec else 'disable'), - '--%s-lcms' % ('enable' if '+lcms' in spec else 'disable'), - '-L'+':'.join(libpath) # NOTE: This does not make it find libjpeg - ] - - #if '+tk' in spec: - # cmd.extend(['--enable-tcl', '--enable-tk']) - #else: - # cmd.extend(['--disable-tcl', '--disable-tk']) - - # --------- Installation - cmd.extend(['install', '--prefix=%s' % prefix]) - - python('setup.py', *cmd) + python('setup.py', 'build_ext', *build_args) + python('setup.py', 'install', '--prefix={0}'.format(prefix)) From c703bfb54d0452418fea95839226fb5bab7e5893 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Sat, 18 Jun 2016 00:48:08 -0700 Subject: [PATCH 435/481] Got rid of init_config flag since we don't need it. Instead we update the cache config if there are any changes --- lib/spack/spack/compilers/__init__.py | 40 +++++++++++++++------------ 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 6de2d87b32..0543ce8af1 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -52,6 +52,7 @@ _imported_compilers_module = 'spack.compilers' _path_instance_vars = ['cc', 'cxx', 'f77', 'fc'] _other_instance_vars = ['modules', 'operating_system'] +_cache_config_file = [] # TODO: customize order in config file if platform.system() == 'Darwin': @@ -82,7 +83,7 @@ def _to_dict(compiler): return {'compiler': d} -def get_compiler_config(scope=None, init_config=True): +def get_compiler_config(scope=None): """Return the compiler configuration for the specified architecture. """ def init_compiler_config(): @@ -97,9 +98,7 @@ def init_compiler_config(): # Update the configuration if there are currently no compilers # configured. Avoid updating automatically if there ARE site # compilers configured but no user ones. -# if (isinstance(arch, basestring) or arch == my_arch) and arch not in config: - empty_config = [] - if not config and init_config: + if not config: if scope is None: # We know no compilers were configured in any scope. init_compiler_config() @@ -112,11 +111,13 @@ def init_compiler_config(): init_compiler_config() config = spack.config.get_config('compilers', scope=scope) return config + elif config: + return config else: - return empty_config + return [] # Return empty list which we will later append to. -def add_compilers_to_config(compilers, scope=None, init_config=True): +def add_compilers_to_config(compilers, scope=None): """Add compilers to the config for the specified architecture. Arguments: @@ -124,10 +125,10 @@ def add_compilers_to_config(compilers, scope=None, init_config=True): - scope: configuration scope to modify. """ compiler_config = get_compiler_config(scope, init_config) - print compiler_config for compiler in compilers: compiler_config.append(_to_dict(compiler)) - + global _cache_config_file + _cache_config_file = compiler_config spack.config.update_config('compilers', compiler_config, scope) @@ -140,32 +141,35 @@ def remove_compiler_from_config(compiler_spec, scope=None): - scope: configuration scope to modify. """ compiler_config = get_compiler_config(scope) - matches = [(a,c) for (a,c) in compiler_config.items() if c['spec'] == compiler_spec] - if len(matches) == 1: - del compiler_config[matches[0][0]] - else: + config_length = len(compiler_config) + + filtered_compiler_config = [comp for comp in compiler_config + if spack.spec.CompilerSpec(comp['compiler']['spec']) != compiler_spec] + # Need a better way for this + global _cache_config_file + _cache_config_file = filtered_compiler_config # Update the cache for changes + if len(filtered_compiler_config) == config_length: # No items removed CompilerSpecInsufficientlySpecificError(compiler_spec) + spack.config.update_config('compilers', filtered_compiler_config, scope) - spack.config.update_config('compilers', compiler_config, scope) -_cache_config_file = {} -def all_compilers_config(scope=None, init_config=True): +def all_compilers_config(scope=None): """Return a set of specs for all the compiler versions currently available to build with. These are instances of CompilerSpec. """ # Get compilers for this architecture. global _cache_config_file #Create a cache of the config file so we don't load all the time. if not _cache_config_file: - _cache_config_file = get_compiler_config(scope, init_config) + _cache_config_file = get_compiler_config(scope) return _cache_config_file else: return _cache_config_file -def all_compilers(scope=None, init_config=True): +def all_compilers(scope=None): # Return compiler specs from the merged config. return [spack.spec.CompilerSpec(s['compiler']['spec']) - for s in all_compilers_config(scope, init_config)] + for s in all_compilers_config(scope)] def default_compiler(): From 9e0c20c794cea3d54fc16ea7470035384985f1c6 Mon Sep 17 00:00:00 2001 From: alalazo Date: Sat, 18 Jun 2016 13:39:08 +0200 Subject: [PATCH 436/481] environment : filter the current environment Previously only the environment obtained after sourcing the file was filtered. This caused the appeareance of spurious unset commands in the list. --- lib/spack/spack/environment.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py index cfba060459..30c6228ca4 100644 --- a/lib/spack/spack/environment.py +++ b/lib/spack/spack/environment.py @@ -297,16 +297,18 @@ def from_sourcing_files(*args, **kwargs): if proc.returncode != 0: raise RuntimeError('sourcing files returned a non-zero exit code') output = ''.join([line for line in proc.stdout]) - # Construct a dictionary with all the variables in the environment + # Construct a dictionary with all the variables in the new environment after_source_env = dict(json.loads(output)) + this_environment = dict(os.environ) - # Filter variables that are due to how we source - after_source_env.pop('SHLVL') - after_source_env.pop('_') - after_source_env.pop('PWD') + # Filter variables that are not related to sourcing a file + to_be_filtered = 'SHLVL', '_', 'PWD', 'OLDPWD' + for d in after_source_env, this_environment: + for name in to_be_filtered: + d.pop(name, None) # Fill the EnvironmentModifications instance - this_environment = dict(os.environ) + # New variables new_variables = set(after_source_env) - set(this_environment) for x in new_variables: From ccb62b46204531fa4ceeadd3b1722b0bb0e30f4f Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Sat, 18 Jun 2016 10:49:39 -0700 Subject: [PATCH 437/481] Getting rid of no longer used init_config arg --- lib/spack/spack/cmd/compiler.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/lib/spack/spack/cmd/compiler.py b/lib/spack/spack/cmd/compiler.py index 3a496492ff..c95045ef85 100644 --- a/lib/spack/spack/cmd/compiler.py +++ b/lib/spack/spack/cmd/compiler.py @@ -77,11 +77,9 @@ def compiler_find(args): paths = get_path('PATH') compilers = [c for c in spack.compilers.find_compilers(*args.add_paths) - if c.spec not in spack.compilers.all_compilers(scope=args.scope, - init_config=False)] + if c.spec not in spack.compilers.all_compilers(scope=args.scope)] if compilers: - spack.compilers.add_compilers_to_config(compilers, scope=args.scope, - init_config=False) + spack.compilers.add_compilers_to_config(compilers, scope=args.scope) n = len(compilers) s = 's' if n > 1 else '' filename = spack.config.get_config_filename(args.scope, 'compilers') @@ -94,7 +92,6 @@ def compiler_find(args): def compiler_remove(args): cspec = CompilerSpec(args.compiler_spec) compilers = spack.compilers.compilers_for_spec(cspec, scope=args.scope) - if not compilers: tty.die("No compilers match spec %s" % cspec) elif not args.all and len(compilers) > 1: From 36275f8e6eec5b7238e17743a3ec9a2aff371941 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Sat, 18 Jun 2016 10:50:55 -0700 Subject: [PATCH 438/481] More clean up of init_config, also no need for a different find_compilers function. --- lib/spack/spack/compilers/__init__.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 0543ce8af1..ae72b743b2 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -124,7 +124,7 @@ def add_compilers_to_config(compilers, scope=None): - compilers: a list of Compiler objects. - scope: configuration scope to modify. """ - compiler_config = get_compiler_config(scope, init_config) + compiler_config = get_compiler_config(scope) for compiler in compilers: compiler_config.append(_to_dict(compiler)) global _cache_config_file @@ -185,12 +185,6 @@ def default_compiler(): def find_compilers(*paths): - """ Call find compilers help and return the list it finds using - the operating system method (PATHS, MODULES) """ - return _find_compilers(*paths) - - -def _find_compilers(*paths): """Return a list of compilers found in the suppied paths. This invokes the find_compilers() method for each operating system associated with the host platform, and appends @@ -203,6 +197,7 @@ def _find_compilers(*paths): compiler_lists.extend(o.find_compilers(*paths)) return compiler_lists + def supported_compilers(): """Return a set of names of compilers supported by Spack. From 15e6b88a8b5f1e7ae1d06b4b4edc1126b4d68420 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Sat, 18 Jun 2016 10:51:38 -0700 Subject: [PATCH 439/481] Adding new tests for compiler command for spack's test suite --- lib/spack/spack/test/cmd/test_compiler_cmd.py | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 lib/spack/spack/test/cmd/test_compiler_cmd.py diff --git a/lib/spack/spack/test/cmd/test_compiler_cmd.py b/lib/spack/spack/test/cmd/test_compiler_cmd.py new file mode 100644 index 0000000000..81bc7aacf2 --- /dev/null +++ b/lib/spack/spack/test/cmd/test_compiler_cmd.py @@ -0,0 +1,34 @@ +import spack.spec +import spack.cmd.compiler +import spack.compilers +from spack.test.mock_packages_test import * + + +class MockArgs(object): + def __init__(self, add_paths=[], scope=None, compiler_spec=None, all=None): + self.add_paths = add_paths + self.scope = scope + self.compiler_spec = compiler_spec + self.all = all + + +class CompilerCmdTest(MockPackagesTest): + """ Test compiler commands for add and remove """ + + def test_compiler_remove(self): + args = MockArgs(all=True, compiler_spec='gcc@4.5.0') + spack.cmd.compiler.compiler_remove(args) + compilers = spack.compilers.all_compilers() + self.assertTrue(spack.spec.CompilerSpec("gcc@4.5.0") not in compilers) + + def test_compiler_add(self): + # Probably not a good a assumption but might try finding local + # compilers + # installed in /usr + compilers = spack.compilers.all_compilers() + s = set(compilers) + args = MockArgs(add_paths=["/usr"]) + spack.cmd.compiler.compiler_find(args) + new_compilers = spack.compilers.all_compilers() + new_compiler = [x for x in new_compilers if x not in s] + self.assertTrue(new_compiler) From 64c83638fff9716477ecbc3d87989b6b8437b05f Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Sat, 18 Jun 2016 11:11:46 -0700 Subject: [PATCH 440/481] Adding test to list of test_names --- lib/spack/spack/test/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py index 97f142e746..fb91f24721 100644 --- a/lib/spack/spack/test/__init__.py +++ b/lib/spack/spack/test/__init__.py @@ -40,7 +40,7 @@ 'cc', 'link_tree', 'spec_yaml', 'optional_deps', 'make_executable', 'configure_guess', 'lock', 'database', 'namespace_trie', 'yaml', 'sbang', 'environment', 'cmd.find', - 'cmd.uninstall', 'cmd.test_install'] + 'cmd.uninstall', 'cmd.test_install', 'cmd.test_compiler_cmd'] def list_tests(): From 1cc04cff27273bdad8962f2edd854f1d18481545 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Sun, 19 Jun 2016 11:59:10 +0200 Subject: [PATCH 441/481] openblas: raise an error without Fortran compiler --- var/spack/repos/builtin/packages/openblas/package.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/var/spack/repos/builtin/packages/openblas/package.py b/var/spack/repos/builtin/packages/openblas/package.py index 22e49daaa7..d09ebd6739 100644 --- a/var/spack/repos/builtin/packages/openblas/package.py +++ b/var/spack/repos/builtin/packages/openblas/package.py @@ -48,6 +48,13 @@ class Openblas(Package): patch('make.patch') def install(self, spec, prefix): + # As of 06/2016 there is no mechanism to specify that packages which + # depends on Blas/Lapack need C or/and Fortran symbols. For now + # require both. + if self.compiler.f77 is None: + raise InstallError('OpenBLAS requires both C and Fortran ', + 'compilers!') + # Configure fails to pick up fortran from FC=/abs/path/to/f77, but # works fine with FC=/abs/path/to/gfortran. # When mixing compilers make sure that From cf4a34c657b05bfbb71ec69d1521f265a156de57 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Sun, 19 Jun 2016 11:59:29 +0200 Subject: [PATCH 442/481] openmpi: raise an error without Fortran compiler --- var/spack/repos/builtin/packages/openmpi/package.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/var/spack/repos/builtin/packages/openmpi/package.py b/var/spack/repos/builtin/packages/openmpi/package.py index 0638628a6c..be3d1342fc 100644 --- a/var/spack/repos/builtin/packages/openmpi/package.py +++ b/var/spack/repos/builtin/packages/openmpi/package.py @@ -121,6 +121,13 @@ def verbs(self): return 'verbs' def install(self, spec, prefix): + # As of 06/2016 there is no mechanism to specify that packages which + # depends on MPI need C or/and Fortran implementation. For now + # require both. + if (self.compiler.f77 is None) or (self.compiler.fc is None): + raise InstallError('OpenMPI requires both C and Fortran ', + 'compilers!') + config_args = ["--prefix=%s" % prefix, "--with-hwloc=%s" % spec['hwloc'].prefix, "--enable-shared", From 973caa3a07a886e1c4fd7bb36363fc738a72da00 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Mon, 20 Jun 2016 11:10:13 -0400 Subject: [PATCH 443/481] modules.yaml: use the right envvar for pkg-config Fixes #1072. --- etc/spack/modules.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/spack/modules.yaml b/etc/spack/modules.yaml index 99be5e7b6d..5f31cdd5c0 100644 --- a/etc/spack/modules.yaml +++ b/etc/spack/modules.yaml @@ -22,8 +22,8 @@ modules: include: - CPATH lib/pkgconfig: - - PKGCONFIG + - PKG_CONFIG_PATH lib64/pkgconfig: - - PKGCONFIG + - PKG_CONFIG_PATH '': - CMAKE_PREFIX_PATH From 83b0c167284545338dffb44e09a9e6057cfb19a6 Mon Sep 17 00:00:00 2001 From: "Robert.French" Date: Mon, 20 Jun 2016 15:07:55 +0000 Subject: [PATCH 444/481] Tmux 2.2 with PEP8 Tmux 2.2 Provide variant to skip utempter Remove utempter variant; 2.2 doesn't need it Flake8ify --- .../repos/builtin/packages/tmux/package.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/var/spack/repos/builtin/packages/tmux/package.py b/var/spack/repos/builtin/packages/tmux/package.py index c46425c0d3..573ee38a79 100644 --- a/var/spack/repos/builtin/packages/tmux/package.py +++ b/var/spack/repos/builtin/packages/tmux/package.py @@ -24,26 +24,33 @@ ############################################################################## from spack import * + class Tmux(Package): """tmux is a terminal multiplexer. What is a terminal multiplexer? It lets - you switch easily between several programs in one terminal, detach them (they - keep running in the background) and reattach them to a different terminal. And - do a lot more. + you switch easily between several programs in one terminal, detach them + (they keep running in the background) and reattach them to a different + terminal. And do a lot more. """ homepage = "http://tmux.github.io" - url = "https://github.com/tmux/tmux/releases/download/2.1/tmux-2.1.tar.gz" + url = "https://github.com/tmux/tmux/releases/download/2.2/tmux-2.2.tar.gz" version('1.9a', 'b07601711f96f1d260b390513b509a2d') version('2.1', '74a2855695bccb51b6e301383ad4818c') + version('2.2', 'bd95ee7205e489c62c616bb7af040099') depends_on('libevent') depends_on('ncurses') def install(self, spec, prefix): + pkg_config_path = ':'.join([ + spec['libevent'].prefix, + spec['ncurses'].prefix + ]) + configure( "--prefix=%s" % prefix, - "PKG_CONFIG_PATH=%s:%s" % (spec['libevent'].prefix, spec['ncurses'].prefix)) + "PKG_CONFIG_PATH=%s" % pkg_config_path) make() make("install") From 84e21703bde307c920e83e6cd4452feaacbaf5f2 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 20 Jun 2016 09:28:24 -0700 Subject: [PATCH 445/481] Test no longer assumes compilers exist in /usr. - makes a fake gcc instead, and tests that. --- lib/spack/spack/test/cmd/test_compiler_cmd.py | 67 ++++++++++++++++--- 1 file changed, 57 insertions(+), 10 deletions(-) diff --git a/lib/spack/spack/test/cmd/test_compiler_cmd.py b/lib/spack/spack/test/cmd/test_compiler_cmd.py index 81bc7aacf2..d89814154b 100644 --- a/lib/spack/spack/test/cmd/test_compiler_cmd.py +++ b/lib/spack/spack/test/cmd/test_compiler_cmd.py @@ -1,8 +1,16 @@ +import os +import shutil +from tempfile import mkdtemp + +from llnl.util.filesystem import set_executable, mkdirp + import spack.spec import spack.cmd.compiler import spack.compilers +from spack.version import Version from spack.test.mock_packages_test import * +test_version = '4.5-spacktest' class MockArgs(object): def __init__(self, add_paths=[], scope=None, compiler_spec=None, all=None): @@ -12,23 +20,62 @@ def __init__(self, add_paths=[], scope=None, compiler_spec=None, all=None): self.all = all +def make_mock_compiler(): + """Make a directory containing a fake, but detectable compiler.""" + mock_compiler_dir = mkdtemp() + bin_dir = os.path.join(mock_compiler_dir, 'bin') + mkdirp(bin_dir) + + gcc_path = os.path.join(bin_dir, 'gcc') + gxx_path = os.path.join(bin_dir, 'g++') + gfortran_path = os.path.join(bin_dir, 'gfortran') + + with open(gcc_path, 'w') as f: + f.write("""\ +#!/bin/sh + +for arg in "$@"; do + if [ "$arg" = -dumpversion ]; then + echo '%s' + fi +done +""" % test_version) + + # Create some mock compilers in the temporary directory + set_executable(gcc_path) + shutil.copy(gcc_path, gxx_path) + shutil.copy(gcc_path, gfortran_path) + + return mock_compiler_dir + + class CompilerCmdTest(MockPackagesTest): """ Test compiler commands for add and remove """ + def test_compiler_remove(self): args = MockArgs(all=True, compiler_spec='gcc@4.5.0') spack.cmd.compiler.compiler_remove(args) compilers = spack.compilers.all_compilers() self.assertTrue(spack.spec.CompilerSpec("gcc@4.5.0") not in compilers) + def test_compiler_add(self): - # Probably not a good a assumption but might try finding local - # compilers - # installed in /usr - compilers = spack.compilers.all_compilers() - s = set(compilers) - args = MockArgs(add_paths=["/usr"]) - spack.cmd.compiler.compiler_find(args) - new_compilers = spack.compilers.all_compilers() - new_compiler = [x for x in new_compilers if x not in s] - self.assertTrue(new_compiler) + # compilers available by default. + old_compilers = set(spack.compilers.all_compilers()) + + # add our new compiler and find again. + compiler_dir = make_mock_compiler() + + try: + args = MockArgs(add_paths=[compiler_dir]) + spack.cmd.compiler.compiler_find(args) + + # ensure new compiler is in there + new_compilers = set(spack.compilers.all_compilers()) + new_compiler = new_compilers - old_compilers + self.assertTrue(new_compiler) + self.assertTrue(new_compiler.pop().version == Version(test_version)) + + finally: + shutil.rmtree(compiler_dir, ignore_errors=True) From 2220784eda80546415e39c783785f2435986a08a Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Thu, 26 May 2016 15:22:17 -0500 Subject: [PATCH 446/481] Add scons support, .zip support, and Cantera package --- lib/spack/spack/build_environment.py | 3 +- lib/spack/spack/cmd/create.py | 160 ++++++++++++------ .../repos/builtin/packages/cantera/package.py | 149 ++++++++++++++++ .../repos/builtin/packages/py-3to2/package.py | 40 +++++ .../builtin/packages/py-unittest2/package.py | 41 +++++ .../packages/py-unittest2py3k/package.py | 42 +++++ .../repos/builtin/packages/serf/package.py | 2 - .../builtin/packages/sundials/package.py | 105 +++++++++++- .../builtin/packages/superlu-mt/package.py | 135 +++++++++++++++ 9 files changed, 615 insertions(+), 62 deletions(-) create mode 100644 var/spack/repos/builtin/packages/cantera/package.py create mode 100644 var/spack/repos/builtin/packages/py-3to2/package.py create mode 100644 var/spack/repos/builtin/packages/py-unittest2/package.py create mode 100644 var/spack/repos/builtin/packages/py-unittest2py3k/package.py create mode 100644 var/spack/repos/builtin/packages/superlu-mt/package.py diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 7c65091d49..c72dc1a4dd 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -290,7 +290,7 @@ def set_module_variables_for_package(pkg, module): """Populate the module scope of install() with some useful functions. This makes things easier for package writers. """ - # number of jobs spack will to build with. + # number of jobs spack will build with. jobs = multiprocessing.cpu_count() if not pkg.parallel: jobs = 1 @@ -303,6 +303,7 @@ def set_module_variables_for_package(pkg, module): # TODO: make these build deps that can be installed if not found. m.make = MakeExecutable('make', jobs) m.gmake = MakeExecutable('gmake', jobs) + m.scons = MakeExecutable('scons', jobs) # easy shortcut to os.environ m.env = os.environ diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index 41bfa741f6..fa7ffb3923 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -1,4 +1,3 @@ -_copyright = """\ ############################################################################## # Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory. @@ -23,10 +22,8 @@ # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -""" import string import os -import hashlib import re from ordereddict_backport import OrderedDict @@ -41,16 +38,37 @@ from spack.spec import Spec from spack.util.naming import * from spack.repository import Repo, RepoError -import spack.util.crypto as crypto from spack.util.executable import which -from spack.stage import Stage description = "Create a new package file from an archive URL" -package_template = string.Template( - _copyright + """ +package_template = string.Template("""\ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## # # This is a template package file for Spack. We've put "FIXME" # next to all the things you'll want to change. Once you've handled @@ -68,8 +86,10 @@ # from spack import * + class ${class_name}(Package): ""\"FIXME: put a proper description of your package here.""\" + # FIXME: add a proper url for your package's homepage here. homepage = "http://www.example.com" url = "${url}" @@ -80,12 +100,10 @@ class ${class_name}(Package): # depends_on("foo") def install(self, spec, prefix): - # FIXME: Modify the configure line to suit your build system here. + # FIXME: Modify the installation instructions here ${configure} - - # FIXME: Add logic to build and install here - make() - make("install") + ${build} + ${install} """) @@ -120,39 +138,78 @@ def setup_parser(subparser): class ConfigureGuesser(object): def __call__(self, stage): - """Try to guess the type of build system used by the project, and return - an appropriate configure line. - """ - autotools = "configure('--prefix=%s' % prefix)" - cmake = "cmake('.', *std_cmake_args)" - python = "python('setup.py', 'install', '--prefix=%s' % prefix)" - r = "R('CMD', 'INSTALL', '--library=%s' % self.module.r_lib_dir, '%s' % self.stage.archive_file)" + """Try to guess the type of build system used by the project. Set the + appropriate default configure, build, and install instructions.""" - config_lines = ((r'/configure$', 'autotools', autotools), - (r'/CMakeLists.txt$', 'cmake', cmake), - (r'/setup.py$', 'python', python), - (r'/NAMESPACE$', 'r', r)) + # Default configure instructions + configureDict = { + 'autotools': "configure('--prefix={0}'.format(prefix))", + 'cmake': "cmake('.', *std_cmake_args)", + 'scons': "", + 'python': "", + 'r': "", + 'unknown': "# FIXME: Unknown build system" + } - # Peek inside the tarball. - tar = which('tar') - output = tar( - "--exclude=*/*/*", "-tf", stage.archive_file, output=str) - lines = output.split("\n") + # Default build instructions + buildDict = { + 'autotools': "make()", + 'cmake': "make()", + 'scons': "scons('prefix={0}'.format(prefix))", + 'python': "", + 'r': "", + 'unknown': "make()", + } - # Set the configure line to the one that matched. - for pattern, bs, cl in config_lines: + # Default install instructions + installDict = { + 'autotools': "make('install')", + 'cmake': "make('install')", + 'scons': "scons('install')", + 'python': "python('setup.py', 'install', " + + "'--prefix={0}'.format(prefix))", + 'r': "R('CMD', 'INSTALL', '--library={0}'.format(" + + "self.module.r_lib_dir), self.stage.archive_file)", + 'unknown': "make('install')", + } + + # A list of clues that give us an idea of the build system a package + # uses. If the regular expression matches a file contained in the + # archive, the corresponding build system is assumed. + clues = [ + (r'/configure$', 'autotools'), + (r'/CMakeLists.txt$', 'cmake'), + (r'/SConstruct$', 'scons'), + (r'/setup.py$', 'python'), + (r'/NAMESPACE$', 'r') + ] + + # Peek inside the compressed file. + output = '' + if stage.archive_file.endswith(('.tar', '.tar.gz', '.tar.bz2', + '.tgz', '.tbz2')): + tar = which('tar') + output = tar('--exclude=*/*/*', '-tf', + stage.archive_file, output=str) + elif stage.archive_file.endswith('.gz'): + gunzip = which('gunzip') + output = gunzip('-l', stage.archive_file, output=str) + elif stage.archive_file.endswith('.zip'): + unzip = which('unzip') + output = unzip('-l', stage.archive_file, output=str) + lines = output.split('\n') + + # Determine the build system based on the files contained + # in the archive. + build_system = 'unknown' + for pattern, bs in clues: if any(re.search(pattern, l) for l in lines): - config_line = cl build_system = bs - break - else: - # None matched -- just put both, with cmake commented out - config_line = "# FIXME: Spack couldn't guess one, so here are some options:\n" - config_line += " # " + autotools + "\n" - config_line += " # " + cmake - build_system = 'unknown' - self.configure = config_line + self.configure = configureDict[build_system] + self.build = buildDict[build_system] + self.install = installDict[build_system] + self.build_system = build_system @@ -168,7 +225,7 @@ def guess_name_and_version(url, args): else: try: name = spack.url.parse_name(url, version) - except spack.url.UndetectableNameError, e: + except spack.url.UndetectableNameError: # Use a user-supplied name if one is present tty.die("Couldn't guess a name for this package. Try running:", "", "spack create --name ") @@ -182,7 +239,8 @@ def guess_name_and_version(url, args): def find_repository(spec, args): # figure out namespace for spec if spec.namespace and args.namespace and spec.namespace != args.namespace: - tty.die("Namespaces '%s' and '%s' do not match." % (spec.namespace, args.namespace)) + tty.die("Namespaces '%s' and '%s' do not match." % (spec.namespace, + args.namespace)) if not spec.namespace and args.namespace: spec.namespace = args.namespace @@ -193,8 +251,8 @@ def find_repository(spec, args): try: repo = Repo(repo_path) if spec.namespace and spec.namespace != repo.namespace: - tty.die("Can't create package with namespace %s in repo with namespace %s" - % (spec.namespace, repo.namespace)) + tty.die("Can't create package with namespace %s in repo with " + "namespace %s" % (spec.namespace, repo.namespace)) except RepoError as e: tty.die(str(e)) else: @@ -214,11 +272,7 @@ def find_repository(spec, args): def fetch_tarballs(url, name, version): """Try to find versions of the supplied archive by scraping the web. - - Prompts the user to select how many to download if many are found. - - - """ + Prompts the user to select how many to download if many are found.""" versions = spack.util.web.find_versions_of_archive(url) rkeys = sorted(versions.keys(), reverse=True) versions = OrderedDict(zip(rkeys, (versions[v] for v in rkeys))) @@ -226,11 +280,11 @@ def fetch_tarballs(url, name, version): archives_to_fetch = 1 if not versions: # If the fetch failed for some reason, revert to what the user provided - versions = { version : url } + versions = {version: url} elif len(versions) > 1: tty.msg("Found %s versions of %s:" % (len(versions), name), *spack.cmd.elide_list( - ["%-10s%s" % (v,u) for v, u in versions.iteritems()])) + ["%-10s%s" % (v, u) for v, u in versions.iteritems()])) print archives_to_fetch = tty.get_number( "Include how many checksums in the package file?", @@ -292,10 +346,12 @@ def create(parser, args): pkg_file.write( package_template.substitute( name=name, - configure=guesser.configure, class_name=mod_to_class(name), url=url, - versions=make_version_calls(ver_hash_tuples))) + versions=make_version_calls(ver_hash_tuples), + configure=guesser.configure, + build=guesser.build, + install=guesser.install)) # If everything checks out, go ahead and edit. spack.editor(pkg_path) diff --git a/var/spack/repos/builtin/packages/cantera/package.py b/var/spack/repos/builtin/packages/cantera/package.py new file mode 100644 index 0000000000..a55f883560 --- /dev/null +++ b/var/spack/repos/builtin/packages/cantera/package.py @@ -0,0 +1,149 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * +import spack +import os + + +class Cantera(Package): + """Cantera is a suite of object-oriented software tools for problems + involving chemical kinetics, thermodynamics, and/or transport processes.""" + + homepage = "http://www.cantera.org/docs/sphinx/html/index.html" + url = "https://github.com/Cantera/cantera/archive/v2.2.1.tar.gz" + + version('2.2.1', '9d1919bdef39ddec54485fc8a741a3aa') + + variant('lapack', default=True, description='Build with external BLAS/LAPACK libraries') + variant('threadsafe', default=True, description='Build threadsafe, requires Boost') + variant('sundials', default=True, description='Build with external Sundials') + variant('python', default=False, description='Build the Cantera Python module') + variant('matlab', default=False, description='Build the Cantera Matlab toolbox') + + # Required dependencies + depends_on('scons') + + # Recommended dependencies + depends_on('blas', when='+lapack') + depends_on('lapack', when='+lapack') + depends_on('boost', when='+threadsafe') + depends_on('sundials', when='+sundials') # must be compiled with -fPIC + + # Python module dependencies + extends('python', when='+python') + depends_on('py-numpy', when='+python') + depends_on('py-scipy', when='+python') + depends_on('py-cython', when='+python') + depends_on('py-3to2', when='+python') + # TODO: these "when" specs don't actually work + #depends_on('py-unittest2', when='+python^python@2.6') + #depends_on('py-unittest2py3k', when='+python^python@3.1') + + # Matlab toolbox dependencies + # TODO: add Matlab package + # TODO: allow packages to extend multiple other packages + #extends('matlab', when='+matlab') + + def install(self, spec, prefix): + # Required options + options = [ + 'prefix={0}'.format(prefix), + 'CC={0}'.format(os.environ['CC']), + 'CXX={0}'.format(os.environ['CXX']), + 'F77={0}'.format(os.environ['F77']), + 'FORTRAN={0}'.format(os.environ['FC']), + 'cc_flags=-fPIC', + # Allow Spack environment variables to propagate through to SCons + 'env_vars=all' + ] + + # BLAS/LAPACK support + if '+lapack' in spec: + options.extend([ + 'blas_lapack_libs=lapack,blas', + 'blas_lapack_dir={0}'.format(spec['lapack'].prefix.lib) + ]) + + # Threadsafe build, requires Boost + if '+threadsafe' in spec: + options.extend([ + 'build_thread_safe=yes', + 'boost_inc_dir={0}'.format(spec['boost'].prefix.include), + 'boost_lib_dir={0}'.format(spec['boost'].prefix.lib), + 'boost_thread_lib=boost_thread-mt' + ]) + else: + options.append('build_thread_safe=no') + + # Sundials support + if '+sundials' in spec: + options.extend([ + 'use_sundials=y', + 'sundials_include={0}'.format(spec['sundials'].prefix.include), + 'sundials_libdir={0}'.format(spec['sundials'].prefix.lib), + 'sundials_license={0}'.format( + join_path(spec['sundials'].prefix, 'LICENSE')) + ]) + else: + options.append('use_sundials=n') + + # Python module + if '+python' in spec: + options.extend([ + 'python_package=full', + 'python_cmd={0}'.format( + join_path(spec['python'].prefix.bin, 'python')), + 'python_array_home={0}'.format(spec['py-numpy'].prefix) + ]) + if spec['python'].satisfies('@3'): + options.extend([ + 'python3_package=y', + 'python3_cmd={0}'.format( + join_path(spec['python'].prefix.bin, 'python')), + 'python3_array_home={0}'.format(spec['py-numpy'].prefix) + ]) + else: + options.append('python3_package=n') + else: + options.append('python_package=none') + options.append('python3_package=n') + + # Matlab toolbox + if '+matlab' in spec: + options.extend([ + 'matlab_toolbox=y', + 'matlab_path={0}'.format(spec['matlab'].prefix) + ]) + else: + options.append('matlab_toolbox=n') + + scons('build', *options) + + if '+python' in spec: + # Tests will always fail if Python dependencies aren't built + #scons('test') # TODO: 3 expected failures, not sure what's wrong + pass + + scons('install') diff --git a/var/spack/repos/builtin/packages/py-3to2/package.py b/var/spack/repos/builtin/packages/py-3to2/package.py new file mode 100644 index 0000000000..577662e583 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-3to2/package.py @@ -0,0 +1,40 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class Py3to2(Package): + """lib3to2 is a set of fixers that are intended to backport code written + for Python version 3.x into Python version 2.x.""" + + homepage = "https://pypi.python.org/pypi/3to2" + url = "https://pypi.python.org/packages/8f/ab/58a363eca982c40e9ee5a7ca439e8ffc5243dde2ae660ba1ffdd4868026b/3to2-1.1.1.zip" + + version('1.1.1', 'cbeed28e350dbdaef86111ace3052824') + + extends('python') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/repos/builtin/packages/py-unittest2/package.py b/var/spack/repos/builtin/packages/py-unittest2/package.py new file mode 100644 index 0000000000..c9b0ad5cdf --- /dev/null +++ b/var/spack/repos/builtin/packages/py-unittest2/package.py @@ -0,0 +1,41 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class PyUnittest2(Package): + """unittest2 is a backport of the new features added to the unittest + testing framework in Python 2.7 and onwards.""" + + homepage = "https://pypi.python.org/pypi/unittest2" + url = "https://pypi.python.org/packages/7f/c4/2b0e2d185d9d60772c10350d9853646832609d2f299a8300ab730f199db4/unittest2-1.1.0.tar.gz" + + version('1.1.0', 'f72dae5d44f091df36b6b513305ea000') + + extends('python') + depends_on('py-setuptools') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/repos/builtin/packages/py-unittest2py3k/package.py b/var/spack/repos/builtin/packages/py-unittest2py3k/package.py new file mode 100644 index 0000000000..b2c7510d94 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-unittest2py3k/package.py @@ -0,0 +1,42 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class PyUnittest2py3k(Package): + """unittest2 is a backport of the new features added to the unittest + testing framework in Python 2.7 and 3.2. This is a Python 3 compatible + version of unittest2.""" + + homepage = "https://pypi.python.org/pypi/unittest2py3k" + url = "https://pypi.python.org/packages/4e/3d/d44421e8d828af1399c1509c196db92e2a58f3764b01a0ee928d7025d1ca/unittest2py3k-0.5.1.tar.gz" + + version('0.5.1', '8824ff92044310d9365f90d892bf0f09') + + extends('python') + depends_on('py-setuptools') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/repos/builtin/packages/serf/package.py b/var/spack/repos/builtin/packages/serf/package.py index 817db68241..074d3d64b2 100644 --- a/var/spack/repos/builtin/packages/serf/package.py +++ b/var/spack/repos/builtin/packages/serf/package.py @@ -41,8 +41,6 @@ class Serf(Package): depends_on('zlib') def install(self, spec, prefix): - scons = which("scons") - options = ['PREFIX=%s' % prefix] options.append('APR=%s' % spec['apr'].prefix) options.append('APU=%s' % spec['apr-util'].prefix) diff --git a/var/spack/repos/builtin/packages/sundials/package.py b/var/spack/repos/builtin/packages/sundials/package.py index ba2340f74c..a31cf5fde4 100644 --- a/var/spack/repos/builtin/packages/sundials/package.py +++ b/var/spack/repos/builtin/packages/sundials/package.py @@ -24,16 +24,107 @@ ############################################################################## from spack import * + class Sundials(Package): - """SUNDIALS (SUite of Nonlinear and DIfferential/ALgebraic equation Solvers)""" + """SUNDIALS (SUite of Nonlinear and DIfferential/ALgebraic equation + Solvers)""" + homepage = "http://computation.llnl.gov/casc/sundials/" - url = "http://computation.llnl.gov/casc/sundials/download/code/sundials-2.5.0.tar.gz" + url = "http://computation.llnl.gov/projects/sundials-suite-nonlinear-differential-algebraic-equation-solvers/download/sundials-2.6.2.tar.gz" - version('2.5.0', 'aba8b56eec600de3109cfb967aa3ba0f') + version('2.6.2', '3deeb0ede9f514184c6bd83ecab77d95') - depends_on("mpi") + variant('mpi', default=True, description='Enable MPI support') + variant('lapack', default=True, description='Build with external BLAS/LAPACK libraries') + variant('klu', default=True, description='Build with SuiteSparse KLU libraries') + variant('superlu', default=True, description='Build with SuperLU_MT libraries') + variant('openmp', default=False, description='Enable OpenMP support') + variant('pthread', default=True, description='Enable POSIX threads support') + + depends_on('mpi', when='+mpi') + depends_on('blas', when='+lapack') + depends_on('lapack', when='+lapack') + depends_on('suite-sparse', when='+klu') + depends_on('superlu-mt+openmp', when='+superlu+openmp') + depends_on('superlu-mt+pthread', when='+superlu+pthread') def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - make() - make("install") + cmake_args = std_cmake_args + cmake_args.extend([ + '-DBUILD_SHARED_LIBS=ON', + '-DCMAKE_C_FLAGS=-fPIC' + ]) + + # MPI support + if '+mpi' in spec: + cmake_args.extend([ + '-DMPI_ENABLE=ON', + '-DMPI_MPICC={0}'.format(spec['mpi'].mpicc), + '-DMPI_MPIF77={0}'.format(spec['mpi'].mpif77) + ]) + else: + cmake_args.append('-DMPI_ENABLE=OFF') + + # Building with LAPACK and BLAS + if '+lapack' in spec: + cmake_args.extend([ + '-DLAPACK_ENABLE=ON', + '-DLAPACK_LIBRARIES={0};{1}'.format( + spec['lapack'].lapack_shared_lib, + spec['blas'].blas_shared_lib + ) + ]) + else: + cmake_args.append('-DLAPACK_ENABLE=OFF') + + # Building with KLU + if '+klu' in spec: + cmake_args.extend([ + '-DKLU_ENABLE=ON', + '-DKLU_INCLUDE_DIR={0}'.format( + spec['suite-sparse'].prefix.include), + '-DKLU_LIBRARY_DIR={0}'.format( + spec['suite-sparse'].prefix.lib) + ]) + else: + cmake_args.append('-DKLU_ENABLE=OFF') + + # Building with SuperLU_MT + if '+superlu' in spec: + cmake_args.extend([ + '-DSUPERLUMT_ENABLE=ON', + '-DSUPERLUMT_INCLUDE_DIR={0}'.format( + spec['superlu-mt'].prefix.include), + '-DSUPERLUMT_LIBRARY_DIR={0}'.format( + spec['superlu-mt'].prefix.lib) + ]) + if '+openmp' in spec: + cmake_args.append('-DSUPERLUMT_THREAD_TYPE=OpenMP') + elif '+pthread' in spec: + cmake_args.append('-DSUPERLUMT_THREAD_TYPE=Pthread') + else: + msg = 'You must choose either +openmp or +pthread when ' + msg += 'building with SuperLU_MT' + raise RuntimeError(msg) + else: + cmake_args.append('-DSUPERLUMT_ENABLE=OFF') + + # OpenMP support + if '+openmp' in spec: + cmake_args.append('-DOPENMP_ENABLE=ON') + else: + cmake_args.append('-DOPENMP_ENABLE=OFF') + + # POSIX threads support + if '+pthread' in spec: + cmake_args.append('-DPTHREAD_ENABLE=ON') + else: + cmake_args.append('-DPTHREAD_ENABLE=OFF') + + with working_dir('build', create=True): + cmake('..', *cmake_args) + + make() + make('install') + + install('LICENSE', prefix) diff --git a/var/spack/repos/builtin/packages/superlu-mt/package.py b/var/spack/repos/builtin/packages/superlu-mt/package.py new file mode 100644 index 0000000000..5eee4bf5d9 --- /dev/null +++ b/var/spack/repos/builtin/packages/superlu-mt/package.py @@ -0,0 +1,135 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * +import spack +import glob +import os + + +class SuperluMt(Package): + """SuperLU is a general purpose library for the direct solution of large, + sparse, nonsymmetric systems of linear equations on high performance + machines. SuperLU_MT is designed for shared memory parallel machines.""" + + homepage = "http://crd-legacy.lbl.gov/~xiaoye/SuperLU/#superlu_mt" + url = "http://crd-legacy.lbl.gov/~xiaoye/SuperLU/superlu_mt_3.1.tar.gz" + + version('3.1', '06ac62f1b4b7d17123fffa0d0c315e91') + + variant('blas', default=True, description='Build with external BLAS library') + + # Must choose one or the other + variant('openmp', default=False, description='Build with OpenMP support') + variant('pthread', default=True, description='Build with POSIX threads support') + + # NOTE: must link with a single-threaded BLAS library + depends_on('blas', when='+blas') + + # Cannot be built in parallel + parallel = False + + def configure(self, spec): + # Validate chosen variants + if '+openmp' in spec and '+pthread' in spec: + msg = 'You cannot choose both +openmp and +pthread' + raise RuntimeError(msg) + if '~openmp' in spec and '~pthread' in spec: + msg = 'You must choose either +openmp or +pthread' + raise RuntimeError(msg) + + # List of configuration options + config = [] + + # The machine (platform) identifier to append to the library names + if '+openmp' in spec: + # OpenMP + config.extend([ + 'PLAT = _OPENMP', + 'TMGLIB = libtmglib.a', + 'MPLIB = {0}'.format(self.compiler.openmp_flag), + 'CFLAGS = {0}'.format(self.compiler.openmp_flag), + 'FFLAGS = {0}'.format(self.compiler.openmp_flag) + ]) + elif '+pthread' in spec: + # POSIX threads + config.extend([ + 'PLAT = _PTHREAD', + 'TMGLIB = libtmglib$(PLAT).a', + 'MPLIB = -lpthread' + ]) + + # The BLAS library + # NOTE: must link with a single-threaded BLAS library + if '+blas' in spec: + config.extend([ + 'BLASDEF = -DUSE_VENDOR_BLAS', + 'BLASLIB = -L{0} -lblas'.format(spec['blas'].prefix.lib) + ]) + else: + config.append('BLASLIB = ../lib/libblas$(PLAT).a') + + # Generic options + config.extend([ + # The name of the libraries to be created/linked to + 'SUPERLULIB = libsuperlu_mt$(PLAT).a', + 'MATHLIB = -lm', + # The archiver and the flag(s) to use when building archives + 'ARCH = ar', + 'ARCHFLAGS = cr', + 'RANLIB = {0}'.format('ranlib' if which('ranlib') else 'echo'), + # Definitions used by CPP + 'PREDEFS = -D_$(PLAT)', + # Compilers and flags + 'CC = {0}'.format(os.environ['CC']), + 'CFLAGS += $(PREDEFS) -D_LONGINT', + 'NOOPTS = -O0', + 'FORTRAN = {0}'.format(os.environ['FC']), + 'LOADER = {0}'.format(os.environ['CC']), + # C preprocessor defs for compilation + 'CDEFS = -DAdd_' + ]) + + # Write configuration options to include file + with open('make.inc', 'w') as inc: + for option in config: + inc.write('{0}\n'.format(option)) + + def install(self, spec, prefix): + # Set up make include file manually + self.configure(spec) + + # BLAS needs to be compiled separately if using internal BLAS library + if '+blas' not in spec: + make('blaslib') + + make() + + # Install manually + install_tree('lib', prefix.lib) + + headers = glob.glob(join_path('SRC', '*.h')) + mkdir(prefix.include) + for h in headers: + install(h, prefix.include) From a21e845ce7e4b7050a3ef28149b349ff0a4ec58e Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Thu, 26 May 2016 16:19:23 -0500 Subject: [PATCH 447/481] Flake8 --- lib/spack/spack/build_environment.py | 30 +++++++++++-------- .../repos/builtin/packages/cantera/package.py | 9 +++--- .../repos/builtin/packages/serf/package.py | 3 +- .../builtin/packages/superlu-mt/package.py | 1 - 4 files changed, 24 insertions(+), 19 deletions(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index c72dc1a4dd..9e06f9f9cc 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -63,7 +63,7 @@ import spack from spack.environment import EnvironmentModifications, validate from spack.util.environment import * -from spack.util.executable import Executable, which +from spack.util.executable import Executable # # This can be set by the user to globally disable parallel builds. @@ -88,7 +88,6 @@ dso_suffix = 'dylib' if sys.platform == 'darwin' else 'so' - class MakeExecutable(Executable): """Special callable executable object for make so the user can specify parallel or not on a per-invocation basis. Using @@ -177,11 +176,13 @@ def set_compiler_environment_variables(pkg, env): flags = pkg.spec.compiler_flags # Set compiler variables used by CMake and autotools - assert all(key in compiler.link_paths for key in ('cc', 'cxx', 'f77', 'fc')) + assert all(key in compiler.link_paths for key in ( + 'cc', 'cxx', 'f77', 'fc')) # Populate an object with the list of environment modifications # and return it - # TODO : add additional kwargs for better diagnostics, like requestor, ttyout, ttyerr, etc. + # TODO : add additional kwargs for better diagnostics, like requestor, + # ttyout, ttyerr, etc. link_dir = spack.build_env_path env.set('CC', join_path(link_dir, compiler.link_paths['cc'])) env.set('CXX', join_path(link_dir, compiler.link_paths['cxx'])) @@ -233,7 +234,8 @@ def set_build_environment_variables(pkg, env): # handled by putting one in the /case-insensitive # directory. Add that to the path too. env_paths = [] - for item in [spack.build_env_path, join_path(spack.build_env_path, pkg.compiler.name)]: + for item in [spack.build_env_path, join_path(spack.build_env_path, + pkg.compiler.name)]: env_paths.append(item) ci = join_path(item, 'case-insensitive') if os.path.isdir(ci): @@ -246,7 +248,8 @@ def set_build_environment_variables(pkg, env): # Prefixes of all of the package's dependencies go in SPACK_DEPENDENCIES dep_prefixes = [d.prefix for d in pkg.spec.traverse(root=False)] env.set_path(SPACK_DEPENDENCIES, dep_prefixes) - env.set_path('CMAKE_PREFIX_PATH', dep_prefixes) # Add dependencies to CMAKE_PREFIX_PATH + # Add dependencies to CMAKE_PREFIX_PATH + env.set_path('CMAKE_PREFIX_PATH', dep_prefixes) # Install prefix env.set(SPACK_PREFIX, pkg.prefix) @@ -262,7 +265,8 @@ def set_build_environment_variables(pkg, env): env.unset('DYLD_LIBRARY_PATH') # Add bin directories from dependencies to the PATH for the build. - bin_dirs = reversed(filter(os.path.isdir, ['%s/bin' % prefix for prefix in dep_prefixes])) + bin_dirs = reversed(filter(os.path.isdir, + ['%s/bin' % prefix for prefix in dep_prefixes])) for item in bin_dirs: env.prepend_path('PATH', item) @@ -326,7 +330,8 @@ def set_module_variables_for_package(pkg, module): # Set up CMake rpath m.std_cmake_args.append('-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=FALSE') - m.std_cmake_args.append('-DCMAKE_INSTALL_RPATH=%s' % ":".join(get_rpaths(pkg))) + m.std_cmake_args.append('-DCMAKE_INSTALL_RPATH=%s' % \ + ":".join(get_rpaths(pkg))) # Put spack compiler paths in module scope. link_dir = spack.build_env_path @@ -373,13 +378,13 @@ def get_rpaths(pkg): def parent_class_modules(cls): - """Get list of super class modules that are all descend from spack.Package""" + """Get list of super class modules that all descend from spack.Package""" if not issubclass(cls, spack.Package) or issubclass(spack.Package, cls): return [] result = [] module = sys.modules.get(cls.__module__) if module: - result = [ module ] + result = [module] for c in cls.__bases__: result.extend(parent_class_modules(c)) return result @@ -411,7 +416,8 @@ def setup_package(pkg): # throwaway environment, but it is kind of dirty. # # TODO: Think about how to avoid this fix and do something cleaner. - for s in pkg.spec.traverse(): s.package.spec = s + for s in pkg.spec.traverse(): + s.package.spec = s set_compiler_environment_variables(pkg, spack_env) set_build_environment_variables(pkg, spack_env) @@ -499,7 +505,7 @@ def child_fun(): # message. Just make the parent exit with an error code. pid, returncode = os.waitpid(pid, 0) if returncode != 0: - raise InstallError("Installation process had nonzero exit code.".format(str(returncode))) + raise InstallError("Installation process had nonzero exit code.") class InstallError(spack.error.SpackError): diff --git a/var/spack/repos/builtin/packages/cantera/package.py b/var/spack/repos/builtin/packages/cantera/package.py index a55f883560..ddaeb2b070 100644 --- a/var/spack/repos/builtin/packages/cantera/package.py +++ b/var/spack/repos/builtin/packages/cantera/package.py @@ -23,7 +23,6 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## from spack import * -import spack import os @@ -58,13 +57,13 @@ class Cantera(Package): depends_on('py-cython', when='+python') depends_on('py-3to2', when='+python') # TODO: these "when" specs don't actually work - #depends_on('py-unittest2', when='+python^python@2.6') - #depends_on('py-unittest2py3k', when='+python^python@3.1') + # depends_on('py-unittest2', when='+python^python@2.6') + # depends_on('py-unittest2py3k', when='+python^python@3.1') # Matlab toolbox dependencies # TODO: add Matlab package # TODO: allow packages to extend multiple other packages - #extends('matlab', when='+matlab') + # extends('matlab', when='+matlab') def install(self, spec, prefix): # Required options @@ -143,7 +142,7 @@ def install(self, spec, prefix): if '+python' in spec: # Tests will always fail if Python dependencies aren't built - #scons('test') # TODO: 3 expected failures, not sure what's wrong + # scons('test') # TODO: 3 expected failures, not sure what's wrong pass scons('install') diff --git a/var/spack/repos/builtin/packages/serf/package.py b/var/spack/repos/builtin/packages/serf/package.py index 074d3d64b2..ff6fd2da9b 100644 --- a/var/spack/repos/builtin/packages/serf/package.py +++ b/var/spack/repos/builtin/packages/serf/package.py @@ -28,10 +28,11 @@ class Serf(Package): """Apache Serf - a high performance C-based HTTP client library built upon the Apache Portable Runtime (APR) library""" + homepage = 'https://serf.apache.org/' url = 'https://archive.apache.org/dist/serf/serf-1.3.8.tar.bz2' - version('1.3.8', '1d45425ca324336ce2f4ae7d7b4cfbc5567c5446') + version('1.3.8', '1d45425ca324336ce2f4ae7d7b4cfbc5567c5446') depends_on('apr') depends_on('apr-util') diff --git a/var/spack/repos/builtin/packages/superlu-mt/package.py b/var/spack/repos/builtin/packages/superlu-mt/package.py index 5eee4bf5d9..5a9429d6e5 100644 --- a/var/spack/repos/builtin/packages/superlu-mt/package.py +++ b/var/spack/repos/builtin/packages/superlu-mt/package.py @@ -23,7 +23,6 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## from spack import * -import spack import glob import os From 9500f2718b7ab9d430dbff2fc10bca1179c61b7b Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Thu, 26 May 2016 16:33:35 -0500 Subject: [PATCH 448/481] More Flake8 --- lib/spack/spack/build_environment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 9e06f9f9cc..901ba1608d 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -330,7 +330,7 @@ def set_module_variables_for_package(pkg, module): # Set up CMake rpath m.std_cmake_args.append('-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=FALSE') - m.std_cmake_args.append('-DCMAKE_INSTALL_RPATH=%s' % \ + m.std_cmake_args.append('-DCMAKE_INSTALL_RPATH=%s' % ":".join(get_rpaths(pkg))) # Put spack compiler paths in module scope. From 81ac3b62fc4d21a677bea654402f3684b7edbc20 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Fri, 27 May 2016 12:39:42 -0500 Subject: [PATCH 449/481] Filter compilers and link boost properly --- .../repos/builtin/packages/cantera/package.py | 54 +++++++++++++++++-- 1 file changed, 51 insertions(+), 3 deletions(-) diff --git a/var/spack/repos/builtin/packages/cantera/package.py b/var/spack/repos/builtin/packages/cantera/package.py index ddaeb2b070..bc8c132ffc 100644 --- a/var/spack/repos/builtin/packages/cantera/package.py +++ b/var/spack/repos/builtin/packages/cantera/package.py @@ -91,7 +91,7 @@ def install(self, spec, prefix): 'build_thread_safe=yes', 'boost_inc_dir={0}'.format(spec['boost'].prefix.include), 'boost_lib_dir={0}'.format(spec['boost'].prefix.lib), - 'boost_thread_lib=boost_thread-mt' + 'boost_thread_lib=boost_thread-mt,boost_system-mt' ]) else: options.append('build_thread_safe=no') @@ -142,7 +142,55 @@ def install(self, spec, prefix): if '+python' in spec: # Tests will always fail if Python dependencies aren't built - # scons('test') # TODO: 3 expected failures, not sure what's wrong - pass + scons('test', parallel=False) scons('install') + + self.filter_compilers() + + def filter_compilers(self): + """Run after install to tell the Makefile and SConstruct files to use + the compilers that Spack built the package with. + + If this isn't done, they'll have CC, CXX, F77, and FC set to Spack's + generic cc, c++, f77, and f90. We want them to be bound to whatever + compiler they were built with.""" + + kwargs = {'ignore_absent': True, 'backup': False, 'string': True} + dirname = os.path.join(self.prefix, 'share/cantera/samples') + + cc_files = [ + 'cxx/rankine/Makefile', 'cxx/NASA_coeffs/Makefile', + 'cxx/kinetics1/Makefile', 'cxx/flamespeed/Makefile', + 'cxx/combustor/Makefile', 'f77/SConstruct' + ] + + cxx_files = [ + 'cxx/rankine/Makefile', 'cxx/NASA_coeffs/Makefile', + 'cxx/kinetics1/Makefile', 'cxx/flamespeed/Makefile', + 'cxx/combustor/Makefile' + ] + + f77_files = [ + 'f77/Makefile', 'f77/SConstruct' + ] + + fc_files = [ + 'f90/Makefile', 'f90/SConstruct' + ] + + for filename in cc_files: + filter_file(os.environ['CC'], self.compiler.cc, + os.path.join(dirname, filename), **kwargs) + + for filename in cxx_files: + filter_file(os.environ['CXX'], self.compiler.cxx, + os.path.join(dirname, filename), **kwargs) + + for filename in f77_files: + filter_file(os.environ['F77'], self.compiler.f77, + os.path.join(dirname, filename), **kwargs) + + for filename in fc_files: + filter_file(os.environ['FC'], self.compiler.fc, + os.path.join(dirname, filename), **kwargs) From 79fae306f6077f637875f2b7df393ae9be95ca9b Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Fri, 3 Jun 2016 10:50:31 -0500 Subject: [PATCH 450/481] Add extensions for Python/R and more configurable install --- lib/spack/spack/cmd/create.py | 102 +++++++++++++++++++--------------- 1 file changed, 56 insertions(+), 46 deletions(-) diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index fa7ffb3923..62e7f97f2a 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -88,22 +88,19 @@ class ${class_name}(Package): - ""\"FIXME: put a proper description of your package here.""\" + ""\"FIXME: Put a proper description of your package here.""\" - # FIXME: add a proper url for your package's homepage here. + # FIXME: Add a proper url for your package's homepage here. homepage = "http://www.example.com" url = "${url}" ${versions} - +${extends} # FIXME: Add dependencies if this package requires them. # depends_on("foo") def install(self, spec, prefix): - # FIXME: Modify the installation instructions here - ${configure} - ${build} - ${install} +${install} """) @@ -138,39 +135,47 @@ def setup_parser(subparser): class ConfigureGuesser(object): def __call__(self, stage): - """Try to guess the type of build system used by the project. Set the - appropriate default configure, build, and install instructions.""" + """Try to guess the type of build system used by the project. + Set the appropriate default installation instructions and any + necessary extensions for Python and R.""" - # Default configure instructions - configureDict = { - 'autotools': "configure('--prefix={0}'.format(prefix))", - 'cmake': "cmake('.', *std_cmake_args)", - 'scons': "", - 'python': "", - 'r': "", - 'unknown': "# FIXME: Unknown build system" - } - - # Default build instructions - buildDict = { - 'autotools': "make()", - 'cmake': "make()", - 'scons': "scons('prefix={0}'.format(prefix))", - 'python': "", - 'r': "", - 'unknown': "make()", - } - - # Default install instructions + # Default installation instructions installDict = { - 'autotools': "make('install')", - 'cmake': "make('install')", - 'scons': "scons('install')", - 'python': "python('setup.py', 'install', " + - "'--prefix={0}'.format(prefix))", - 'r': "R('CMD', 'INSTALL', '--library={0}'.format(" + - "self.module.r_lib_dir), self.stage.archive_file)", - 'unknown': "make('install')", + 'autotools': """\ + # FIXME: Modify the configure line to suit your build system here. + configure('--prefix={0}'.format(prefix)) + + # FIXME: Add logic to build and install here. + make() + make('install')""", + + 'cmake': """\ + with working_dir('spack-build', create=True): + # FIXME: Modify the cmake line to suit your build system here. + cmake('..', *std_cmake_args) + + # FIXME: Add logic to build and install here. + make() + make('install')""", + + 'scons': """\ + # FIXME: Add logic to build and install here. + scons('prefix={0}'.format(prefix)) + scons('install')""", + + 'python': """\ + # FIXME: Add logic to build and install here. + python('setup.py', 'install', '--prefix={0}'.format(prefix))""", + + 'R': """\ + # FIXME: Add logic to build and install here. + R('CMD', 'INSTALL', '--library={0}'.format(self.module.r_lib_dir),""" + + " self.stage.archive_file)""", + + 'unknown': """\ + # FIXME: Unknown build system + make() + make('install')""" } # A list of clues that give us an idea of the build system a package @@ -181,7 +186,7 @@ def __call__(self, stage): (r'/CMakeLists.txt$', 'cmake'), (r'/SConstruct$', 'scons'), (r'/setup.py$', 'python'), - (r'/NAMESPACE$', 'r') + (r'/NAMESPACE$', 'R') ] # Peek inside the compressed file. @@ -206,12 +211,18 @@ def __call__(self, stage): if any(re.search(pattern, l) for l in lines): build_system = bs - self.configure = configureDict[build_system] - self.build = buildDict[build_system] - self.install = installDict[build_system] - self.build_system = build_system + # Set any necessary extensions for Python and R + extensions = '' + if build_system in ['python', 'R']: + extensions = "\n extends('{0}')\n".format(build_system) + + self.extends = extensions + + # Set the appropriate default installation instructions + self.install = installDict[build_system] + def guess_name_and_version(url, args): # Try to deduce name and version of the new package from the URL @@ -331,7 +342,7 @@ def create(parser, args): name = 'py-%s' % name # Prepend 'r-' to R package names, by convention. - if guesser.build_system == 'r': + if guesser.build_system == 'R': name = 'r-%s' % name # Create a directory for the new package. @@ -349,8 +360,7 @@ def create(parser, args): class_name=mod_to_class(name), url=url, versions=make_version_calls(ver_hash_tuples), - configure=guesser.configure, - build=guesser.build, + extends=guesser.extends, install=guesser.install)) # If everything checks out, go ahead and edit. From 5a55bb3f8de9707de43512c5f2ae41072baeb630 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Fri, 3 Jun 2016 14:22:56 -0500 Subject: [PATCH 451/481] Modify R installation template --- lib/spack/spack/cmd/create.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index 62e7f97f2a..c976b98b17 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -170,7 +170,7 @@ def __call__(self, stage): 'R': """\ # FIXME: Add logic to build and install here. R('CMD', 'INSTALL', '--library={0}'.format(self.module.r_lib_dir),""" + - " self.stage.archive_file)""", + " self.stage.source_path)""", 'unknown': """\ # FIXME: Unknown build system From 7e1ee463ca677e3209c800a6c3b111dd8851b78d Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Mon, 6 Jun 2016 13:25:37 -0500 Subject: [PATCH 452/481] Install examples for sundials --- .../repos/builtin/packages/cantera/package.py | 1 + .../builtin/packages/sundials/package.py | 53 +++++++++++++++++-- 2 files changed, 49 insertions(+), 5 deletions(-) diff --git a/var/spack/repos/builtin/packages/cantera/package.py b/var/spack/repos/builtin/packages/cantera/package.py index bc8c132ffc..e9e5da4486 100644 --- a/var/spack/repos/builtin/packages/cantera/package.py +++ b/var/spack/repos/builtin/packages/cantera/package.py @@ -142,6 +142,7 @@ def install(self, spec, prefix): if '+python' in spec: # Tests will always fail if Python dependencies aren't built + # In addition, 3 of the tests fail when run in parallel scons('test', parallel=False) scons('install') diff --git a/var/spack/repos/builtin/packages/sundials/package.py b/var/spack/repos/builtin/packages/sundials/package.py index a31cf5fde4..c55198a850 100644 --- a/var/spack/repos/builtin/packages/sundials/package.py +++ b/var/spack/repos/builtin/packages/sundials/package.py @@ -23,6 +23,7 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## from spack import * +import os class Sundials(Package): @@ -30,14 +31,14 @@ class Sundials(Package): Solvers)""" homepage = "http://computation.llnl.gov/casc/sundials/" - url = "http://computation.llnl.gov/projects/sundials-suite-nonlinear-differential-algebraic-equation-solvers/download/sundials-2.6.2.tar.gz" + url = "http://computation.llnl.gov/projects/sundials-suite-nonlinear-differential-algebraic-equation-solvers/download/sundials-2.6.2.tar.gz" version('2.6.2', '3deeb0ede9f514184c6bd83ecab77d95') variant('mpi', default=True, description='Enable MPI support') variant('lapack', default=True, description='Build with external BLAS/LAPACK libraries') - variant('klu', default=True, description='Build with SuiteSparse KLU libraries') - variant('superlu', default=True, description='Build with SuperLU_MT libraries') + variant('klu', default=False, description='Build with SuiteSparse KLU libraries') + variant('superlu', default=False, description='Build with SuperLU_MT libraries') variant('openmp', default=False, description='Enable OpenMP support') variant('pthread', default=True, description='Enable POSIX threads support') @@ -49,10 +50,14 @@ class Sundials(Package): depends_on('superlu-mt+pthread', when='+superlu+pthread') def install(self, spec, prefix): - cmake_args = std_cmake_args + cmake_args = std_cmake_args[:] cmake_args.extend([ '-DBUILD_SHARED_LIBS=ON', - '-DCMAKE_C_FLAGS=-fPIC' + '-DCMAKE_C_FLAGS=-fPIC', + '-DCMAKE_Fortran_FLAGS=-fPIC', + '-DEXAMPLES_ENABLE=ON', + '-DEXAMPLES_INSTALL=ON', + '-DFCMIX_ENABLE=ON' ]) # MPI support @@ -128,3 +133,41 @@ def install(self, spec, prefix): make('install') install('LICENSE', prefix) + + self.filter_compilers() + + def filter_compilers(self): + """Run after install to tell the Makefiles to use + the compilers that Spack built the package with. + + If this isn't done, they'll have CC, CPP, and F77 set to + Spack's generic cc and f77. We want them to be bound to + whatever compiler they were built with.""" + + kwargs = {'ignore_absent': True, 'backup': False, 'string': True} + dirname = os.path.join(self.prefix, 'examples') + + cc_files = [ + 'arkode/C_serial/Makefile', 'arkode/C_parallel/Makefile', + 'cvode/serial/Makefile', 'cvode/parallel/Makefile', + 'cvodes/serial/Makefile', 'cvodes/parallel/Makefile', + 'ida/serial/Makefile', 'ida/parallel/Makefile', + 'idas/serial/Makefile', 'idas/parallel/Makefile', + 'kinsol/serial/Makefile', 'kinsol/parallel/Makefile', + 'nvector/serial/Makefile', 'nvector/parallel/Makefile', + 'nvector/pthreads/Makefile' + ] + + f77_files = [ + 'arkode/F77_serial/Makefile', 'cvode/fcmix_serial/Makefile', + 'ida/fcmix_serial/Makefile', 'ida/fcmix_pthreads/Makefile', + 'kinsol/fcmix_serial/Makefile' + ] + + for filename in cc_files: + filter_file(os.environ['CC'], self.compiler.cc, + os.path.join(dirname, filename), **kwargs) + + for filename in f77_files: + filter_file(os.environ['F77'], self.compiler.f77, + os.path.join(dirname, filename), **kwargs) From 98d03c74e114137537ee1865244c389d687aa5d6 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Mon, 6 Jun 2016 15:31:04 -0500 Subject: [PATCH 453/481] Add support for less common compression schemes --- lib/spack/spack/cmd/create.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index c976b98b17..dc927757fb 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -190,18 +190,19 @@ def __call__(self, stage): ] # Peek inside the compressed file. - output = '' - if stage.archive_file.endswith(('.tar', '.tar.gz', '.tar.bz2', - '.tgz', '.tbz2')): - tar = which('tar') - output = tar('--exclude=*/*/*', '-tf', + if stage.archive_file.endswith('.zip'): + try: + unzip = which('unzip') + output = unzip('-l', stage.archive_file, output=str) + except: + output = '' + else: + try: + tar = which('tar') + output = tar('--exclude=*/*/*', '-tf', stage.archive_file, output=str) - elif stage.archive_file.endswith('.gz'): - gunzip = which('gunzip') - output = gunzip('-l', stage.archive_file, output=str) - elif stage.archive_file.endswith('.zip'): - unzip = which('unzip') - output = unzip('-l', stage.archive_file, output=str) + except: + output = '' lines = output.split('\n') # Determine the build system based on the files contained From 5dfc2052bdb2f255a02e747eae9874c6645176fb Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Mon, 6 Jun 2016 15:42:30 -0500 Subject: [PATCH 454/481] Flake8 change --- lib/spack/spack/cmd/create.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index dc927757fb..bc835668c2 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -200,7 +200,7 @@ def __call__(self, stage): try: tar = which('tar') output = tar('--exclude=*/*/*', '-tf', - stage.archive_file, output=str) + stage.archive_file, output=str) except: output = '' lines = output.split('\n') From c9eb5f81735dbb0c79e5d1294c410399cdeeb01a Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Fri, 10 Jun 2016 14:48:29 -0500 Subject: [PATCH 455/481] Use non-checksummed PyPi download URLs --- var/spack/repos/builtin/packages/py-3to2/package.py | 2 +- var/spack/repos/builtin/packages/py-unittest2/package.py | 2 +- var/spack/repos/builtin/packages/py-unittest2py3k/package.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/var/spack/repos/builtin/packages/py-3to2/package.py b/var/spack/repos/builtin/packages/py-3to2/package.py index 577662e583..1071a3c209 100644 --- a/var/spack/repos/builtin/packages/py-3to2/package.py +++ b/var/spack/repos/builtin/packages/py-3to2/package.py @@ -30,7 +30,7 @@ class Py3to2(Package): for Python version 3.x into Python version 2.x.""" homepage = "https://pypi.python.org/pypi/3to2" - url = "https://pypi.python.org/packages/8f/ab/58a363eca982c40e9ee5a7ca439e8ffc5243dde2ae660ba1ffdd4868026b/3to2-1.1.1.zip" + url = "https://pypi.python.org/packages/source/3/3to2/3to2-1.1.1.zip" version('1.1.1', 'cbeed28e350dbdaef86111ace3052824') diff --git a/var/spack/repos/builtin/packages/py-unittest2/package.py b/var/spack/repos/builtin/packages/py-unittest2/package.py index c9b0ad5cdf..f669a500ec 100644 --- a/var/spack/repos/builtin/packages/py-unittest2/package.py +++ b/var/spack/repos/builtin/packages/py-unittest2/package.py @@ -30,7 +30,7 @@ class PyUnittest2(Package): testing framework in Python 2.7 and onwards.""" homepage = "https://pypi.python.org/pypi/unittest2" - url = "https://pypi.python.org/packages/7f/c4/2b0e2d185d9d60772c10350d9853646832609d2f299a8300ab730f199db4/unittest2-1.1.0.tar.gz" + url = "https://pypi.python.org/packages/source/u/unittest2/unittest2-1.1.0.tar.gz" version('1.1.0', 'f72dae5d44f091df36b6b513305ea000') diff --git a/var/spack/repos/builtin/packages/py-unittest2py3k/package.py b/var/spack/repos/builtin/packages/py-unittest2py3k/package.py index b2c7510d94..ca857395fb 100644 --- a/var/spack/repos/builtin/packages/py-unittest2py3k/package.py +++ b/var/spack/repos/builtin/packages/py-unittest2py3k/package.py @@ -31,7 +31,7 @@ class PyUnittest2py3k(Package): version of unittest2.""" homepage = "https://pypi.python.org/pypi/unittest2py3k" - url = "https://pypi.python.org/packages/4e/3d/d44421e8d828af1399c1509c196db92e2a58f3764b01a0ee928d7025d1ca/unittest2py3k-0.5.1.tar.gz" + url = "https://pypi.python.org/packages/source/u/unittest2py3k/unittest2py3k-0.5.1.tar.gz" version('0.5.1', '8824ff92044310d9365f90d892bf0f09') From afff40e584364a0707ee1bee54b8f72b0335c6e5 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Wed, 15 Jun 2016 10:50:43 -0500 Subject: [PATCH 456/481] Flake8 fix for R templates --- lib/spack/spack/cmd/create.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index bc835668c2..2d3cfbb9ca 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -169,8 +169,8 @@ def __call__(self, stage): 'R': """\ # FIXME: Add logic to build and install here. - R('CMD', 'INSTALL', '--library={0}'.format(self.module.r_lib_dir),""" + - " self.stage.source_path)""", + R('CMD', 'INSTALL', '--library={0}'.format(self.module.r_lib_dir), + self.stage.source_path)""", 'unknown': """\ # FIXME: Unknown build system From 28b2e36230625a26868122b15c18253192b9ba1d Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Wed, 15 Jun 2016 10:54:08 -0500 Subject: [PATCH 457/481] Add ctest executable --- lib/spack/spack/build_environment.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 901ba1608d..66faee1409 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -321,6 +321,7 @@ def set_module_variables_for_package(pkg, module): # TODO: Currently, everything is a link dependency, but tools like # TODO: this shouldn't be. m.cmake = Executable('cmake') + m.ctest = Executable('ctest') # standard CMake arguments m.std_cmake_args = ['-DCMAKE_INSTALL_PREFIX=%s' % pkg.prefix, From be407f531e0589dca82a6d5c7c52a59b5e0a8a3b Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Wed, 15 Jun 2016 11:01:18 -0500 Subject: [PATCH 458/481] Move around extension logic --- lib/spack/spack/cmd/create.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index 2d3cfbb9ca..8cbb367f86 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -214,6 +214,9 @@ def __call__(self, stage): self.build_system = build_system + # Set the appropriate default installation instructions + self.install = installDict[build_system] + # Set any necessary extensions for Python and R extensions = '' if build_system in ['python', 'R']: @@ -221,9 +224,6 @@ def __call__(self, stage): self.extends = extensions - # Set the appropriate default installation instructions - self.install = installDict[build_system] - def guess_name_and_version(url, args): # Try to deduce name and version of the new package from the URL From aa86488fd9809ced16704e4bd4d607c89d6dda75 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Mon, 20 Jun 2016 12:47:17 -0500 Subject: [PATCH 459/481] Flake8 --- lib/spack/spack/build_environment.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 66faee1409..c5efa97c7b 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -112,12 +112,13 @@ def __call__(self, *args, **kwargs): return super(MakeExecutable, self).__call__(*args, **kwargs) + def load_module(mod): """Takes a module name and removes modules until it is possible to load that module. It then loads the provided module. Depends on the modulecmd implementation of modules used in cray and lmod. """ - #Create an executable of the module command that will output python code + # Create an executable of the module command that will output python code modulecmd = which('modulecmd') modulecmd.add_default_arg('python') @@ -128,11 +129,13 @@ def load_module(mod): text = modulecmd('show', mod, output=str, error=str).split() for i, word in enumerate(text): if word == 'conflict': - exec(compile(modulecmd('unload', text[i+1], output=str, error=str), '', 'exec')) + exec(compile(modulecmd('unload', text[i + 1], output=str, + error=str), '', 'exec')) # Load the module now that there are no conflicts load = modulecmd('load', mod, output=str, error=str) exec(compile(load, '', 'exec')) + def get_path_from_module(mod): """Inspects a TCL module for entries that indicate the absolute path at which the library supported by said module can be found. @@ -145,7 +148,7 @@ def get_path_from_module(mod): text = modulecmd('show', mod, output=str, error=str).split('\n') # If it lists its package directory, return that for line in text: - if line.find(mod.upper()+'_DIR') >= 0: + if line.find(mod.upper() + '_DIR') >= 0: words = line.split() return words[2] @@ -153,23 +156,24 @@ def get_path_from_module(mod): for line in text: rpath = line.find('-rpath/') if rpath >= 0: - return line[rpath+6:line.find('/lib')] + return line[rpath + 6:line.find('/lib')] # If it lists a -L instruction, use that for line in text: L = line.find('-L/') if L >= 0: - return line[L+2:line.find('/lib')] + return line[L + 2:line.find('/lib')] # If it sets the LD_LIBRARY_PATH or CRAY_LD_LIBRARY_PATH, use that for line in text: - if line.find('LD_LIBRARY_PATH') >= 0: + if line.find('LD_LIBRARY_PATH') >= 0: words = line.split() path = words[2] return path[:path.find('/lib')] # Unable to find module path return None + def set_compiler_environment_variables(pkg, env): assert(pkg.spec.concrete) compiler = pkg.compiler @@ -281,8 +285,7 @@ def set_build_environment_variables(pkg, env): for directory in ('lib', 'lib64', 'share'): pcdir = join_path(pre, directory, 'pkgconfig') if os.path.isdir(pcdir): - #pkg_config_dirs.append(pcdir) - env.prepend_path('PKG_CONFIG_PATH',pcdir) + env.prepend_path('PKG_CONFIG_PATH', pcdir) if pkg.spec.architecture.target.module_name: load_module(pkg.spec.architecture.target.module_name) @@ -372,7 +375,7 @@ def get_rpaths(pkg): rpaths.extend(d.prefix.lib64 for d in pkg.spec.dependencies.values() if os.path.isdir(d.prefix.lib64)) # Second module is our compiler mod name. We use that to get rpaths from - # module show output. + # module show output. if pkg.compiler.modules and len(pkg.compiler.modules) > 1: rpaths.append(get_path_from_module(pkg.compiler.modules[1])) return rpaths @@ -397,7 +400,8 @@ def load_external_modules(pkg): for dep in list(pkg.spec.traverse()): if dep.external_module: load_module(dep.external_module) - + + def setup_package(pkg): """Execute all environment setup routines.""" spack_env = EnvironmentModifications() From 0377f3580b0985bd65fb64e82a6fbb9b397b56ab Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Mon, 20 Jun 2016 10:53:19 -0700 Subject: [PATCH 460/481] switching argument naming --- lib/spack/spack/cmd/list.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/cmd/list.py b/lib/spack/spack/cmd/list.py index 228ab76780..9ce63fa1a4 100644 --- a/lib/spack/spack/cmd/list.py +++ b/lib/spack/spack/cmd/list.py @@ -42,7 +42,7 @@ def setup_parser(subparser): '-i', '--insensitive', action='store_true', default=False, help='Filtering will be case insensitive.') subparser.add_argument( - '-s', '--search_description', action='store_true', default=False, + '-d', '--search-description', action='store_true', default=False, help='Filtering will also search the description for a match.') From fe301135021a9f64069a9bf2e3a074c8d4b712ef Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 20 Jun 2016 10:57:10 -0700 Subject: [PATCH 461/481] Adding init_config back. Don't initalize config rather let compiler_find do all the work. spack compiler list and spack compilers can do the work --- lib/spack/spack/cmd/compiler.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/cmd/compiler.py b/lib/spack/spack/cmd/compiler.py index c95045ef85..ff26bc6ba2 100644 --- a/lib/spack/spack/cmd/compiler.py +++ b/lib/spack/spack/cmd/compiler.py @@ -76,8 +76,13 @@ def compiler_find(args): if not paths: paths = get_path('PATH') - compilers = [c for c in spack.compilers.find_compilers(*args.add_paths) - if c.spec not in spack.compilers.all_compilers(scope=args.scope)] + # Don't initialize compilers config via compilers.get_compiler_config. + # Just let compiler_find do the + # entire process and return an empty config from all_compilers + # Default for any other process is init_config=True + compilers = [c for c in spack.compilers.find_compilers(*paths) + if c.spec not in spack.compilers.all_compilers( + scope=args.scope, init_config=False)] if compilers: spack.compilers.add_compilers_to_config(compilers, scope=args.scope) n = len(compilers) From 1303b5a6a9e83a572c5a02ddfde1c2353ca60d41 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 20 Jun 2016 10:57:58 -0700 Subject: [PATCH 462/481] Adding more init_config as args to all_compilers,all_compilers_config, and get_compiler_config --- lib/spack/spack/compilers/__init__.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index ae72b743b2..c4b41394e0 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -83,7 +83,7 @@ def _to_dict(compiler): return {'compiler': d} -def get_compiler_config(scope=None): +def get_compiler_config(scope=None, init_config=True): """Return the compiler configuration for the specified architecture. """ def init_compiler_config(): @@ -98,7 +98,7 @@ def init_compiler_config(): # Update the configuration if there are currently no compilers # configured. Avoid updating automatically if there ARE site # compilers configured but no user ones. - if not config: + if not config and init_config: if scope is None: # We know no compilers were configured in any scope. init_compiler_config() @@ -153,23 +153,23 @@ def remove_compiler_from_config(compiler_spec, scope=None): spack.config.update_config('compilers', filtered_compiler_config, scope) -def all_compilers_config(scope=None): +def all_compilers_config(scope=None, init_config=True): """Return a set of specs for all the compiler versions currently available to build with. These are instances of CompilerSpec. """ # Get compilers for this architecture. global _cache_config_file #Create a cache of the config file so we don't load all the time. if not _cache_config_file: - _cache_config_file = get_compiler_config(scope) + _cache_config_file = get_compiler_config(scope, init_config) return _cache_config_file else: return _cache_config_file -def all_compilers(scope=None): +def all_compilers(scope=None, init_config=True): # Return compiler specs from the merged config. return [spack.spec.CompilerSpec(s['compiler']['spec']) - for s in all_compilers_config(scope)] + for s in all_compilers_config(scope, init_config)] def default_compiler(): From 88888f5ba02cd63cdd43d42912d368cb9f323557 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 20 Jun 2016 10:58:20 -0700 Subject: [PATCH 463/481] Got rid of unnecessary module arg --- lib/spack/spack/compilers/gcc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/compilers/gcc.py b/lib/spack/spack/compilers/gcc.py index 3f552eaece..2fae6688db 100644 --- a/lib/spack/spack/compilers/gcc.py +++ b/lib/spack/spack/compilers/gcc.py @@ -77,9 +77,9 @@ def fc_version(cls, fc): return get_compiler_version( fc, '-dumpversion', # older gfortran versions don't have simple dumpversion output. - r'(?:GNU Fortran \(GCC\))?(\d+\.\d+(?:\.\d+)?)', module) + r'(?:GNU Fortran \(GCC\))?(\d+\.\d+(?:\.\d+)?)') @classmethod def f77_version(cls, f77): - return cls.fc_version(f77, module) + return cls.fc_version(f77) From 6180e6c047ba09a8c7359ba9327ca6a332b4e073 Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Mon, 20 Jun 2016 11:33:56 -0700 Subject: [PATCH 464/481] automatic case-insensitive like smartsearch, docs New documentation for the new options, also case insensitive by default unless the user includes an upper-case character in their pattern. --- lib/spack/docs/basic_usage.rst | 25 +++++++++++++++++++++---- lib/spack/spack/cmd/list.py | 3 ++- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst index 50c48b802b..f4d42c23ad 100644 --- a/lib/spack/docs/basic_usage.rst +++ b/lib/spack/docs/basic_usage.rst @@ -24,12 +24,29 @@ Spack can install: .. command-output:: spack list -The packages are listed by name in alphabetical order. You can also -do wildcats searches using ``*``: +The packages are listed by name in alphabetical order. If you specify a +pattern to match, it will follow this set of rules. A pattern with no +wildcards, ``*`` or ``?``, will be treated as though it started and ended iwth +``*``, so ``util`` is equivalent to ``*util*``. A pattern with no capitol +letters will be treated as case-insensitive. You can also add the ``-i`` flag +to specify a case insensitive search, or ``-d`` to search the description of +the package in addition to the name. Some examples: -.. command-output:: spack list m* +All packages whose names contain "sql" case insensitive: -.. command-output:: spack list *util* +.. command-output:: spack list sql + +All packages whose names start with a capitol M: + +.. command-output:: spack list M* + +All packages whose names or descriptions contain Documentation: + +.. command-output:: spack list -d Documentation + +All packages whose names contain documentation case insensitive: + +.. command-output:: spack list -d documentation .. _spack-info: diff --git a/lib/spack/spack/cmd/list.py b/lib/spack/spack/cmd/list.py index 9ce63fa1a4..974c4778ab 100644 --- a/lib/spack/spack/cmd/list.py +++ b/lib/spack/spack/cmd/list.py @@ -58,7 +58,8 @@ def list(parser, args): r = fnmatch.translate('*' + f + '*') else: r = fnmatch.translate(f) - rc = re.compile(r, flags=re.I if args.insensitive else 0) + rc = re.compile(r, flags=re.I if args.insensitive or not any( + l.isupper() for l in f) else 0) res.append(rc) if args.search_description: From ac728d3e60ef9ef8c52333449592c5933b9c59ce Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Mon, 20 Jun 2016 11:38:39 -0700 Subject: [PATCH 465/481] fixing capitol->capital typos --- lib/spack/docs/basic_usage.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst index f4d42c23ad..ceec8c7723 100644 --- a/lib/spack/docs/basic_usage.rst +++ b/lib/spack/docs/basic_usage.rst @@ -27,7 +27,7 @@ Spack can install: The packages are listed by name in alphabetical order. If you specify a pattern to match, it will follow this set of rules. A pattern with no wildcards, ``*`` or ``?``, will be treated as though it started and ended iwth -``*``, so ``util`` is equivalent to ``*util*``. A pattern with no capitol +``*``, so ``util`` is equivalent to ``*util*``. A pattern with no capital letters will be treated as case-insensitive. You can also add the ``-i`` flag to specify a case insensitive search, or ``-d`` to search the description of the package in addition to the name. Some examples: @@ -36,7 +36,7 @@ All packages whose names contain "sql" case insensitive: .. command-output:: spack list sql -All packages whose names start with a capitol M: +All packages whose names start with a capital M: .. command-output:: spack list M* From f6aa864bc42f2ed517a836f398fb5ee00a2494b3 Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Mon, 20 Jun 2016 11:41:36 -0700 Subject: [PATCH 466/481] adding quotes to make sure the glob is interpreted correctly --- lib/spack/docs/basic_usage.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst index ceec8c7723..f103f7f5a5 100644 --- a/lib/spack/docs/basic_usage.rst +++ b/lib/spack/docs/basic_usage.rst @@ -38,7 +38,7 @@ All packages whose names contain "sql" case insensitive: All packages whose names start with a capital M: -.. command-output:: spack list M* +.. command-output:: spack list 'M*' All packages whose names or descriptions contain Documentation: From 45482187b473d3b613700bb29af82748a54c4d1a Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Mon, 20 Jun 2016 11:42:35 -0700 Subject: [PATCH 467/481] one more typo issue --- lib/spack/docs/basic_usage.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst index f103f7f5a5..ec193e767d 100644 --- a/lib/spack/docs/basic_usage.rst +++ b/lib/spack/docs/basic_usage.rst @@ -26,7 +26,7 @@ Spack can install: The packages are listed by name in alphabetical order. If you specify a pattern to match, it will follow this set of rules. A pattern with no -wildcards, ``*`` or ``?``, will be treated as though it started and ended iwth +wildcards, ``*`` or ``?``, will be treated as though it started and ended with ``*``, so ``util`` is equivalent to ``*util*``. A pattern with no capital letters will be treated as case-insensitive. You can also add the ``-i`` flag to specify a case insensitive search, or ``-d`` to search the description of From b86e3047f34ab24ef8ff4586c433fcf79580aeca Mon Sep 17 00:00:00 2001 From: alalazo Date: Mon, 20 Jun 2016 20:46:11 +0200 Subject: [PATCH 468/481] openmpi : revert part of #1079 (OpenMPI can be built without fortran) Building OpenMPI without fortran is possible, and was already handled. Raising an install error we effectively renounce at clang+OpenMPI. build_environment : sets 'CC', 'CXX', and similar variables only if a compiler is found --- lib/spack/spack/build_environment.py | 8 ++--- .../repos/builtin/packages/openmpi/package.py | 31 +++++++++++++------ 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 7c65091d49..bdad0f8b33 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -183,20 +183,20 @@ def set_compiler_environment_variables(pkg, env): # and return it # TODO : add additional kwargs for better diagnostics, like requestor, ttyout, ttyerr, etc. link_dir = spack.build_env_path - env.set('CC', join_path(link_dir, compiler.link_paths['cc'])) - env.set('CXX', join_path(link_dir, compiler.link_paths['cxx'])) - env.set('F77', join_path(link_dir, compiler.link_paths['f77'])) - env.set('FC', join_path(link_dir, compiler.link_paths['fc'])) # Set SPACK compiler variables so that our wrapper knows what to call if compiler.cc: env.set('SPACK_CC', compiler.cc) + env.set('CC', join_path(link_dir, compiler.link_paths['cc'])) if compiler.cxx: env.set('SPACK_CXX', compiler.cxx) + env.set('CXX', join_path(link_dir, compiler.link_paths['cxx'])) if compiler.f77: env.set('SPACK_F77', compiler.f77) + env.set('F77', join_path(link_dir, compiler.link_paths['f77'])) if compiler.fc: env.set('SPACK_FC', compiler.fc) + env.set('FC', join_path(link_dir, compiler.link_paths['fc'])) # Set SPACK compiler rpath flags so that our wrapper knows what to use env.set('SPACK_CC_RPATH_ARG', compiler.cc_rpath_arg) diff --git a/var/spack/repos/builtin/packages/openmpi/package.py b/var/spack/repos/builtin/packages/openmpi/package.py index 695fb82f62..588d862d96 100644 --- a/var/spack/repos/builtin/packages/openmpi/package.py +++ b/var/spack/repos/builtin/packages/openmpi/package.py @@ -24,6 +24,8 @@ ############################################################################## import os +import llnl.util.tty as tty + from spack import * @@ -118,6 +120,21 @@ def setup_dependent_package(self, module, dep_spec): self.spec.mpifc = join_path(self.prefix.bin, 'mpif90') self.spec.mpif77 = join_path(self.prefix.bin, 'mpif77') + def setup_environment(self, spack_env, run_env): + # As of 06/2016 there is no mechanism to specify that packages which + # depends on MPI need C or/and Fortran implementation. For now + # require both. + if (self.compiler.f77 is None) or (self.compiler.fc is None): + tty.warn('OpenMPI : FORTRAN compiler not found') + tty.warn('OpenMPI : FORTRAN bindings will be disabled') + spack_env.unset('FC') + spack_env.unset('F77') + # Setting an attribute here and using it in the 'install' + # method is needed to ensure tty.warn is actually displayed + # to user and not redirected to spack-build.out + self.config_extra = ['--enable-mpi-fortran=none', + '--disable-oshmem-fortran'] + @property def verbs(self): # Up through version 1.6, this option was previously named @@ -129,17 +146,14 @@ def verbs(self): return 'verbs' def install(self, spec, prefix): - # As of 06/2016 there is no mechanism to specify that packages which - # depends on MPI need C or/and Fortran implementation. For now - # require both. - if (self.compiler.f77 is None) or (self.compiler.fc is None): - raise InstallError('OpenMPI requires both C and Fortran ', - 'compilers!') - config_args = ["--prefix=%s" % prefix, "--with-hwloc=%s" % spec['hwloc'].prefix, "--enable-shared", "--enable-static"] + + if getattr(self, 'config_extra', None) is not None: + config_args.extend(self.config_extra) + # Variant based arguments config_args.extend([ # Schedulers @@ -170,9 +184,6 @@ def install(self, spec, prefix): if self.version == ver("1.6.5") and '+lanl' in spec: config_args.append("--with-platform=contrib/platform/lanl/tlcc2/optimized-nopanasas") # NOQA: ignore=E501 - if not self.compiler.f77 and not self.compiler.fc: - config_args.append("--enable-mpi-fortran=no") - configure(*config_args) make() make("install") From 5aa3131385843dbf69c121bf7a134469a312a0ec Mon Sep 17 00:00:00 2001 From: alalazo Date: Mon, 20 Jun 2016 21:11:33 +0200 Subject: [PATCH 469/481] qa : fixed flake8 issues --- lib/spack/spack/build_environment.py | 104 +++++++++++++++------------ 1 file changed, 60 insertions(+), 44 deletions(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index bdad0f8b33..3fcfb151b8 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -75,20 +75,19 @@ # set_build_environment_variables and used to pass parameters to # Spack's compiler wrappers. # -SPACK_ENV_PATH = 'SPACK_ENV_PATH' -SPACK_DEPENDENCIES = 'SPACK_DEPENDENCIES' -SPACK_PREFIX = 'SPACK_PREFIX' -SPACK_INSTALL = 'SPACK_INSTALL' -SPACK_DEBUG = 'SPACK_DEBUG' -SPACK_SHORT_SPEC = 'SPACK_SHORT_SPEC' -SPACK_DEBUG_LOG_DIR = 'SPACK_DEBUG_LOG_DIR' +SPACK_ENV_PATH = 'SPACK_ENV_PATH' +SPACK_DEPENDENCIES = 'SPACK_DEPENDENCIES' +SPACK_PREFIX = 'SPACK_PREFIX' +SPACK_INSTALL = 'SPACK_INSTALL' +SPACK_DEBUG = 'SPACK_DEBUG' +SPACK_SHORT_SPEC = 'SPACK_SHORT_SPEC' +SPACK_DEBUG_LOG_DIR = 'SPACK_DEBUG_LOG_DIR' # Platform-specific library suffix. dso_suffix = 'dylib' if sys.platform == 'darwin' else 'so' - class MakeExecutable(Executable): """Special callable executable object for make so the user can specify parallel or not on a per-invocation basis. Using @@ -99,6 +98,7 @@ class MakeExecutable(Executable): Note that if the SPACK_NO_PARALLEL_MAKE env var is set it overrides everything. """ + def __init__(self, name, jobs): super(MakeExecutable, self).__init__(name) self.jobs = jobs @@ -113,12 +113,13 @@ def __call__(self, *args, **kwargs): return super(MakeExecutable, self).__call__(*args, **kwargs) + def load_module(mod): """Takes a module name and removes modules until it is possible to load that module. It then loads the provided module. Depends on the modulecmd implementation of modules used in cray and lmod. """ - #Create an executable of the module command that will output python code + # Create an executable of the module command that will output python code modulecmd = which('modulecmd') modulecmd.add_default_arg('python') @@ -129,11 +130,13 @@ def load_module(mod): text = modulecmd('show', mod, output=str, error=str).split() for i, word in enumerate(text): if word == 'conflict': - exec(compile(modulecmd('unload', text[i+1], output=str, error=str), '', 'exec')) + exec(compile(modulecmd('unload', text[ + i + 1], output=str, error=str), '', 'exec')) # Load the module now that there are no conflicts load = modulecmd('load', mod, output=str, error=str) exec(compile(load, '', 'exec')) + def get_path_from_module(mod): """Inspects a TCL module for entries that indicate the absolute path at which the library supported by said module can be found. @@ -146,7 +149,7 @@ def get_path_from_module(mod): text = modulecmd('show', mod, output=str, error=str).split('\n') # If it lists its package directory, return that for line in text: - if line.find(mod.upper()+'_DIR') >= 0: + if line.find(mod.upper() + '_DIR') >= 0: words = line.split() return words[2] @@ -154,34 +157,37 @@ def get_path_from_module(mod): for line in text: rpath = line.find('-rpath/') if rpath >= 0: - return line[rpath+6:line.find('/lib')] + return line[rpath + 6:line.find('/lib')] # If it lists a -L instruction, use that for line in text: L = line.find('-L/') if L >= 0: - return line[L+2:line.find('/lib')] + return line[L + 2:line.find('/lib')] # If it sets the LD_LIBRARY_PATH or CRAY_LD_LIBRARY_PATH, use that for line in text: - if line.find('LD_LIBRARY_PATH') >= 0: + if line.find('LD_LIBRARY_PATH') >= 0: words = line.split() path = words[2] return path[:path.find('/lib')] # Unable to find module path return None + def set_compiler_environment_variables(pkg, env): assert(pkg.spec.concrete) compiler = pkg.compiler flags = pkg.spec.compiler_flags # Set compiler variables used by CMake and autotools - assert all(key in compiler.link_paths for key in ('cc', 'cxx', 'f77', 'fc')) + assert all(key in compiler.link_paths for key in ( + 'cc', 'cxx', 'f77', 'fc')) # Populate an object with the list of environment modifications # and return it - # TODO : add additional kwargs for better diagnostics, like requestor, ttyout, ttyerr, etc. + # TODO : add additional kwargs for better diagnostics, like requestor, + # ttyout, ttyerr, etc. link_dir = spack.build_env_path # Set SPACK compiler variables so that our wrapper knows what to call @@ -233,7 +239,8 @@ def set_build_environment_variables(pkg, env): # handled by putting one in the /case-insensitive # directory. Add that to the path too. env_paths = [] - for item in [spack.build_env_path, join_path(spack.build_env_path, pkg.compiler.name)]: + compiler_specific = join_path(spack.build_env_path, pkg.compiler.name) + for item in [spack.build_env_path, compiler_specific]: env_paths.append(item) ci = join_path(item, 'case-insensitive') if os.path.isdir(ci): @@ -246,7 +253,8 @@ def set_build_environment_variables(pkg, env): # Prefixes of all of the package's dependencies go in SPACK_DEPENDENCIES dep_prefixes = [d.prefix for d in pkg.spec.traverse(root=False)] env.set_path(SPACK_DEPENDENCIES, dep_prefixes) - env.set_path('CMAKE_PREFIX_PATH', dep_prefixes) # Add dependencies to CMAKE_PREFIX_PATH + # Add dependencies to CMAKE_PREFIX_PATH + env.set_path('CMAKE_PREFIX_PATH', dep_prefixes) # Install prefix env.set(SPACK_PREFIX, pkg.prefix) @@ -262,7 +270,8 @@ def set_build_environment_variables(pkg, env): env.unset('DYLD_LIBRARY_PATH') # Add bin directories from dependencies to the PATH for the build. - bin_dirs = reversed(filter(os.path.isdir, ['%s/bin' % prefix for prefix in dep_prefixes])) + bin_dirs = reversed( + filter(os.path.isdir, ['%s/bin' % prefix for prefix in dep_prefixes])) for item in bin_dirs: env.prepend_path('PATH', item) @@ -277,8 +286,8 @@ def set_build_environment_variables(pkg, env): for directory in ('lib', 'lib64', 'share'): pcdir = join_path(pre, directory, 'pkgconfig') if os.path.isdir(pcdir): - #pkg_config_dirs.append(pcdir) - env.prepend_path('PKG_CONFIG_PATH',pcdir) + # pkg_config_dirs.append(pcdir) + env.prepend_path('PKG_CONFIG_PATH', pcdir) if pkg.spec.architecture.target.module_name: load_module(pkg.spec.architecture.target.module_name) @@ -301,7 +310,7 @@ def set_module_variables_for_package(pkg, module): m.make_jobs = jobs # TODO: make these build deps that can be installed if not found. - m.make = MakeExecutable('make', jobs) + m.make = MakeExecutable('make', jobs) m.gmake = MakeExecutable('gmake', jobs) # easy shortcut to os.environ @@ -325,33 +334,34 @@ def set_module_variables_for_package(pkg, module): # Set up CMake rpath m.std_cmake_args.append('-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=FALSE') - m.std_cmake_args.append('-DCMAKE_INSTALL_RPATH=%s' % ":".join(get_rpaths(pkg))) + m.std_cmake_args.append('-DCMAKE_INSTALL_RPATH=%s' % + ":".join(get_rpaths(pkg))) # Put spack compiler paths in module scope. link_dir = spack.build_env_path - m.spack_cc = join_path(link_dir, pkg.compiler.link_paths['cc']) + m.spack_cc = join_path(link_dir, pkg.compiler.link_paths['cc']) m.spack_cxx = join_path(link_dir, pkg.compiler.link_paths['cxx']) m.spack_f77 = join_path(link_dir, pkg.compiler.link_paths['f77']) - m.spack_fc = join_path(link_dir, pkg.compiler.link_paths['fc']) + m.spack_fc = join_path(link_dir, pkg.compiler.link_paths['fc']) # Emulate some shell commands for convenience - m.pwd = os.getcwd - m.cd = os.chdir - m.mkdir = os.mkdir - m.makedirs = os.makedirs - m.remove = os.remove - m.removedirs = os.removedirs - m.symlink = os.symlink + m.pwd = os.getcwd + m.cd = os.chdir + m.mkdir = os.mkdir + m.makedirs = os.makedirs + m.remove = os.remove + m.removedirs = os.removedirs + m.symlink = os.symlink - m.mkdirp = mkdirp - m.install = install + m.mkdirp = mkdirp + m.install = install m.install_tree = install_tree - m.rmtree = shutil.rmtree - m.move = shutil.move + m.rmtree = shutil.rmtree + m.move = shutil.move # Useful directories within the prefix are encapsulated in # a Prefix object. - m.prefix = pkg.prefix + m.prefix = pkg.prefix # Platform-specific library suffix. m.dso_suffix = dso_suffix @@ -365,20 +375,22 @@ def get_rpaths(pkg): rpaths.extend(d.prefix.lib64 for d in pkg.spec.dependencies.values() if os.path.isdir(d.prefix.lib64)) # Second module is our compiler mod name. We use that to get rpaths from - # module show output. + # module show output. if pkg.compiler.modules and len(pkg.compiler.modules) > 1: rpaths.append(get_path_from_module(pkg.compiler.modules[1])) return rpaths def parent_class_modules(cls): - """Get list of super class modules that are all descend from spack.Package""" + """ + Get list of super class modules that are all descend from spack.Package + """ if not issubclass(cls, spack.Package) or issubclass(spack.Package, cls): return [] result = [] module = sys.modules.get(cls.__module__) if module: - result = [ module ] + result = [module] for c in cls.__bases__: result.extend(parent_class_modules(c)) return result @@ -390,11 +402,12 @@ def load_external_modules(pkg): for dep in list(pkg.spec.traverse()): if dep.external_module: load_module(dep.external_module) - + + def setup_package(pkg): """Execute all environment setup routines.""" spack_env = EnvironmentModifications() - run_env = EnvironmentModifications() + run_env = EnvironmentModifications() # Before proceeding, ensure that specs and packages are consistent # @@ -410,7 +423,8 @@ def setup_package(pkg): # throwaway environment, but it is kind of dirty. # # TODO: Think about how to avoid this fix and do something cleaner. - for s in pkg.spec.traverse(): s.package.spec = s + for s in pkg.spec.traverse(): + s.package.spec = s set_compiler_environment_variables(pkg, spack_env) set_build_environment_variables(pkg, spack_env) @@ -498,7 +512,9 @@ def child_fun(): # message. Just make the parent exit with an error code. pid, returncode = os.waitpid(pid, 0) if returncode != 0: - raise InstallError("Installation process had nonzero exit code.".format(str(returncode))) + message = "Installation process had nonzero exit code : {code}" + strcode = str(returncode) + raise InstallError(message.format(code=strcode)) class InstallError(spack.error.SpackError): From 4c4f3e974723210ca15e00e44daff3ad1e1098e7 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 20 Jun 2016 14:46:28 -0700 Subject: [PATCH 470/481] Add init_config to compiler_find method --- lib/spack/spack/cmd/compiler.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/cmd/compiler.py b/lib/spack/spack/cmd/compiler.py index ff26bc6ba2..c325628ebf 100644 --- a/lib/spack/spack/cmd/compiler.py +++ b/lib/spack/spack/cmd/compiler.py @@ -84,7 +84,8 @@ def compiler_find(args): if c.spec not in spack.compilers.all_compilers( scope=args.scope, init_config=False)] if compilers: - spack.compilers.add_compilers_to_config(compilers, scope=args.scope) + spack.compilers.add_compilers_to_config(compilers, scope=args.scope, + init_config=False) n = len(compilers) s = 's' if n > 1 else '' filename = spack.config.get_config_filename(args.scope, 'compilers') From d515877d91300c6fb57e84f83011ded15737ca73 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 20 Jun 2016 14:54:26 -0700 Subject: [PATCH 471/481] Add init_config args in compilers/__init__ --- lib/spack/spack/compilers/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index c4b41394e0..a70d42982f 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -93,7 +93,7 @@ def init_compiler_config(): for compiler in compilers: compilers_dict.append(_to_dict(compiler)) spack.config.update_config('compilers', compilers_dict, scope=scope) - + config = spack.config.get_config('compilers', scope=scope) # Update the configuration if there are currently no compilers # configured. Avoid updating automatically if there ARE site @@ -117,14 +117,14 @@ def init_compiler_config(): return [] # Return empty list which we will later append to. -def add_compilers_to_config(compilers, scope=None): +def add_compilers_to_config(compilers, scope=None, init_config=True): """Add compilers to the config for the specified architecture. Arguments: - compilers: a list of Compiler objects. - scope: configuration scope to modify. """ - compiler_config = get_compiler_config(scope) + compiler_config = get_compiler_config(scope, init_config) for compiler in compilers: compiler_config.append(_to_dict(compiler)) global _cache_config_file From 70a25c45871656b0e4007ffe5912bd6e51a5e4dd Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 20 Jun 2016 17:17:25 -0700 Subject: [PATCH 472/481] Change --insensitive to --sensitive in `spack list` --- lib/spack/spack/cmd/list.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/cmd/list.py b/lib/spack/spack/cmd/list.py index 974c4778ab..c921efd1bd 100644 --- a/lib/spack/spack/cmd/list.py +++ b/lib/spack/spack/cmd/list.py @@ -39,8 +39,9 @@ def setup_parser(subparser): 'filter', nargs=argparse.REMAINDER, help='Optional glob patterns to filter results.') subparser.add_argument( - '-i', '--insensitive', action='store_true', default=False, - help='Filtering will be case insensitive.') + '-s', '--sensitive', action='store_true', default=False, + help='Use case-sensitive filtering. Default is case sensitive, ' + 'unless the query contains a capital letter.') subparser.add_argument( '-d', '--search-description', action='store_true', default=False, help='Filtering will also search the description for a match.') @@ -58,8 +59,11 @@ def list(parser, args): r = fnmatch.translate('*' + f + '*') else: r = fnmatch.translate(f) - rc = re.compile(r, flags=re.I if args.insensitive or not any( - l.isupper() for l in f) else 0) + + re_flags = re.I + if any(l.isupper for l in f) or args.sensitive: + re_flags = 0 + rc = re.compile(r, flags=re_flags) res.append(rc) if args.search_description: From 77cd474a2d129bf0df932a09e1e50ae05c6f7eff Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Mon, 20 Jun 2016 20:10:44 -0500 Subject: [PATCH 473/481] Make all packages lowercase. --- var/spack/repos/builtin/packages/{r-DBI => r-dbi}/package.py | 0 var/spack/repos/builtin/packages/{r-rJava => r-rjava}/package.py | 0 .../repos/builtin/packages/{r-RMySQL => r-rmysql}/package.py | 0 var/spack/repos/builtin/packages/{r-RODBC => r-rodbc}/package.py | 0 .../builtin/packages/{r-RPostgreSQL => r-rpostgresql}/package.py | 0 .../repos/builtin/packages/{r-RSQLite => r-rsqlite}/package.py | 0 .../builtin/packages/{r-XLConnect => r-xlconnect}/package.py | 0 .../packages/{r-XLConnectJars => r-xlconnectjars}/package.py | 0 .../repos/builtin/packages/{unixODBC => unixodbc}/package.py | 0 9 files changed, 0 insertions(+), 0 deletions(-) rename var/spack/repos/builtin/packages/{r-DBI => r-dbi}/package.py (100%) rename var/spack/repos/builtin/packages/{r-rJava => r-rjava}/package.py (100%) rename var/spack/repos/builtin/packages/{r-RMySQL => r-rmysql}/package.py (100%) rename var/spack/repos/builtin/packages/{r-RODBC => r-rodbc}/package.py (100%) rename var/spack/repos/builtin/packages/{r-RPostgreSQL => r-rpostgresql}/package.py (100%) rename var/spack/repos/builtin/packages/{r-RSQLite => r-rsqlite}/package.py (100%) rename var/spack/repos/builtin/packages/{r-XLConnect => r-xlconnect}/package.py (100%) rename var/spack/repos/builtin/packages/{r-XLConnectJars => r-xlconnectjars}/package.py (100%) rename var/spack/repos/builtin/packages/{unixODBC => unixodbc}/package.py (100%) diff --git a/var/spack/repos/builtin/packages/r-DBI/package.py b/var/spack/repos/builtin/packages/r-dbi/package.py similarity index 100% rename from var/spack/repos/builtin/packages/r-DBI/package.py rename to var/spack/repos/builtin/packages/r-dbi/package.py diff --git a/var/spack/repos/builtin/packages/r-rJava/package.py b/var/spack/repos/builtin/packages/r-rjava/package.py similarity index 100% rename from var/spack/repos/builtin/packages/r-rJava/package.py rename to var/spack/repos/builtin/packages/r-rjava/package.py diff --git a/var/spack/repos/builtin/packages/r-RMySQL/package.py b/var/spack/repos/builtin/packages/r-rmysql/package.py similarity index 100% rename from var/spack/repos/builtin/packages/r-RMySQL/package.py rename to var/spack/repos/builtin/packages/r-rmysql/package.py diff --git a/var/spack/repos/builtin/packages/r-RODBC/package.py b/var/spack/repos/builtin/packages/r-rodbc/package.py similarity index 100% rename from var/spack/repos/builtin/packages/r-RODBC/package.py rename to var/spack/repos/builtin/packages/r-rodbc/package.py diff --git a/var/spack/repos/builtin/packages/r-RPostgreSQL/package.py b/var/spack/repos/builtin/packages/r-rpostgresql/package.py similarity index 100% rename from var/spack/repos/builtin/packages/r-RPostgreSQL/package.py rename to var/spack/repos/builtin/packages/r-rpostgresql/package.py diff --git a/var/spack/repos/builtin/packages/r-RSQLite/package.py b/var/spack/repos/builtin/packages/r-rsqlite/package.py similarity index 100% rename from var/spack/repos/builtin/packages/r-RSQLite/package.py rename to var/spack/repos/builtin/packages/r-rsqlite/package.py diff --git a/var/spack/repos/builtin/packages/r-XLConnect/package.py b/var/spack/repos/builtin/packages/r-xlconnect/package.py similarity index 100% rename from var/spack/repos/builtin/packages/r-XLConnect/package.py rename to var/spack/repos/builtin/packages/r-xlconnect/package.py diff --git a/var/spack/repos/builtin/packages/r-XLConnectJars/package.py b/var/spack/repos/builtin/packages/r-xlconnectjars/package.py similarity index 100% rename from var/spack/repos/builtin/packages/r-XLConnectJars/package.py rename to var/spack/repos/builtin/packages/r-xlconnectjars/package.py diff --git a/var/spack/repos/builtin/packages/unixODBC/package.py b/var/spack/repos/builtin/packages/unixodbc/package.py similarity index 100% rename from var/spack/repos/builtin/packages/unixODBC/package.py rename to var/spack/repos/builtin/packages/unixodbc/package.py From 0521b9bb32387dd533d76dbb09f5226d865d8c06 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 21 Jun 2016 00:08:12 -0700 Subject: [PATCH 474/481] Fix merge error. --- lib/spack/spack/build_environment.py | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 83468be8e6..60703a9df9 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -130,13 +130,8 @@ def load_module(mod): text = modulecmd('show', mod, output=str, error=str).split() for i, word in enumerate(text): if word == 'conflict': -<<<<<<< HEAD - exec(compile(modulecmd('unload', text[ - i + 1], output=str, error=str), '', 'exec')) -======= exec(compile(modulecmd('unload', text[i + 1], output=str, error=str), '', 'exec')) ->>>>>>> aa86488fd9809ced16704e4bd4d607c89d6dda75 # Load the module now that there are no conflicts load = modulecmd('load', mod, output=str, error=str) exec(compile(load, '', 'exec')) @@ -244,13 +239,8 @@ def set_build_environment_variables(pkg, env): # handled by putting one in the /case-insensitive # directory. Add that to the path too. env_paths = [] -<<<<<<< HEAD compiler_specific = join_path(spack.build_env_path, pkg.compiler.name) for item in [spack.build_env_path, compiler_specific]: -======= - for item in [spack.build_env_path, join_path(spack.build_env_path, - pkg.compiler.name)]: ->>>>>>> aa86488fd9809ced16704e4bd4d607c89d6dda75 env_paths.append(item) ci = join_path(item, 'case-insensitive') if os.path.isdir(ci): @@ -280,13 +270,8 @@ def set_build_environment_variables(pkg, env): env.unset('DYLD_LIBRARY_PATH') # Add bin directories from dependencies to the PATH for the build. -<<<<<<< HEAD bin_dirs = reversed( filter(os.path.isdir, ['%s/bin' % prefix for prefix in dep_prefixes])) -======= - bin_dirs = reversed(filter(os.path.isdir, - ['%s/bin' % prefix for prefix in dep_prefixes])) ->>>>>>> aa86488fd9809ced16704e4bd4d607c89d6dda75 for item in bin_dirs: env.prepend_path('PATH', item) @@ -301,10 +286,6 @@ def set_build_environment_variables(pkg, env): for directory in ('lib', 'lib64', 'share'): pcdir = join_path(pre, directory, 'pkgconfig') if os.path.isdir(pcdir): -<<<<<<< HEAD - # pkg_config_dirs.append(pcdir) -======= ->>>>>>> aa86488fd9809ced16704e4bd4d607c89d6dda75 env.prepend_path('PKG_CONFIG_PATH', pcdir) if pkg.spec.architecture.target.module_name: @@ -402,13 +383,9 @@ def get_rpaths(pkg): def parent_class_modules(cls): -<<<<<<< HEAD """ Get list of super class modules that are all descend from spack.Package """ -======= - """Get list of super class modules that all descend from spack.Package""" ->>>>>>> aa86488fd9809ced16704e4bd4d607c89d6dda75 if not issubclass(cls, spack.Package) or issubclass(spack.Package, cls): return [] result = [] From 58cff1290dccce42fef418c0a7e42222a02f38b9 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 21 Jun 2016 00:19:28 -0700 Subject: [PATCH 475/481] Make environment test less sensitive to user config files. --- lib/spack/spack/test/environment.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/test/environment.py b/lib/spack/spack/test/environment.py index f3644cb0b7..2396961888 100644 --- a/lib/spack/spack/test/environment.py +++ b/lib/spack/spack/test/environment.py @@ -123,7 +123,12 @@ def test_source_files(self): env = EnvironmentModifications.from_sourcing_files(*files) modifications = env.group_by_name() - self.assertEqual(len(modifications), 4) + # This is sensitive to the user's environment; can include + # spurious entries for things like PS1 + # + # TODO: figure out how to make a bit more robust. + self.assertTrue(len(modifications) >= 4) + # Set new variables self.assertEqual(len(modifications['NEW_VAR']), 1) self.assertTrue(isinstance(modifications['NEW_VAR'][0], SetEnv)) From 4041f45b6c387ab07c3e9681d5b3423eb31739a2 Mon Sep 17 00:00:00 2001 From: "Gregory L. Lee" Date: Mon, 20 Jun 2016 16:17:07 -0700 Subject: [PATCH 476/481] fixed conflicts in python package --- var/spack/repos/builtin/packages/cairo/package.py | 1 + var/spack/repos/builtin/packages/freetype/package.py | 2 +- var/spack/repos/builtin/packages/harfbuzz/package.py | 2 ++ var/spack/repos/builtin/packages/python/package.py | 3 +++ 4 files changed, 7 insertions(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/cairo/package.py b/var/spack/repos/builtin/packages/cairo/package.py index 5c7c2fae22..f2aba08ef1 100644 --- a/var/spack/repos/builtin/packages/cairo/package.py +++ b/var/spack/repos/builtin/packages/cairo/package.py @@ -34,6 +34,7 @@ class Cairo(Package): depends_on("libpng") depends_on("glib") depends_on("pixman") + depends_on("freetype") depends_on("fontconfig@2.10.91:") # Require newer version of fontconfig. def install(self, spec, prefix): diff --git a/var/spack/repos/builtin/packages/freetype/package.py b/var/spack/repos/builtin/packages/freetype/package.py index 897c011396..f0b9a40a21 100644 --- a/var/spack/repos/builtin/packages/freetype/package.py +++ b/var/spack/repos/builtin/packages/freetype/package.py @@ -34,7 +34,7 @@ class Freetype(Package): depends_on('libpng') def install(self, spec, prefix): - configure("--prefix=%s" % prefix) + configure("--prefix=%s" % prefix, "--with-harfbuzz=no") make() make("install") diff --git a/var/spack/repos/builtin/packages/harfbuzz/package.py b/var/spack/repos/builtin/packages/harfbuzz/package.py index e4c1b80de3..e26f2843b5 100644 --- a/var/spack/repos/builtin/packages/harfbuzz/package.py +++ b/var/spack/repos/builtin/packages/harfbuzz/package.py @@ -34,6 +34,8 @@ class Harfbuzz(Package): depends_on("glib") depends_on("icu") depends_on("freetype") + depends_on("cairo") + depends_on("zlib") def patch(self): change_sed_delimiter('@', ';', 'src/Makefile.in') diff --git a/var/spack/repos/builtin/packages/python/package.py b/var/spack/repos/builtin/packages/python/package.py index b6a62bf742..15133baea3 100644 --- a/var/spack/repos/builtin/packages/python/package.py +++ b/var/spack/repos/builtin/packages/python/package.py @@ -203,8 +203,11 @@ def python_ignore(self, ext_pkg, args): r'site-packages/site[^/]*\.pyc?$', r'site-packages/__pycache__/site[^/]*\.pyc?$' ]) + if ext_pkg.name != 'py-pygments': + patterns.append(r'bin/pygmentize$') if ext_pkg.name != 'py-numpy': patterns.append(r'bin/f2py$') + patterns.append(r'bin/f2py3$') return match_predicate(ignore_arg, patterns) From 8f3954c1bdc329f2cb09e5e5baa140419222df8f Mon Sep 17 00:00:00 2001 From: "Gregory L. Lee" Date: Mon, 20 Jun 2016 20:21:10 -0700 Subject: [PATCH 477/481] flake 8 fixes for cairo, freetype, and harfbuzz --- var/spack/repos/builtin/packages/cairo/package.py | 8 +++++--- var/spack/repos/builtin/packages/freetype/package.py | 3 ++- var/spack/repos/builtin/packages/harfbuzz/package.py | 1 + 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/var/spack/repos/builtin/packages/cairo/package.py b/var/spack/repos/builtin/packages/cairo/package.py index f2aba08ef1..ddb8d2fd03 100644 --- a/var/spack/repos/builtin/packages/cairo/package.py +++ b/var/spack/repos/builtin/packages/cairo/package.py @@ -24,8 +24,10 @@ ############################################################################## from spack import * + class Cairo(Package): - """Cairo is a 2D graphics library with support for multiple output devices.""" + """Cairo is a 2D graphics library with support for multiple output + devices.""" homepage = "http://cairographics.org" url = "http://cairographics.org/releases/cairo-1.14.0.tar.xz" @@ -35,11 +37,11 @@ class Cairo(Package): depends_on("glib") depends_on("pixman") depends_on("freetype") - depends_on("fontconfig@2.10.91:") # Require newer version of fontconfig. + depends_on("fontconfig@2.10.91:") # Require newer version of fontconfig. def install(self, spec, prefix): configure("--prefix=%s" % prefix, - "--disable-trace", # can cause problems with libiberty + "--disable-trace", # can cause problems with libiberty "--enable-tee") make() make("install") diff --git a/var/spack/repos/builtin/packages/freetype/package.py b/var/spack/repos/builtin/packages/freetype/package.py index f0b9a40a21..dd181623d0 100644 --- a/var/spack/repos/builtin/packages/freetype/package.py +++ b/var/spack/repos/builtin/packages/freetype/package.py @@ -24,12 +24,13 @@ ############################################################################## from spack import * + class Freetype(Package): """Font package""" homepage = "http://http://www.freetype.org" url = "http://download.savannah.gnu.org/releases/freetype/freetype-2.5.3.tar.gz" - version('2.5.3' , 'cafe9f210e45360279c730d27bf071e9') + version('2.5.3', 'cafe9f210e45360279c730d27bf071e9') depends_on('libpng') diff --git a/var/spack/repos/builtin/packages/harfbuzz/package.py b/var/spack/repos/builtin/packages/harfbuzz/package.py index e26f2843b5..f8d73d651b 100644 --- a/var/spack/repos/builtin/packages/harfbuzz/package.py +++ b/var/spack/repos/builtin/packages/harfbuzz/package.py @@ -24,6 +24,7 @@ ############################################################################## from spack import * + class Harfbuzz(Package): """The Harfbuzz package contains an OpenType text shaping engine.""" homepage = "http://www.freedesktop.org/wiki/Software/HarfBuzz/" From e0fbf14f9907aec7eef5e813febca6d3c0e335c9 Mon Sep 17 00:00:00 2001 From: "Gregory L. Lee" Date: Tue, 21 Jun 2016 09:23:58 -0700 Subject: [PATCH 478/481] consolidated f2py regex --- var/spack/repos/builtin/packages/python/package.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/python/package.py b/var/spack/repos/builtin/packages/python/package.py index 15133baea3..8033c52b91 100644 --- a/var/spack/repos/builtin/packages/python/package.py +++ b/var/spack/repos/builtin/packages/python/package.py @@ -206,8 +206,7 @@ def python_ignore(self, ext_pkg, args): if ext_pkg.name != 'py-pygments': patterns.append(r'bin/pygmentize$') if ext_pkg.name != 'py-numpy': - patterns.append(r'bin/f2py$') - patterns.append(r'bin/f2py3$') + patterns.append(r'bin/f2py3?$') return match_predicate(ignore_arg, patterns) From 2efc67da607f29b9246ad70cb74a4ba5f35f8107 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 21 Jun 2016 14:50:33 -0500 Subject: [PATCH 479/481] Remove +tk variant, provide pil --- .../builtin/packages/openjpeg/package.py | 2 + .../repos/builtin/packages/py-pil/package.py | 7 ++- .../builtin/packages/py-pillow/package.py | 61 +++++++++---------- 3 files changed, 36 insertions(+), 34 deletions(-) diff --git a/var/spack/repos/builtin/packages/openjpeg/package.py b/var/spack/repos/builtin/packages/openjpeg/package.py index 9b2063593a..41ca3b44fb 100644 --- a/var/spack/repos/builtin/packages/openjpeg/package.py +++ b/var/spack/repos/builtin/packages/openjpeg/package.py @@ -24,6 +24,7 @@ ############################################################################## from spack import * + class Openjpeg(Package): """ OpenJPEG is an open-source JPEG 2000 codec written in C language. @@ -42,6 +43,7 @@ class Openjpeg(Package): version('1.5.2', '545f98923430369a6b046ef3632ef95c') version('1.5.1', 'd774e4b5a0db5f0f171c4fc0aabfa14e') + depends_on('cmake') def install(self, spec, prefix): cmake('.', *std_cmake_args) diff --git a/var/spack/repos/builtin/packages/py-pil/package.py b/var/spack/repos/builtin/packages/py-pil/package.py index dc4418fdfe..2b3cce26d3 100644 --- a/var/spack/repos/builtin/packages/py-pil/package.py +++ b/var/spack/repos/builtin/packages/py-pil/package.py @@ -24,14 +24,19 @@ ############################################################################## from spack import * + class PyPil(Package): - """The Python Imaging Library (PIL) adds image processing capabilities to your Python interpreter. This library supports many file formats, and provides powerful image processing and graphics capabilities.""" + """The Python Imaging Library (PIL) adds image processing capabilities + to your Python interpreter. This library supports many file formats, + and provides powerful image processing and graphics capabilities.""" homepage = "http://www.pythonware.com/products/pil/" url = "http://effbot.org/media/downloads/Imaging-1.1.7.tar.gz" version('1.1.7', 'fc14a54e1ce02a0225be8854bfba478e') + provides('pil') + extends('python') def install(self, spec, prefix): diff --git a/var/spack/repos/builtin/packages/py-pillow/package.py b/var/spack/repos/builtin/packages/py-pillow/package.py index 5784b9f181..987ff383da 100644 --- a/var/spack/repos/builtin/packages/py-pillow/package.py +++ b/var/spack/repos/builtin/packages/py-pillow/package.py @@ -34,27 +34,35 @@ class PyPillow(Package): homepage = "https://python-pillow.org/" url = "https://pypi.python.org/packages/source/P/Pillow/Pillow-3.0.0.tar.gz" - version('3.2.0', '7cfd093c11205d9e2ebe3c51dfcad510') + # TODO: This version should be deleted once the next release comes out. + # TODO: It fixes a bug that prevented us from linking to Tk/Tcl. + # TODO: Tk/Tcl support is necessary for tkinter bitmap and photo images. + # TODO: If you require this support, run the following command: + # TODO: `spack install py-pillow@3.3.0.dev0 ^python+tk` + version('3.3.0.dev0', git='https://github.com/python-pillow/Pillow.git', + commit='30eced62868141a6c859a4370efd40b9434a7c3f') + + version('3.2.0', '7cfd093c11205d9e2ebe3c51dfcad510', preferred=True) version('3.0.0', 'fc8ac44e93da09678eac7e30c9b7377d') + provides('pil') + # These defaults correspond to Pillow defaults - variant('jpeg', default=True, description='Provide JPEG functionality') - variant('zlib', default=True, description='Access to compressed PNGs') - variant('tiff', default=False, description='Access to TIFF files') + variant('jpeg', default=True, description='Provide JPEG functionality') + variant('zlib', default=True, description='Access to compressed PNGs') + variant('tiff', default=False, description='Access to TIFF files') variant('freetype', default=False, description='Font related services') - variant('lcms', default=False, description='Color management') - variant('tk', default=False, description='Support for tkinter bitmap and photo images') + variant('lcms', default=False, description='Color management') variant('jpeg2000', default=False, description='Provide JPEG 2000 functionality') # Spack does not (yet) support these modes of building - # variant('webp', default=False, description='Provide the WebP format') - # variant('webpmux', default=False, description='WebP metadata, relies on WebP support') + # variant('webp', default=False, description='Provide the WebP format') + # variant('webpmux', default=False, description='WebP metadata, relies on WebP support') # variant('imagequant', default=False, description='Provide improved color quantization') - provides('PIL') - # Required dependencies extends('python') + depends_on('binutils') depends_on('py-setuptools') # Recommended dependencies @@ -62,16 +70,14 @@ class PyPillow(Package): depends_on('zlib', when='+zlib') # Optional dependencies - depends_on('libtiff', when='+tiff') + depends_on('libtiff', when='+tiff') depends_on('freetype', when='+freetype') - depends_on('lcms', when='+lcms') - depends_on('tcl', when='+tk') - depends_on('tk', when='+tk') + depends_on('lcms', when='+lcms') depends_on('openjpeg', when='+jpeg2000') # Spack does not (yet) support these modes of building - # depends_on('webp', when='+webp') - # depends_on('webpmux', when='+webpmux') + # depends_on('webp', when='+webp') + # depends_on('webpmux', when='+webpmux') # depends_on('imagequant', when='+imagequant') def patch(self): @@ -106,11 +112,6 @@ def patch(self): 'LCMS_ROOT = ("{0}", "{1}")'.format( spec['lcms'].prefix.lib, spec['lcms'].prefix.include)) - if '+tk' in spec: - setup.filter('TCL_ROOT = None', - 'TCL_ROOT = ("{0}", "{1}")'.format( - spec['tcl'].prefix.lib, - spec['tcl'].prefix.include)) if '+jpeg2000' in spec: setup.filter('JPEG2K_ROOT = None', 'JPEG2K_ROOT = ("{0}", "{1}")'.format( @@ -118,18 +119,12 @@ def patch(self): spec['openjpeg'].prefix.include)) def install(self, spec, prefix): - build_args = [ - '--{0}-jpeg'.format('enable' if '+jpeg' in spec else 'disable'), - '--{0}-zlib'.format('enable' if '+zlib' in spec else 'disable'), - '--{0}-tiff'.format('enable' if '+tiff' in spec else 'disable'), - '--{0}-freetype'.format( - 'enable' if '+freetype' in spec else 'disable'), - '--{0}-lcms'.format('enable' if '+lcms' in spec else 'disable'), - '--{0}-tk'.format('enable' if '+tk' in spec else 'disable'), - '--{0}-tcl'.format('enable' if '+tk' in spec else 'disable'), - '--{0}-jpeg2000'.format( - 'enable' if '+jpeg2000' in spec else 'disable') - ] + def variant_to_flag(variant): + able = 'enable' if '+{0}'.format(variant) in spec else 'disable' + return '--{0}-{1}'.format(able, variant) + + variants = ['jpeg', 'zlib', 'tiff', 'freetype', 'lcms', 'jpeg2000'] + build_args = list(map(variant_to_flag, variants)) python('setup.py', 'build_ext', *build_args) python('setup.py', 'install', '--prefix={0}'.format(prefix)) From 66089c89aa85369f1e19ae1636ed307e82fdb708 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 21 Jun 2016 14:55:49 -0500 Subject: [PATCH 480/481] Depend on pil --- var/spack/repos/builtin/packages/openjpeg/package.py | 4 ++-- var/spack/repos/builtin/packages/py-basemap/package.py | 8 +++++--- .../repos/builtin/packages/py-scikit-image/package.py | 7 +++++-- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/var/spack/repos/builtin/packages/openjpeg/package.py b/var/spack/repos/builtin/packages/openjpeg/package.py index 41ca3b44fb..1d00edb06d 100644 --- a/var/spack/repos/builtin/packages/openjpeg/package.py +++ b/var/spack/repos/builtin/packages/openjpeg/package.py @@ -37,9 +37,9 @@ class Openjpeg(Package): homepage = "https://github.com/uclouvain/openjpeg" url = "https://github.com/uclouvain/openjpeg/archive/version.2.1.tar.gz" - version('2.1' , '3e1c451c087f8462955426da38aa3b3d') + version('2.1', '3e1c451c087f8462955426da38aa3b3d') version('2.0.1', '105876ed43ff7dbb2f90b41b5a43cfa5') - version('2.0' , 'cdf266530fee8af87454f15feb619609') + version('2.0', 'cdf266530fee8af87454f15feb619609') version('1.5.2', '545f98923430369a6b046ef3632ef95c') version('1.5.1', 'd774e4b5a0db5f0f171c4fc0aabfa14e') diff --git a/var/spack/repos/builtin/packages/py-basemap/package.py b/var/spack/repos/builtin/packages/py-basemap/package.py index 95d02bce91..13f604bacd 100644 --- a/var/spack/repos/builtin/packages/py-basemap/package.py +++ b/var/spack/repos/builtin/packages/py-basemap/package.py @@ -23,10 +23,12 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## from spack import * -import os + class PyBasemap(Package): - """The matplotlib basemap toolkit is a library for plotting 2D data on maps in Python.""" + """The matplotlib basemap toolkit is a library for plotting + 2D data on maps in Python.""" + homepage = "http://matplotlib.org/basemap/" url = "https://downloads.sourceforge.net/project/matplotlib/matplotlib-toolkits/basemap-1.0.7/basemap-1.0.7.tar.gz" @@ -36,7 +38,7 @@ class PyBasemap(Package): depends_on('py-setuptools') depends_on('py-numpy') depends_on('py-matplotlib+gui') - depends_on('py-pillow') + depends_on('pil') depends_on("geos") def install(self, spec, prefix): diff --git a/var/spack/repos/builtin/packages/py-scikit-image/package.py b/var/spack/repos/builtin/packages/py-scikit-image/package.py index 0479e9eb0a..4c8fd0942a 100644 --- a/var/spack/repos/builtin/packages/py-scikit-image/package.py +++ b/var/spack/repos/builtin/packages/py-scikit-image/package.py @@ -24,8 +24,11 @@ ############################################################################## from spack import * + class PyScikitImage(Package): - """Image processing algorithms for SciPy, including IO, morphology, filtering, warping, color manipulation, object detection, etc.""" + """Image processing algorithms for SciPy, including IO, morphology, + filtering, warping, color manipulation, object detection, etc.""" + homepage = "http://scikit-image.org/" url = "https://pypi.python.org/packages/source/s/scikit-image/scikit-image-0.12.3.tar.gz" @@ -34,7 +37,7 @@ class PyScikitImage(Package): extends('python', ignore=r'bin/.*\.py$') depends_on('py-dask') - depends_on('py-pillow') + depends_on('pil') depends_on('py-networkx') depends_on('py-six') depends_on('py-scipy') From a7ffb2c1a4526acf11edbf121770adcb058fc228 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 21 Jun 2016 15:21:53 -0500 Subject: [PATCH 481/481] Flake8 --- .../builtin/packages/py-pillow/package.py | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/var/spack/repos/builtin/packages/py-pillow/package.py b/var/spack/repos/builtin/packages/py-pillow/package.py index 987ff383da..25cf412b31 100644 --- a/var/spack/repos/builtin/packages/py-pillow/package.py +++ b/var/spack/repos/builtin/packages/py-pillow/package.py @@ -57,8 +57,8 @@ class PyPillow(Package): # Spack does not (yet) support these modes of building # variant('webp', default=False, description='Provide the WebP format') - # variant('webpmux', default=False, description='WebP metadata, relies on WebP support') - # variant('imagequant', default=False, description='Provide improved color quantization') + # variant('webpmux', default=False, description='WebP metadata, relies on WebP support') # NOQA: ignore=E501 + # variant('imagequant', default=False, description='Provide improved color quantization') # NOQA: ignore=E501 # Required dependencies extends('python') @@ -90,33 +90,33 @@ def patch(self): if '+jpeg' in spec: setup.filter('JPEG_ROOT = None', 'JPEG_ROOT = ("{0}", "{1}")'.format( - spec['jpeg'].prefix.lib, - spec['jpeg'].prefix.include)) + spec['jpeg'].prefix.lib, + spec['jpeg'].prefix.include)) if '+zlib' in spec: setup.filter('ZLIB_ROOT = None', 'ZLIB_ROOT = ("{0}", "{1}")'.format( - spec['zlib'].prefix.lib, - spec['zlib'].prefix.include)) + spec['zlib'].prefix.lib, + spec['zlib'].prefix.include)) if '+tiff' in spec: setup.filter('TIFF_ROOT = None', 'TIFF_ROOT = ("{0}", "{1}")'.format( - spec['libtiff'].prefix.lib, - spec['libtiff'].prefix.include)) + spec['libtiff'].prefix.lib, + spec['libtiff'].prefix.include)) if '+freetype' in spec: setup.filter('FREETYPE_ROOT = None', 'FREETYPE_ROOT = ("{0}", "{1}")'.format( - spec['freetype'].prefix.lib, - spec['freetype'].prefix.include)) + spec['freetype'].prefix.lib, + spec['freetype'].prefix.include)) if '+lcms' in spec: setup.filter('LCMS_ROOT = None', 'LCMS_ROOT = ("{0}", "{1}")'.format( - spec['lcms'].prefix.lib, - spec['lcms'].prefix.include)) + spec['lcms'].prefix.lib, + spec['lcms'].prefix.include)) if '+jpeg2000' in spec: setup.filter('JPEG2K_ROOT = None', 'JPEG2K_ROOT = ("{0}", "{1}")'.format( - spec['openjpeg'].prefix.lib, - spec['openjpeg'].prefix.include)) + spec['openjpeg'].prefix.lib, + spec['openjpeg'].prefix.include)) def install(self, spec, prefix): def variant_to_flag(variant):