From 9786bd932bdc958bb9b6cfc55d53aa5904535c64 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Mon, 11 Mar 2024 14:33:16 +0100 Subject: [PATCH] Update TensorFlow ecosystem (#41069) --- .../stacks/ml-linux-x86_64-cpu/spack.yaml | 2 +- .../stacks/ml-linux-x86_64-cuda/spack.yaml | 2 +- .../stacks/ml-linux-x86_64-rocm/spack.yaml | 2 +- .../repos/builtin/packages/py-h5py/package.py | 14 +++-- .../builtin/packages/py-keras/package.py | 6 +- .../packages/py-tensorboard/package.py | 48 +++++++++++++--- .../builtin/packages/py-tensorflow/package.py | 56 ++++++++++++++----- 7 files changed, 97 insertions(+), 33 deletions(-) diff --git a/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-cpu/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-cpu/spack.yaml index ec28ec1fa3..fd22335885 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-cpu/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-cpu/spack.yaml @@ -21,7 +21,7 @@ spack: - py-jaxlib # Keras - # - py-keras backend=tensorflow + - py-keras backend=tensorflow - py-keras backend=jax - py-keras backend=torch - py-keras-applications diff --git a/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-cuda/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-cuda/spack.yaml index f9d74f6a4b..5053f28fd3 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-cuda/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-cuda/spack.yaml @@ -25,7 +25,7 @@ spack: - py-jaxlib # Keras - # - py-keras backend=tensorflow + - py-keras backend=tensorflow - py-keras backend=jax - py-keras backend=torch - py-keras-applications diff --git a/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-rocm/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-rocm/spack.yaml index c83b4757fc..4fb9e0eddf 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-rocm/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-rocm/spack.yaml @@ -29,7 +29,7 @@ spack: # Keras # - py-keras backend=tensorflow - py-keras backend=jax - # - py-keras backend=torch + - py-keras backend=torch - py-keras-applications - py-keras-preprocessing - py-keras2onnx diff --git a/var/spack/repos/builtin/packages/py-h5py/package.py b/var/spack/repos/builtin/packages/py-h5py/package.py index 9aad3c0396..aadeb4b900 100644 --- a/var/spack/repos/builtin/packages/py-h5py/package.py +++ b/var/spack/repos/builtin/packages/py-h5py/package.py @@ -18,6 +18,8 @@ class PyH5py(PythonPackage): license("BSD-3-Clause") version("master", branch="master") + version("3.10.0", sha256="d93adc48ceeb33347eb24a634fb787efc7ae4644e6ea4ba733d099605045c049") + version("3.9.0", sha256="e604db6521c1e367c6bd7fad239c847f53cc46646f2d2651372d05ae5e95f817") version("3.8.0", sha256="6fead82f0c4000cf38d53f9c030780d81bfa0220218aee13b90b7701c937d95f") version("3.7.0", sha256="3fcf37884383c5da64846ab510190720027dca0768def34dd8dcb659dbe5cbf3") version("3.6.0", sha256="8752d2814a92aba4e2b2a5922d2782d0029102d99caaf3c201a566bc0b40db29") @@ -40,11 +42,10 @@ class PyH5py(PythonPackage): variant("mpi", default=True, description="Build with MPI support") # Python versions - depends_on("python@3.7:", type=("build", "run"), when="@3.2:") - depends_on("python@3.6:", type=("build", "run"), when="@3:3.1") depends_on("python@:3.9", type=("build", "run"), when="@:2.8") # Build dependencies + depends_on("py-cython@0.29.31:0", type="build", when="@3.9:") depends_on("py-cython@0.29.15:0", type=("build"), when="@3:3.7 ^python@3.9.0:") depends_on("py-cython@0.29.14:0", type=("build"), when="@3:3.7 ^python@3.8.0:3.8") depends_on("py-cython@0.29:0", type=("build"), when="@3:") @@ -52,15 +53,13 @@ class PyH5py(PythonPackage): depends_on("py-pkgconfig", type="build") depends_on("py-setuptools@61:", type="build", when="@3.8.0:") depends_on("py-setuptools", type="build") - depends_on("py-wheel", type="build", when="@3:") # Build and runtime dependencies - depends_on("py-cached-property@1.5:", type=("build", "run"), when="@:3.6 ^python@:3.7") + depends_on("py-numpy@1.17.3:", type=("build", "run"), when="@3.9:") depends_on("py-numpy@1.19.3:", type=("build", "run"), when="@3:3.5 ^python@3.9.0:") depends_on("py-numpy@1.17.5:", type=("build", "run"), when="@3:3.5 ^python@3.8.0:3.8") depends_on("py-numpy@1.14.5:", type=("build", "run"), when="@3:") depends_on("py-numpy@1.7:", type=("build", "run"), when="@:2") - depends_on("py-six", type=("build", "run"), when="@:2") # Link dependencies (py-h5py v2 cannot build against HDF5 1.12 regardless # of API setting) @@ -71,9 +70,14 @@ class PyH5py(PythonPackage): # MPI dependencies depends_on("hdf5+mpi", when="+mpi") depends_on("mpi", when="+mpi") + depends_on("py-mpi4py@3.1.1:", when="@3.8: +mpi", type=("build", "run")) depends_on("py-mpi4py@3.0.2:", when="@3: +mpi", type=("build", "run")) depends_on("py-mpi4py", when="@:2 +mpi", type=("build", "run")) + # Historical dependencies + depends_on("py-cached-property@1.5:", type=("build", "run"), when="@:3.6 ^python@:3.7") + depends_on("py-six", type=("build", "run"), when="@:2") + def flag_handler(self, name, flags): if name == "cflags": if self.spec.satisfies("%oneapi@2023.0.0:"): diff --git a/var/spack/repos/builtin/packages/py-keras/package.py b/var/spack/repos/builtin/packages/py-keras/package.py index f9eb62b9a6..924dd22a38 100644 --- a/var/spack/repos/builtin/packages/py-keras/package.py +++ b/var/spack/repos/builtin/packages/py-keras/package.py @@ -25,6 +25,7 @@ class PyKeras(PythonPackage): version("3.0.2", sha256="526b6c053cdd880a33467c5bfd5c460a5bdc0c58869c2683171c2dec2ad3c2d0") version("3.0.1", sha256="d993721510fa654582132192193f69b1b3165418a6e00a73c3edce615b3cc672") version("3.0.0", sha256="82a9fa4b32a049b38151d11188ed15d74f21f853f163e78da0950dce1f244ccc") + version("2.15.0", sha256="b281ce09226576e0593b8dab0d9e5d42c334e053ce6f4f154dc6cd745ab93d2f") version("2.14.0", sha256="a845d446b6ae626f61dde5ab2fa952530b6c17b4f9ed03e9362bd20172d00cca") version("2.13.1", sha256="b3591493cce75a69adef7b192cec6be222e76e2386d132cd4e34aa190b0ecbd5") version("2.12.0", sha256="6336cebb6b2b0a91f7efd3ff3a9db3a94f2abccf07a40323138afb80826aec62") @@ -75,8 +76,7 @@ class PyKeras(PythonPackage): depends_on("py-protobuf", type=("build", "run"), when="@3:") # requirements-tensorflow-cuda.txt - conflicts("backend=tensorflow", msg="Requires TensorFlow 2.16, not yet released") - # depends_on("py-tensorflow@2.16.0", type=("build", "run"), when="@3.0 backend=tensorflow") + depends_on("py-tensorflow@2.16.1", type=("build", "run"), when="@3.0 backend=tensorflow") # requirements-jax-cuda.txt depends_on("py-jax@0.4.23", type=("build", "run"), when="@3.0.5: backend=jax") @@ -98,7 +98,7 @@ class PyKeras(PythonPackage): depends_on("py-pydot", type=("build", "run"), when="@:2") depends_on("py-pyyaml", type=("build", "run"), when="@:2") depends_on("py-six", type=("build", "run"), when="@:2") - for minor_ver in range(6, 15): + for minor_ver in range(6, 16): depends_on( "py-tensorflow@2.{}".format(minor_ver), type=("build", "run"), diff --git a/var/spack/repos/builtin/packages/py-tensorboard/package.py b/var/spack/repos/builtin/packages/py-tensorboard/package.py index 0f3322f49b..9f39ece8c6 100644 --- a/var/spack/repos/builtin/packages/py-tensorboard/package.py +++ b/var/spack/repos/builtin/packages/py-tensorboard/package.py @@ -21,6 +21,36 @@ class PyTensorboard(PythonPackage): license("Apache-2.0") + version( + "2.16.2", + sha256="9f2b4e7dad86667615c0e5cd072f1ea8403fc032a299f0072d6f74855775cc45", + expand=False, + ) + version( + "2.16.1", + sha256="928b62567911a8eeb2ebeb7482a9e4599b35f6713a6f2c56655259c18a139569", + expand=False, + ) + version( + "2.16.0", + sha256="263b909a2009cb3a79daa6abe64c1785cc317c25a54e4db2fecb6429ffc54c58", + expand=False, + ) + version( + "2.15.2", + sha256="a6f6443728064d962caea6d34653e220e34ef8df764cb06a8212c17e1a8f0622", + expand=False, + ) + version( + "2.15.1", + sha256="c46c1d1cf13a458c429868a78b2531d8ff5f682058d69ec0840b0bc7a38f1c0f", + expand=False, + ) + version( + "2.15.0", + sha256="c05b4d02a3a9fd4bd6c25265087d52b49b790a871ddf98f4fb32fe97cbbc7ad0", + expand=False, + ) version( "2.14.1", sha256="3db108fb58f023b6439880e177743c5f1e703e9eeb5fb7d597871f949f85fd58", @@ -138,22 +168,26 @@ class PyTensorboard(PythonPackage): depends_on("py-grpcio@1.48.2:", type=("build", "run"), when="@2.12:") depends_on("py-grpcio@1.24.3:", type=("build", "run"), when="@2.3:") depends_on("py-grpcio@1.23.3:", type=("build", "run"), when="@2.2") - depends_on("py-google-auth@1.6.3:2", type=("build", "run"), when="@2.7:") - depends_on("py-google-auth@1.6.3:1", type=("build", "run"), when="@:2.6") - depends_on("py-google-auth-oauthlib@0.5:1.0", type=("build", "run"), when="@2.12.1:") - depends_on("py-google-auth-oauthlib@0.4.1:0.4", type=("build", "run"), when="@:2.12.0") depends_on("py-markdown@2.6.8:", type=("build", "run")) depends_on("py-numpy@1.12.0:", type=("build", "run")) - depends_on("py-protobuf@3.19.6:", type=("build", "run"), when="@2.12:") + depends_on("py-protobuf@3.19.6:4.23,4.24.1:", type=("build", "run"), when="@2.15.2:") + depends_on("py-protobuf@3.19.6:4.23", type=("build", "run"), when="@2.12:2.15.1") depends_on("py-protobuf@3.9.2:3", type=("build", "run"), when="@2.11") depends_on("py-protobuf@3.9.2:3.19", type=("build", "run"), when="@2.9:2.10") depends_on("py-protobuf@3.6.0:3.19", type=("build", "run"), when="@:2.8") - depends_on("py-requests@2.21.0:2", type=("build", "run")) depends_on("py-setuptools@41.0.0:", type=("build", "run")) depends_on("py-six@1.10.0:", type=("build", "run"), when="@:2.4,2.14:") depends_on("py-tensorboard-data-server@0.7", type=("build", "run"), when="@2.12:") depends_on("py-tensorboard-data-server@0.6", type=("build", "run"), when="@2.5:2.11") - depends_on("py-tensorboard-plugin-wit@1.6.0:", type=("build", "run"), when="@:2.13") depends_on("py-werkzeug@1.0.1:", type=("build", "run"), when="@2.9:") depends_on("py-werkzeug@0.11.15:", type=("build", "run")) + + # Historical dependencies + depends_on("py-google-auth@1.6.3:2", type=("build", "run"), when="@2.7:2.15") + depends_on("py-google-auth@1.6.3:1", type=("build", "run"), when="@:2.6") + depends_on("py-google-auth-oauthlib@0.5:1", type=("build", "run"), when="@2.15") + depends_on("py-google-auth-oauthlib@0.5:1.0", type=("build", "run"), when="@2.12.1:2.14") + depends_on("py-google-auth-oauthlib@0.4.1:0.4", type=("build", "run"), when="@:2.12.0") + depends_on("py-requests@2.21.0:2", type=("build", "run"), when="@:2.15") + depends_on("py-tensorboard-plugin-wit@1.6.0:", type=("build", "run"), when="@:2.13") depends_on("py-wheel@0.26:", type="build", when="@:2.13") diff --git a/var/spack/repos/builtin/packages/py-tensorflow/package.py b/var/spack/repos/builtin/packages/py-tensorflow/package.py index 20fdd358a6..5de25247e2 100644 --- a/var/spack/repos/builtin/packages/py-tensorflow/package.py +++ b/var/spack/repos/builtin/packages/py-tensorflow/package.py @@ -20,6 +20,10 @@ class PyTensorflow(Package, CudaPackage, ROCmPackage, PythonExtension): license("Apache-2.0") + version("2.16.1", sha256="c729e56efc945c6df08efe5c9f5b8b89329c7c91b8f40ad2bb3e13900bd4876d") + version("2.15.1", sha256="f36416d831f06fe866e149c7cd752da410a11178b01ff5620e9f265511ed57cf") + version("2.15.0", sha256="9cec5acb0ecf2d47b16891f8bc5bc6fbfdffe1700bdadc0d9ebe27ea34f0c220") + version("2.14.1", sha256="6b31ed347ed7a03c45b906aa41628ac91c3db7c84cb816971400d470e58ba494") version("2.14.0", sha256="ce357fd0728f0d1b0831d1653f475591662ec5bca736a94ff789e6b1944df19f") version("2.13.1", sha256="89c07aebd4f41fbe0d08cc88aef00305542134f2f16d3b62918dc3c1182f33e2") version("2.13.0", sha256="e58c939079588623e6fa1d054aec2f90f95018266e0a970fd353a5244f5173dc") @@ -111,14 +115,16 @@ class PyTensorflow(Package, CudaPackage, ROCmPackage, PythonExtension): extends("python") # Python support based on wheel availability - depends_on("python@3.9:3.11", when="@2.14:", type=("build", "run")) + depends_on("python@3.9:3.12", when="@2.16:", type=("build", "run")) + depends_on("python@3.9:3.11", when="@2.14:2.15", type=("build", "run")) depends_on("python@3.8:3.11", when="@2.12:2.13", type=("build", "run")) depends_on("python@:3.10", when="@2.8:2.11", type=("build", "run")) depends_on("python@:3.9", when="@2.5:2.7", type=("build", "run")) depends_on("python@:3.8", when="@2.2:2.4", type=("build", "run")) # See .bazelversion - depends_on("bazel@6.1.0", type="build", when="@2.14:") + depends_on("bazel@6.5.0", type="build", when="@2.16:") + depends_on("bazel@6.1.0", type="build", when="@2.14:2.15") depends_on("bazel@5.3.0", type="build", when="@2.11:2.13") depends_on("bazel@5.1.1", type="build", when="@2.10") # See _TF_MIN_BAZEL_VERSION and _TF_MAX_BAZEL_VERSION in configure.py @@ -159,7 +165,8 @@ class PyTensorflow(Package, CudaPackage, ROCmPackage, PythonExtension): depends_on("py-google-pasta@0.2:0", type=("build", "run"), when="@2.4:2.6") depends_on("py-google-pasta@0.1.8:", type=("build", "run"), when="@2.2:2.3") depends_on("py-google-pasta@0.1.6:", type=("build", "run"), when="@:2.1") - depends_on("py-h5py@2.9:", type=("build", "run"), when="@2.7:") + depends_on("py-h5py@3.10:", type=("build", "run"), when="@2.16:") + depends_on("py-h5py@2.9:", type=("build", "run"), when="@2.7:2.15") depends_on("py-h5py@3.1", type=("build", "run"), when="@2.5:2.6") depends_on("py-h5py@2.10", type=("build", "run"), when="@2.2:2.4") depends_on("py-h5py@:2.10.0", type=("build", "run"), when="@2.1.3:2.1") @@ -170,7 +177,9 @@ class PyTensorflow(Package, CudaPackage, ROCmPackage, PythonExtension): depends_on("hdf5~mpi", type="build", when="@2.1.3:~mpi") depends_on("py-libclang@13:", type=("build", "run"), when="@2.9:") depends_on("py-libclang@9.0.1:", type=("build", "run"), when="@2.7:2.8") - depends_on("py-ml-dtypes@0.2.0", type=("build", "run"), when="@2.14:") + depends_on("py-ml-dtypes@0.3.1:0.3", type=("build", "run"), when="@2.15.1:") + depends_on("py-ml-dtypes@0.2", type=("build", "run"), when="@2.15.0") + depends_on("py-ml-dtypes@0.2.0", type=("build", "run"), when="@2.14") depends_on("py-numpy@1.23.5:", type=("build", "run"), when="@2.14:") depends_on("py-numpy@1.22:1.24.3", type=("build", "run"), when="@2.13:") depends_on("py-numpy@1.22:1.23", type=("build", "run"), when="@2.12") @@ -190,6 +199,8 @@ class PyTensorflow(Package, CudaPackage, ROCmPackage, PythonExtension): depends_on("py-protobuf@:3.19", type=("build", "run"), when="@:2.11") # Must be matching versions of py-protobuf and protobuf conflicts("^py-protobuf~cpp") + depends_on("py-requests@2.21:2", type=("build", "run"), when="@2.16:") + depends_on("py-requests", type=("build", "run")) depends_on("py-setuptools", type=("build", "run")) depends_on("py-six@1.12:", type=("build", "run"), when="@:2.3,2.7:") depends_on("py-six@1.15", type=("build", "run"), when="@2.4:2.6") @@ -198,8 +209,8 @@ class PyTensorflow(Package, CudaPackage, ROCmPackage, PythonExtension): depends_on("py-typing-extensions@3.6.6:", type=("build", "run"), when="@2.7:2.12,2.14:") depends_on("py-typing-extensions@3.6.6:4.5", type=("build", "run"), when="@2.13") depends_on("py-typing-extensions@3.7.4:3.7", type=("build", "run"), when="@2.4:2.6") - depends_on("py-wrapt@1.11:1.14", type=("build", "run"), when="@2.12,2.14:") - depends_on("py-wrapt@1.11:", type=("build", "run"), when="@2.7:2.11,2.13") + depends_on("py-wrapt@1.11:", type=("build", "run"), when="@2.7:2.11,2.13,2.16:") + depends_on("py-wrapt@1.11:1.14", type=("build", "run"), when="@2.12,2.14:2.15") depends_on("py-wrapt@1.12.1:1.12", type=("build", "run"), when="@2.4:2.6") depends_on("py-wrapt@1.11.1:", type=("build", "run"), when="@:2.3") @@ -215,7 +226,7 @@ class PyTensorflow(Package, CudaPackage, ROCmPackage, PythonExtension): depends_on("py-grpcio@1.32", type=("build", "run"), when="@2.4") depends_on("py-grpcio@1.8.6:", type=("build", "run"), when="@:2.3") - for minor_ver in range(5, 15): + for minor_ver in range(5, 17): depends_on( "py-tensorboard@2.{}".format(minor_ver), type=("build", "run"), @@ -226,8 +237,8 @@ class PyTensorflow(Package, CudaPackage, ROCmPackage, PythonExtension): # depends_on('py-tensorflow-estimator') # circular dep # depends_on('py-keras') # circular dep - # Undocumented dependencies - depends_on("py-requests", type=("build", "run")) + # tensorflow/tools/pip_package/build_pip_package.sh + depends_on("patchelf", when="@2.13: platform=linux", type="build") # https://github.com/tensorflow/tensorflow/issues/60179#issuecomment-1491238631 depends_on("coreutils", when="@2.13: platform=darwin", type="build") @@ -247,12 +258,27 @@ class PyTensorflow(Package, CudaPackage, ROCmPackage, PythonExtension): depends_on("curl", when="+gcp") # depends_on('computecpp', when='+opencl+computecpp') # depends_on('trisycl', when='+opencl~computepp') - depends_on("cuda@:10.2", when="+cuda @:2.3") - depends_on("cuda@:11.4", when="+cuda @2.4:2.7") - # avoid problem fixed by commit a76f797b9cd4b9b15bec4c503b16236a804f676f - depends_on("cuda@:11.7.0", when="+cuda @:2.9") - depends_on("cudnn", when="+cuda") - depends_on("cudnn@:7", when="@:2.2 +cuda") + with when("+cuda"): + # https://www.tensorflow.org/install/source#gpu + depends_on("cuda@12.3:", when="@2.16:") + depends_on("cuda@12.2:", when="@2.15:") + depends_on("cuda@11.8:", when="@2.12:") + depends_on("cuda@11.2:", when="@2.5:") + depends_on("cuda@11.0:", when="@2.4:") + depends_on("cuda@10.1:", when="@2.1:") + + depends_on("cuda@:11.7.0", when="@:2.9") + depends_on("cuda@:11.4", when="@2.4:2.7") + depends_on("cuda@:10.2", when="@:2.3") + + depends_on("cudnn@8.9:", when="@2.15:") + depends_on("cudnn@8.7:", when="@2.14:") + depends_on("cudnn@8.6:", when="@2.12:") + depends_on("cudnn@8.1:", when="@2.5:") + depends_on("cudnn@8.0:", when="@2.4:") + depends_on("cudnn@7.6:", when="@2.1:") + + depends_on("cudnn@:7", when="@:2.2") # depends_on('tensorrt', when='+tensorrt') depends_on("nccl", when="+nccl+cuda") depends_on("mpi", when="+mpi")