From f802b64e7a6071f36c928150351c1ac30bc83d4b Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Thu, 13 Jul 2023 14:42:07 -0500 Subject: [PATCH] Update TensorFlow ecosystem (#38747) * Update TensorFlow ecosystem * Re-add +cpp * Do not use system protobuf * Let bazel auto-detect macOS SDK version * Unnecessary duplicated dep * Remove unused import --- .../packages/py-flatbuffers/package.py | 1 + .../builtin/packages/py-keras/package.py | 3 +- .../packages/py-tensorboard/package.py | 23 +++- .../builtin/packages/py-tensorflow/package.py | 103 +++--------------- 4 files changed, 43 insertions(+), 87 deletions(-) diff --git a/var/spack/repos/builtin/packages/py-flatbuffers/package.py b/var/spack/repos/builtin/packages/py-flatbuffers/package.py index b03bd2a94d..d87b1cbdc2 100644 --- a/var/spack/repos/builtin/packages/py-flatbuffers/package.py +++ b/var/spack/repos/builtin/packages/py-flatbuffers/package.py @@ -18,6 +18,7 @@ class PyFlatbuffers(PythonPackage): maintainers("gperrotta") + version("23.5.26", sha256="9ea1144cac05ce5d86e2859f431c6cd5e66cd9c78c558317c7955fb8d4c78d89") version("2.0.7", sha256="0ae7d69c5b82bf41962ca5fde9cc43033bc9501311d975fd5a25e8a7d29c1245") version("2.0", sha256="12158ab0272375eab8db2d663ae97370c33f152b27801fa6024e1d6105fd4dd2") version("1.12", sha256="63bb9a722d5e373701913e226135b28a6f6ac200d5cc7b4d919fa38d73b44610") diff --git a/var/spack/repos/builtin/packages/py-keras/package.py b/var/spack/repos/builtin/packages/py-keras/package.py index ba5ca20a2b..40351bf697 100644 --- a/var/spack/repos/builtin/packages/py-keras/package.py +++ b/var/spack/repos/builtin/packages/py-keras/package.py @@ -21,6 +21,7 @@ class PyKeras(PythonPackage): git = "https://github.com/keras-team/keras.git" url = "https://github.com/keras-team/keras/archive/refs/tags/v2.7.0.tar.gz" + version("2.13.1", sha256="b3591493cce75a69adef7b192cec6be222e76e2386d132cd4e34aa190b0ecbd5") version("2.12.0", sha256="6336cebb6b2b0a91f7efd3ff3a9db3a94f2abccf07a40323138afb80826aec62") version("2.11.0", sha256="e7a7c4199ac76ea750d145c1d84ae1b932e68b9bca34e83596bd66b2fc2ad79e") version("2.10.0", sha256="b1d8d9358700f4a585455854a142d88cc987419c1638ef935b440842d593ad04") @@ -60,7 +61,7 @@ class PyKeras(PythonPackage): depends_on("py-pydot", type=("build", "run")) depends_on("py-scipy", type=("build", "run")) depends_on("py-six", type=("build", "run")) - for minor_ver in range(6, 13): + for minor_ver in range(6, 14): depends_on( "py-tensorflow@2.{}".format(minor_ver), type=("build", "run"), diff --git a/var/spack/repos/builtin/packages/py-tensorboard/package.py b/var/spack/repos/builtin/packages/py-tensorboard/package.py index ca98f8ced2..f1cd059f69 100644 --- a/var/spack/repos/builtin/packages/py-tensorboard/package.py +++ b/var/spack/repos/builtin/packages/py-tensorboard/package.py @@ -17,6 +17,26 @@ class PyTensorboard(PythonPackage): maintainers("aweits") + version( + "2.13.0", + sha256="ab69961ebddbddc83f5fa2ff9233572bdad5b883778c35e4fe94bf1798bd8481", + expand=False, + ) + version( + "2.12.3", + sha256="b4a69366784bc347e02fbe7d847e01896a649ca52f8948a11005e205dcf724fb", + expand=False, + ) + version( + "2.12.2", + sha256="811ab0d27a139445836db9fd4f974424602c3dce12379364d379bcba7c783a68", + expand=False, + ) + version( + "2.12.1", + sha256="58f1c2a25b4829b9c48d2b1ec951dedc9325dcd1ea4b0f601d241d2887d0ed65", + expand=False, + ) version( "2.12.0", sha256="3cbdc32448d7a28dc1bf0b1754760c08b8e0e2e37c451027ebd5ff4896613012", @@ -105,7 +125,8 @@ class PyTensorboard(PythonPackage): depends_on("py-grpcio@1.23.3:", type=("build", "run"), when="@2.2") depends_on("py-google-auth@1.6.3:2", type=("build", "run"), when="@2.7:") depends_on("py-google-auth@1.6.3:1", type=("build", "run"), when="@:2.6") - depends_on("py-google-auth-oauthlib@0.4.1:0.4", type=("build", "run")) + depends_on("py-google-auth-oauthlib@0.5:1.0", type=("build", "run"), when="@2.13:") + depends_on("py-google-auth-oauthlib@0.4.1:0.4", type=("build", "run"), when="@:2.12") depends_on("py-markdown@2.6.8:", type=("build", "run")) depends_on("py-numpy@1.12.0:", type=("build", "run")) depends_on("py-protobuf@3.19.6:", type=("build", "run"), when="@2.12:") diff --git a/var/spack/repos/builtin/packages/py-tensorflow/package.py b/var/spack/repos/builtin/packages/py-tensorflow/package.py index 35f33eb919..2296ff40e0 100644 --- a/var/spack/repos/builtin/packages/py-tensorflow/package.py +++ b/var/spack/repos/builtin/packages/py-tensorflow/package.py @@ -6,7 +6,6 @@ import sys import tempfile -from spack.operating_systems.mac_os import macos_version from spack.package import * @@ -30,6 +29,8 @@ class PyTensorflow(Package, CudaPackage, ROCmPackage, PythonExtension): maintainers("adamjstewart", "aweits") import_modules = ["tensorflow"] + version("2.13.0", sha256="e58c939079588623e6fa1d054aec2f90f95018266e0a970fd353a5244f5173dc") + version("2.12.1", sha256="6bc4600cc0b88e9e40f1800096f5bddbbd3b6e5527a030dea631b87f2ae46b5b") version("2.12.0", sha256="c030cb1905bff1d2446615992aad8d8d85cbe90c4fb625cee458c63bf466bc8e") version("2.11.1", sha256="624ed1cc170cdcc19e8a15d8cdde989a9a1c6b0534c90b38a6b2f06fb2963e5f") version("2.11.0", sha256="99c732b92b1b37fc243a559e02f9aef5671771e272758aa4aec7f34dc92dac48") @@ -202,14 +203,13 @@ class PyTensorflow(Package, CudaPackage, ROCmPackage, PythonExtension): depends_on("py-astunparse@1.6:", type=("build", "run"), when="@2.7:") depends_on("py-astunparse@1.6.3:1.6", type=("build", "run"), when="@2.4:2.6") depends_on("py-astunparse@1.6.3", type=("build", "run"), when="@2.2:2.3") - depends_on("py-flatbuffers@2:", type=("build", "run"), when="@2.10:") + depends_on("py-flatbuffers@23.1.21:", type=("build", "run"), when="@2.13:") + depends_on("py-flatbuffers@2:", type=("build", "run"), when="@2.10:2.12") depends_on("py-flatbuffers@1.12:1", type=("build", "run"), when="@2.9") depends_on("py-flatbuffers@1.12:", type=("build", "run"), when="@2.8") depends_on("py-flatbuffers@1.12:2", type=("build", "run"), when="@2.7") depends_on("py-flatbuffers@1.12", type=("build", "run"), when="@2.4:2.6") - # Unpin overly strict dependency requirements to ease concretization of TensorFlow - depends_on("py-gast@0.2.1:", type=("build", "run"), when="@2.9:") - # depends_on("py-gast@0.2.1:0.4.0", type=("build", "run"), when="@2.9:") + depends_on("py-gast@0.2.1:0.4.0", type=("build", "run"), when="@2.9:") depends_on("py-gast@0.2.1:", type=("build", "run"), when="@2.8") depends_on("py-gast@0.2.1:0.4", type=("build", "run"), when="@2.7") depends_on("py-gast@0.4.0", type=("build", "run"), when="@2.5:2.6") @@ -230,10 +230,10 @@ class PyTensorflow(Package, CudaPackage, ROCmPackage, PythonExtension): depends_on("py-h5py~mpi", type=("build", "run"), when="@1.15.5,2.0.4,2.1.3:~mpi") depends_on("hdf5+mpi", type="build", when="@1.15.5,2.0.4,2.1.3:+mpi") depends_on("hdf5~mpi", type="build", when="@1.15.5,2.0.4,2.1.3:~mpi") - depends_on("py-jax@0.3.15:", type=("build", "run"), when="@2.12:") depends_on("py-libclang@13:", type=("build", "run"), when="@2.9:") depends_on("py-libclang@9.0.1:", type=("build", "run"), when="@2.7:2.8") - depends_on("py-numpy@1.22:1.23", type=("build", "run"), when="@2.12:") + depends_on("py-numpy@1.22:1.24.3", type=("build", "run"), when="@2.13:") + depends_on("py-numpy@1.22:1.23", type=("build", "run"), when="@2.12") depends_on("py-numpy@1.20:", type=("build", "run"), when="@2.8:2.11") depends_on("py-numpy@1.14.5:", type=("build", "run"), when="@2.7") depends_on("py-numpy@1.19.2:1.19", type=("build", "run"), when="@2.4:2.6") @@ -261,29 +261,23 @@ class PyTensorflow(Package, CudaPackage, ROCmPackage, PythonExtension): # https://github.com/protocolbuffers/protobuf/issues/10051 # https://github.com/tensorflow/tensorflow/issues/56266 depends_on("py-protobuf@:3.19", type=("build", "run"), when="@:2.11") - depends_on("py-protobuf+cpp", type=("build", "run")) - depends_on("protobuf@:3.21.9", when="@:2.12") - depends_on("protobuf@:3.19", when="@:2.11") - depends_on("protobuf@:3.17", when="@:2.11") - depends_on("protobuf@:3.12", when="@:2.4") - depends_on("protobuf", type=("build", "run")) depends_on("py-setuptools", type=("build", "run")) depends_on("py-six@1.12:", type=("build", "run"), when="@2.1:2.3,2.7:") depends_on("py-six@1.15", type=("build", "run"), when="@2.4:2.6") depends_on("py-six@1.10:", type=("build", "run"), when="@:2.0") depends_on("py-termcolor@1.1:", type=("build", "run"), when="@1.6:2.3,2.7:") depends_on("py-termcolor@1.1", type=("build", "run"), when="@2.4:2.6") - depends_on("py-typing-extensions@3.6.6:", type=("build", "run"), when="@2.7:") + depends_on("py-typing-extensions@3.6.6:4.5", type=("build", "run"), when="@2.13:") + depends_on("py-typing-extensions@3.6.6:", type=("build", "run"), when="@2.7:2.12") depends_on("py-typing-extensions@3.7.4:3.7", type=("build", "run"), when="@2.4:2.6") - depends_on("py-wrapt@1.11:1.14", type=("build", "run"), when="@2.12:") + depends_on("py-wrapt@1.11:", type=("build", "run"), when="@2.13:") + depends_on("py-wrapt@1.11:1.14", type=("build", "run"), when="@2.12") depends_on("py-wrapt@1.11:", type=("build", "run"), when="@2.7:2.11") depends_on("py-wrapt@1.12.1:1.12", type=("build", "run"), when="@2.4:2.6") depends_on("py-wrapt@1.11.1:", type=("build", "run"), when="@1.12.1,1.14:2.3") - # TODO: add package for this dependency - # depends_on('py-tensorflow-io-gcs-filesystem@0.23.1:', - # type=('build', 'run'), when='@2.8:') - # depends_on('py-tensorflow-io-gcs-filesystem@0.21:', - # type=('build', 'run'), when='@2.7') + # TODO: add packages for these dependencies + # depends_on('py-tensorflow-io-gcs-filesystem@0.23.1:', type=('build', 'run'), when='@2.8:') + # depends_on('py-tensorflow-io-gcs-filesystem@0.21:', type=('build', 'run'), when='@2.7') with when("+rocm"): depends_on("hip") depends_on("rocrand") @@ -308,7 +302,7 @@ class PyTensorflow(Package, CudaPackage, ROCmPackage, PythonExtension): depends_on("py-grpcio@1.32", type=("build", "run"), when="@2.4") depends_on("py-grpcio@1.8.6:", type=("build", "run"), when="@1.6:2.3") - for minor_ver in range(5, 13): + for minor_ver in range(5, 14): depends_on( "py-tensorboard@2.{}".format(minor_ver), type=("build", "run"), @@ -321,10 +315,13 @@ class PyTensorflow(Package, CudaPackage, ROCmPackage, PythonExtension): # Undocumented dependencies depends_on("py-requests", type=("build", "run")) + # https://github.com/tensorflow/tensorflow/issues/60179#issuecomment-1491238631 + depends_on("coreutils", when="@2.13: platform=darwin", type="build") # No longer a dependency in latest versions depends_on("py-astor@0.6:", type=("build", "run"), when="@1.6:2.1") depends_on("py-backports-weakref@1.0rc1", type=("build", "run"), when="@1.2") + depends_on("py-jax@0.3.15:", type=("build", "run"), when="@2.12") depends_on("py-keras-applications@1.0.8:", type=("build", "run"), when="@1.15:2.1") depends_on("py-keras-applications@1.0.6:", type=("build", "run"), when="@1.12:1.14") depends_on("py-keras-applications@1.0.5:", type=("build", "run"), when="@1.11") @@ -753,67 +750,6 @@ def setup_build_environment(self, env): tmp_path = tempfile.mkdtemp(prefix="spack") env.set("TEST_TMPDIR", tmp_path) - env.set("TF_SYSTEM_LIBS", "com_google_protobuf") - if spec.satisfies("@:2.3"): - # NOTE: INCLUDEDIR is not just relevant to protobuf - # see third_party/systemlibs/jsoncpp.BUILD - env.set("INCLUDEDIR", spec["protobuf"].prefix.include) - - def patch(self): - filter_file( - '"-U_FORTIFY_SOURCE",', - '"-U_FORTIFY_SOURCE", "-I%s",' % self.spec["protobuf"].prefix.include, - "third_party/gpus/crosstool/BUILD.rocm.tpl", - ) - if self.spec.satisfies("@2.12:"): - filter_file( - 'genproto_deps.append("@com_google_protobuf//:well_known_types_py_pb2_genproto")', - "pass", - "tensorflow/tsl/platform/default/build_config.bzl", - string=True, - ) - if self.spec.satisfies("@2.11:"): - filter_file( - "deps = protodeps + well_known_proto_libs(),", - "deps = protodeps,", - "tensorflow/tsl/platform/default/build_config.bzl", - string=True, - ) - if self.spec.satisfies("@2.3:2.10"): - filter_file( - "deps = protodeps + well_known_proto_libs(),", - "deps = protodeps,", - "tensorflow/core/platform/default/build_config.bzl", - string=True, - ) - if self.spec.satisfies("@2.4.0:2.5"): - text = """ -def protobuf_deps(): - pass -""" - with open("third_party/systemlibs/protobuf_deps.bzl", "w") as f: - f.write(text) - - if self.spec.satisfies("@2.5.0"): - file_to_patch = "tensorflow/workspace2.bzl" - else: - file_to_patch = "tensorflow/workspace.bzl" - - filter_file( - '"//third_party/systemlibs:protobuf.bzl": "protobuf.bzl",', - '"//third_party/systemlibs:protobuf.bzl": "protobuf.bzl",\n' - '"//third_party/systemlibs:protobuf_deps.bzl": "protobuf_deps.bzl",', # noqa: E501 - file_to_patch, - string=True, - ) - - # Set protobuf path - filter_file( - r"(^build:linux --define=PROTOBUF_INCLUDE_PATH=).*", - r"\1{0}".format(self.spec["protobuf"].prefix.include), - ".bazelrc", - ) - def configure(self, spec, prefix): # NOTE: configure script is interactive. If you set the appropriate # environment variables, this interactivity is skipped. If you don't, @@ -1009,9 +945,6 @@ def build(self, spec, prefix): # removed in bazel 3.6 args.append("--incompatible_no_support_tools_in_action_inputs=false") - if spec.satisfies("@2.9: platform=darwin"): - args.append("--macos_sdk_version={}".format(macos_version())) - # See .bazelrc for when each config flag is supported if spec.satisfies("@1.12.1:"): if "+mkl" in spec: