From 76fbb8cd8f1ceecbc9eb36f5ee2d78f502660fda Mon Sep 17 00:00:00 2001 From: Matt Drozt Date: Thu, 24 Aug 2023 15:03:52 -0700 Subject: [PATCH] [py-smartsim] New Package (#39306) * Create a smartsim package * rm ss 0.4.2 * no py upper bound, add build dep * add setup_build_env * add comment to find ml deps lower bounds * Apply suggestions from code review Correct dep versions, use `python_purelib` Co-authored-by: Adam J. Stewart * remove the cuda/rocm vars * point editors to bin deps version constraints * Apply suggestions from code review Loosen `py-smartredis` constraint, enforce `setup.cfg` py version Co-authored-by: Adam J. Stewart * style * rm rai lower bound * lower bound setuptools Co-authored-by: Adam J. Stewart --------- Co-authored-by: Adam J. Stewart --- .../builtin/packages/py-smartsim/package.py | 83 +++++++++++++++++++ .../ss-0-5-0-remove-cli-build-fns.patch | 22 +++++ .../py-smartsim/ss-dont-build-db.patch | 27 ++++++ 3 files changed, 132 insertions(+) create mode 100644 var/spack/repos/builtin/packages/py-smartsim/package.py create mode 100644 var/spack/repos/builtin/packages/py-smartsim/ss-0-5-0-remove-cli-build-fns.patch create mode 100644 var/spack/repos/builtin/packages/py-smartsim/ss-dont-build-db.patch diff --git a/var/spack/repos/builtin/packages/py-smartsim/package.py b/var/spack/repos/builtin/packages/py-smartsim/package.py new file mode 100644 index 0000000000..4011e0c8dc --- /dev/null +++ b/var/spack/repos/builtin/packages/py-smartsim/package.py @@ -0,0 +1,83 @@ +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import os + +from spack.package import * + + +class PySmartsim(PythonPackage): + """A workflow library to easily integrate machine learning libraries with + high performance computing simulations and applications + """ + + homepage = "https://www.craylabs.org/docs/overview.html" + git = "https://github.com/CrayLabs/SmartSim" + pypi = "smartsim/smartsim-0.5.0.tar.gz" + + maintainers("MattToast") + + version("0.5.0", sha256="35b36243dc84af62261a7f772bae92f0b3502faf01401423899cb2a48339858c") + + variant("torch", default=True, description="Build with the pytorch backend") + + depends_on("python@3.8:3.10", type=("build", "run")) + depends_on("py-setuptools@39.2:", type=("build",)) + depends_on("py-cmake@3.13:", type=("build",)) + + depends_on("py-psutil@5.7.2:", type=("build", "run")) + depends_on("py-coloredlogs@10:", type=("build", "run")) + depends_on("py-tabulate@0.8.9:", type=("build", "run")) + depends_on("py-redis@4.5:", type=("build", "run")) + depends_on("py-tqdm@4.50.2:", type=("build", "run")) + depends_on("py-filelock@3.4.2:", type=("build", "run")) + depends_on("py-protobuf@3.20:3", type=("build", "run")) + + # Companion libs + depends_on("py-smartredis@0.4.1:", type=("build", "run"), when="@0.5.0") + + # Backends + # SmartSim defines sensible and well tested lower bounds for redis + # for the CLI to fetch in the `smartsim._core._install.buildenv.Versioner` + # class (lower versions are unable to parse the default `redis.conf` shipped + # with SmartSim), but allows users to upgrade explicitly by setting + # environment variables. + depends_on("redis@7.0.5:", type=("build", "run")) + depends_on("redis-ai", type=("build", "run")) + + # ML Deps + # The lower bound for these py-* deps can be found in the source code + # at `smartsim/_core/_install/buildenv.py`. + with when("+torch"): + depends_on("redis-ai+torch", type=("build", "run")) + depends_on("py-torch@1.11.0:", type=("build", "run")) + depends_on("py-torchvision@0.12.0:", type=("build", "run")) + + # By default, the SmartSim `setup.py` will attempt to fetch and build + # its own copy of Redis. This should be patched out and the version of + # Redis retrieved through spack should be used instead. + patch("ss-dont-build-db.patch") + + # SmartSim provides its own CLI to fetch and build its own + # copy of Redis, RedisAI, and ML deps. This functionality should be + # patched out so that users do not accidentally overwrite/break + # dependencies fetched though Spack + patch("ss-0-5-0-remove-cli-build-fns.patch") + + def setup_build_environment(self, env): + env.set("BUILD_JOBS", make_jobs) + + @run_after("install") + def symlink_bin_deps(self): + ss_core_path = join_path(python_purelib, "smartsim", "_core") + os.symlink( + self.spec["redis"].prefix.bin.join("redis-server"), + join_path(ss_core_path, "bin", "redis-server"), + ) + os.symlink( + self.spec["redis"].prefix.bin.join("redis-cli"), + join_path(ss_core_path, "bin", "redis-cli"), + ) + os.symlink(self.spec["redis-ai"].prefix, join_path(ss_core_path, "lib")) diff --git a/var/spack/repos/builtin/packages/py-smartsim/ss-0-5-0-remove-cli-build-fns.patch b/var/spack/repos/builtin/packages/py-smartsim/ss-0-5-0-remove-cli-build-fns.patch new file mode 100644 index 0000000000..4dbb0f4273 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-smartsim/ss-0-5-0-remove-cli-build-fns.patch @@ -0,0 +1,22 @@ +diff --git a/smartsim/_core/_cli/cli.py b/smartsim/_core/_cli/cli.py +index 7dbb144..93da137 100644 +--- a/smartsim/_core/_cli/cli.py ++++ b/smartsim/_core/_cli/cli.py +@@ -68,14 +68,17 @@ class SmartCli: + getattr(self, args.command)() + + def build(self) -> None: ++ raise RuntimeError("`smart build` not supported through spack install") + Build() + sys.exit(0) + + def clean(self) -> None: ++ raise RuntimeError("`smart clean` not supported through spack install") + Clean() + sys.exit(0) + + def clobber(self) -> None: ++ raise RuntimeError("`smart clobber` not supported through spack install") + Clean(clean_all=True) + sys.exit(0) + diff --git a/var/spack/repos/builtin/packages/py-smartsim/ss-dont-build-db.patch b/var/spack/repos/builtin/packages/py-smartsim/ss-dont-build-db.patch new file mode 100644 index 0000000000..0553a499c5 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-smartsim/ss-dont-build-db.patch @@ -0,0 +1,27 @@ +diff --git a/setup.py b/setup.py +index 6758bec..499567e 100644 +--- a/setup.py ++++ b/setup.py +@@ -134,14 +134,14 @@ class InstallPlatlib(install): + class SmartSimBuild(build_py): + + def run(self): +- database_builder = builder.DatabaseBuilder(build_env(), +- build_env.MALLOC, +- build_env.JOBS) +- if not database_builder.is_built: +- database_builder.build_from_git(versions.REDIS_URL, +- versions.REDIS) +- +- database_builder.cleanup() ++ # database_builder = builder.DatabaseBuilder(build_env(), ++ # build_env.MALLOC, ++ # build_env.JOBS) ++ # if not database_builder.is_built: ++ # database_builder.build_from_git(versions.REDIS_URL, ++ # versions.REDIS) ++ # ++ # database_builder.cleanup() + + # run original build_py command + super().run()