From 4208cf66bea84c8c00c28dbd495c85087be3e49c Mon Sep 17 00:00:00 2001 From: vsoch Date: Fri, 9 Jul 2021 21:32:38 -0600 Subject: [PATCH] `spack style`: automatically bootstrap dependencies This uses our bootstrapping logic to automatically install dependencies for `spack style`. Users should no longer have to pre-install all of the tools (`isort`, `mypy`, `black`, `flake8`). The command will do it for them. - [x] add logic to bootstrap specs with specific version requirements in `spack style` - [x] remove style tools from CI requirements (to ensure we test bootstrapping) - [x] rework dependencies for `mypy` and `py-typed-ast` - `py-typed-ast` needs to be a link dependency - it needs to be at 1.4.1 or higher to work with python 3.9 Signed-off-by: vsoch --- .github/workflows/unit_tests.yaml | 3 +- lib/spack/docs/developer_guide.rst | 23 +++++++++++++++ lib/spack/spack/cmd/style.py | 28 +++++++++++++------ share/spack/qa/run-style-tests | 1 - .../repos/builtin/packages/py-mypy/package.py | 6 ++-- .../builtin/packages/py-typed-ast/package.py | 6 +++- 6 files changed, 52 insertions(+), 15 deletions(-) diff --git a/.github/workflows/unit_tests.yaml b/.github/workflows/unit_tests.yaml index b924362a25..9caef56566 100644 --- a/.github/workflows/unit_tests.yaml +++ b/.github/workflows/unit_tests.yaml @@ -39,7 +39,7 @@ jobs: python-version: 3.9 - name: Install Python packages run: | - pip install --upgrade pip six setuptools flake8 isort>=4.3.5 mypy>=0.900 black types-six + pip install --upgrade pip six setuptools types-six - name: Setup git configuration run: | # Need this for the git tests to succeed. @@ -370,7 +370,6 @@ jobs: run: | pip install --upgrade pip six setuptools pip install --upgrade codecov coverage[toml] - pip install --upgrade flake8 isort>=4.3.5 mypy>=0.900 - name: Setup Homebrew packages run: | brew install dash fish gcc gnupg2 kcov diff --git a/lib/spack/docs/developer_guide.rst b/lib/spack/docs/developer_guide.rst index 31311e87f7..71aac1fd93 100644 --- a/lib/spack/docs/developer_guide.rst +++ b/lib/spack/docs/developer_guide.rst @@ -818,6 +818,29 @@ Developer commands ``spack doc`` ^^^^^^^^^^^^^ +.. _cmd-spack-style: + +^^^^^^^^^^^^^^^ +``spack style`` +^^^^^^^^^^^^^^^ + +spack style exists to help the developer user to check imports and style with +mypy, flake8, isort, and (soon) black. To run all style checks, simply do: + +.. code-block:: console + + $ spack style + +To run automatic fixes for isort you can do: + +.. code-block:: console + + $ spack style --fix + +You do not need any of these Python packages installed on your system for +the checks to work! Spack will bootstrap install them from packages for +your use. + ^^^^^^^^^^^^^^^^^^^ ``spack unit-test`` ^^^^^^^^^^^^^^^^^^^ diff --git a/lib/spack/spack/cmd/style.py b/lib/spack/spack/cmd/style.py index 0eceb4dc2d..2841a4af0f 100644 --- a/lib/spack/spack/cmd/style.py +++ b/lib/spack/spack/cmd/style.py @@ -14,6 +14,7 @@ import llnl.util.tty.color as color from llnl.util.filesystem import working_dir +import spack.bootstrap import spack.paths from spack.util.executable import which @@ -44,9 +45,15 @@ def grouper(iterable, n, fillvalue=None): #: List of directories to exclude from checks. exclude_directories = [spack.paths.external_path] -#: order in which tools should be run. flake8 is last so that it can +#: Order in which tools should be run. flake8 is last so that it can #: double-check the results of other tools (if, e.g., --fix was provided) -tool_order = ["isort", "mypy", "black", "flake8"] +#: The list maps an executable name to a spack spec needed to install it. +tool_order = [ + ("isort", "py-isort@4.3.5:"), + ("mypy", "py-mypy@0.900:"), + ("black", "py-black"), + ("flake8", "py-flake8"), +] #: tools we run in spack style tools = {} @@ -210,7 +217,7 @@ def translate(match): def print_style_header(file_list, args): - tools = [tool for tool in tool_order if getattr(args, tool)] + tools = [tool for tool, _ in tool_order if getattr(args, tool)] tty.msg("Running style checks on spack:", "selected: " + ", ".join(tools)) # translate modified paths to cwd_relative if needed @@ -338,15 +345,20 @@ def prefix_relative(path): # run tools in order defined in tool_order returncode = 0 - for tool_name in tool_order: + for tool_name, tool_spec in tool_order: if getattr(args, tool_name): run_function, required = tools[tool_name] print_tool_header(tool_name) - cmd = which(tool_name, required=required) - if not cmd: - color.cprint(" @y{%s not in PATH, skipped}" % tool_name) - continue + # Bootstrap tools so we don't need to require install + with spack.bootstrap.ensure_bootstrap_configuration(): + spec = spack.spec.Spec(tool_spec) + cmd = spack.bootstrap.get_executable( + tool_name, spec=spec, install=True + ) + if not cmd: + color.cprint(" @y{%s not in PATH, skipped}" % tool_name) + continue returncode |= run_function(cmd, file_list, args) diff --git a/share/spack/qa/run-style-tests b/share/spack/qa/run-style-tests index 4eb503dae2..86abeeeac1 100755 --- a/share/spack/qa/run-style-tests +++ b/share/spack/qa/run-style-tests @@ -15,7 +15,6 @@ # run-flake8-tests # . "$(dirname $0)/setup.sh" -check_dependencies flake8 mypy # verify that the code style is correct spack style --root-relative diff --git a/var/spack/repos/builtin/packages/py-mypy/package.py b/var/spack/repos/builtin/packages/py-mypy/package.py index a5d2cc15e3..1725986b4f 100644 --- a/var/spack/repos/builtin/packages/py-mypy/package.py +++ b/var/spack/repos/builtin/packages/py-mypy/package.py @@ -13,15 +13,15 @@ class PyMypy(PythonPackage): pypi = "mypy/mypy-0.740.tar.gz" version('0.910', sha256='704098302473cb31a218f1775a873b376b30b4c18229421e9e9dc8916fd16150') + version('0.900', sha256='65c78570329c54fb40f956f7645e2359af5da9d8c54baa44f461cdc7f4984108') version('0.800', sha256='e0202e37756ed09daf4b0ba64ad2c245d357659e014c3f51d8cd0681ba66940a') version('0.740', sha256='48c8bc99380575deb39f5d3400ebb6a8a1cb5cc669bbba4d3bb30f904e0a0e7d') variant('python2', default=False, description='Enable checking of python 2 type annotations') - depends_on('python@3.5:', type=('build', 'run')) + depends_on("python@3.5:", type=("build", "run")) depends_on('py-setuptools', type=('build', 'run')) - depends_on('py-typed-ast@1.4.0:1.4.999', when='@0.900:+python2', type=('build', 'run')) - depends_on('py-typed-ast@1.4.0:1.4.999', when='@:899', type=('build', 'run')) + depends_on('py-typed-ast@1.4.0:1.4.999', type=('build', 'run')) depends_on('py-typing-extensions@3.7.4:', type=('build', 'run')) depends_on('py-mypy-extensions@0.4.3:0.4.999', type=('build', 'run')) depends_on('py-toml', when='@0.900:', type=('build', 'run')) diff --git a/var/spack/repos/builtin/packages/py-typed-ast/package.py b/var/spack/repos/builtin/packages/py-typed-ast/package.py index 18a28505a8..2de58678a3 100644 --- a/var/spack/repos/builtin/packages/py-typed-ast/package.py +++ b/var/spack/repos/builtin/packages/py-typed-ast/package.py @@ -12,7 +12,11 @@ class PyTypedAst(PythonPackage): homepage = "https://github.com/python/typed_ast" pypi = "typed-ast/typed_ast-1.4.0.tar.gz" + version('1.4.3', sha256='fb1bbeac803adea29cedd70781399c99138358c26d05fcbd23c13016b7f5ec65') + version('1.4.2', sha256='9fc0b3cb5d1720e7141d103cf4819aea239f7d136acf9ee4a69b047b7986175a') + version('1.4.1', sha256='8c8aaad94455178e3187ab22c8b01a3837f8ee50e09cf31f1ba129eb293ec30b') version('1.4.0', sha256='66480f95b8167c9c5c5c87f32cf437d585937970f3fc24386f313a4c97b44e34') - depends_on('python@3.3:', type=('build', 'run')) + depends_on('python@3.3:', type=('build', 'link', 'run')) + depends_on('python@:3.8', when="@:1.4.0") # build errors with 3.9 until 1.4.1 depends_on('py-setuptools', type='build')