From 1427ddaa59df661e9fe3f4b8939eb92fd1b83a88 Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Sat, 10 Sep 2022 15:25:44 +0200 Subject: [PATCH] ci: restore coverage computation (#32585) * ci: restore coverage computation * Mark "test_foreground_background" as xfail * Mark "test_foreground_background_output" as xfail * Make number of processes explicit, remove verbosity on linux * Run coverage on just 3 Python jobs for linux * Run coverage on just 3 Python jobs for linux * Run coverage on just 2 Python jobs for linux * Add back verbose, since before we didn't encounter the xdist internal error * Reduce the workers to 2 * Try to use command line --- .github/workflows/unit_tests.yaml | 28 ++++++++++++----------- lib/spack/spack/test/cmd/debug.py | 4 +++- lib/spack/spack/test/llnl/util/tty/log.py | 2 ++ pyproject.toml | 1 + share/spack/qa/run-unit-tests | 22 ++++++++++++++---- 5 files changed, 38 insertions(+), 19 deletions(-) diff --git a/.github/workflows/unit_tests.yaml b/.github/workflows/unit_tests.yaml index 5de7d86c56..c3c2b3242e 100644 --- a/.github/workflows/unit_tests.yaml +++ b/.github/workflows/unit_tests.yaml @@ -52,7 +52,7 @@ jobs: patchelf cmake bison libbison-dev kcov - name: Install Python packages run: | - pip install --upgrade pip six setuptools pytest codecov "coverage[toml]<=6.2" pytest-xdist + pip install --upgrade pip six setuptools pytest codecov[toml] pytest-cov pytest-xdist # ensure style checks are not skipped in unit tests for python >= 3.6 # note that true/false (i.e., 1/0) are opposite in conditions in python and bash if python -c 'import sys; sys.exit(not sys.version_info >= (3, 6))'; then @@ -75,16 +75,18 @@ jobs: . share/spack/setup-env.sh spack bootstrap untrust spack-install spack -v solve zlib - - name: Run unit tests (full suite with coverage) + - name: Run unit tests env: SPACK_PYTHON: python - COVERAGE: true SPACK_TEST_SOLVER: ${{ matrix.concretizer }} + SPACK_TEST_PARALLEL: 2 + COVERAGE: true + UNIT_TEST_COVERAGE: ${{ (matrix.concretizer == 'original' && matrix.python-version == '2.7') || (matrix.python-version == '3.10') }} run: | share/spack/qa/run-unit-tests - coverage combine + coverage combine -a coverage xml - - uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # @v2.1.0 + - uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 with: flags: unittests,linux,${{ matrix.concretizer }} # Test shell integration @@ -115,7 +117,7 @@ jobs: COVERAGE: true run: | share/spack/qa/run-shell-tests - - uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # @v2.1.0 + - uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 with: flags: shelltests,linux @@ -163,7 +165,7 @@ jobs: patchelf kcov - name: Install Python packages run: | - pip install --upgrade pip six setuptools pytest codecov coverage[toml]==6.2 clingo pytest-xdist + pip install --upgrade pip six setuptools pytest codecov coverage[toml] pytest-cov clingo pytest-xdist - name: Setup git configuration run: | # Need this for the git tests to succeed. @@ -175,7 +177,7 @@ jobs: SPACK_TEST_SOLVER: clingo run: | share/spack/qa/run-unit-tests - coverage combine + coverage combine -a coverage xml - uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # @v2.1.0 with: @@ -196,13 +198,14 @@ jobs: - name: Install Python packages run: | pip install --upgrade pip six setuptools - pip install --upgrade pytest codecov coverage[toml]==6.2 pytest-xdist + pip install --upgrade pytest codecov coverage[toml] pytest-xdist pytest-cov - name: Setup Homebrew packages run: | brew install dash fish gcc gnupg2 kcov - name: Run unit tests env: SPACK_TEST_SOLVER: clingo + SPACK_TEST_PARALLEL: 4 run: | git --version . .github/workflows/setup_git.sh @@ -210,13 +213,12 @@ jobs: $(which spack) bootstrap untrust spack-install $(which spack) solve zlib common_args=(--dist loadfile --tx '4*popen//python=./bin/spack-tmpconfig python -u ./bin/spack python' -x) - coverage run $(which spack) unit-test "${common_args[@]}" - coverage combine + $(which spack) unit-test --cov --cov-config=pyproject.toml "${common_args[@]}" + coverage combine -a coverage xml # Delete the symlink going from ./lib/spack/docs/_spack_root back to # the initial directory, since it causes ELOOP errors with codecov/actions@2 rm lib/spack/docs/_spack_root - - uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # @v2.1.0 + - uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 with: - files: ./coverage.xml flags: unittests,macos diff --git a/lib/spack/spack/test/cmd/debug.py b/lib/spack/spack/test/cmd/debug.py index ec52234517..681987f873 100644 --- a/lib/spack/spack/test/cmd/debug.py +++ b/lib/spack/spack/test/cmd/debug.py @@ -27,7 +27,9 @@ def test_create_db_tarball(tmpdir, database): # get the first non-dotfile to avoid coverage files in the directory files = os.listdir(os.getcwd()) - tarball_name = next(f for f in files if not f.startswith(".")) + tarball_name = next( + f for f in files if not f.startswith(".") and not f.startswith("tests") + ) # debug command made an archive assert os.path.exists(tarball_name) diff --git a/lib/spack/spack/test/llnl/util/tty/log.py b/lib/spack/spack/test/llnl/util/tty/log.py index b100f209e8..333088b6af 100644 --- a/lib/spack/spack/test/llnl/util/tty/log.py +++ b/lib/spack/spack/test/llnl/util/tty/log.py @@ -341,6 +341,7 @@ def no_termios(): (mock_shell_tstp_tstp_cont_cont, no_termios), ], ) +@pytest.mark.xfail(reason="Fails almost consistently when run with coverage and xdist") def test_foreground_background(test_fn, termios_on_or_off, tmpdir): """Functional tests for foregrounding and backgrounding a logged process. @@ -460,6 +461,7 @@ def mock_shell_v_v_no_termios(proc, ctl, **kwargs): (mock_shell_v_v_no_termios, no_termios), ], ) +@pytest.mark.xfail(reason="Fails almost consistently when run with coverage and xdist") def test_foreground_background_output(test_fn, capfd, termios_on_or_off, tmpdir): """Tests hitting 'v' toggles output, and that force_echo works.""" if sys.version_info >= (3, 8) and sys.platform == "darwin" and termios_on_or_off == no_termios: diff --git a/pyproject.toml b/pyproject.toml index 446c7891f2..f5fed2df4b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -90,6 +90,7 @@ parallel = true concurrency = ["multiprocessing"] branch = true source = ["bin", "lib"] +data_file = "./tests-coverage/.coverage" omit = [ 'lib/spack/spack/test/*', 'lib/spack/docs/*', diff --git a/share/spack/qa/run-unit-tests b/share/spack/qa/run-unit-tests index da8334f76b..2de6b6101d 100755 --- a/share/spack/qa/run-unit-tests +++ b/share/spack/qa/run-unit-tests @@ -56,13 +56,25 @@ if python -m pytest --trace-config 2>&1 | grep xdist; then export PYTEST_ADDOPTS="$PYTEST_ADDOPTS --dist loadfile --tx '${SPACK_TEST_PARALLEL:=3}*popen//python=./bin/spack-tmpconfig python -u ./bin/spack python'" fi -$coverage_run $(which spack) unit-test -x --verbose +# We are running pytest-cov after the addition of pytest-xdist, since it integrates +# other pugins for pytest automatically. We still need to use "coverage" explicitly +# for the commands above. +# +# There is a need to pass the configuration file explicitly due to a bug: +# https://github.com/pytest-dev/pytest-cov/issues/243 +# https://github.com/pytest-dev/pytest-cov/issues/237 +# where it seems that otherwise the configuration file might not be located by subprocesses +# in some, not better specified, cases. +if [[ "$UNIT_TEST_COVERAGE" == "true" ]]; then + $(which spack) unit-test -x --verbose --cov --cov-config=pyproject.toml +else + $(which spack) unit-test -x --verbose +fi + + bash "$QA_DIR/test-env-cfg.sh" # Delete the symlink going from ./lib/spack/docs/_spack_root back to # the initial directory, since it causes ELOOP errors with codecov/actions@2 -if [[ "$COVERAGE" == "true" ]]; then - rm lib/spack/docs/_spack_root -fi - +rm lib/spack/docs/_spack_root