Skip to content

Commit

Permalink
Add PantsIntegrationTest.setup_tmpdir() (pantsbuild#10673)
Browse files Browse the repository at this point in the history 
A common pattern in our integration tests is to set up a tmpdir that simulates a testproject. In fact, we want to delete `examples/` and `testprojects/` in favor of inlined test setups because example folders have several issues like brittleness and confusing users that they are blessed examples of how to use Pants.

See https://www.pantsbuild.org/v2.0/docs/rules-api-testing for how we now document `PantsIntegrationTest` with this utility now added.

[ci skip-rust]
[ci skip-build-wheels]
  • Loading branch information
@Eric-Arellano
Eric-Arellano authored Aug 23, 2020
1 parent 44422d9 commit f7b260e
Show file tree
Hide file tree
Showing 10 changed files with 242 additions and 367 deletions.
295 changes: 125 additions & 170 deletions src/python/pants/backend/python/rules/coverage_integration_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,139 +7,107 @@

from pants.base.build_environment import get_buildroot
from pants.testutil.pants_integration_test import PantsIntegrationTest, PantsResult
from pants.util.contextutil import temporary_dir


class CoverageIntegrationTest(PantsIntegrationTest):
def _prepare_sources(self, tmpdir: str, build_root: str) -> Path:
tmpdir_relative = Path(tmpdir).relative_to(build_root)
src_root = Path(tmpdir, "src", "python", "project")
src_root.mkdir(parents=True)
(src_root / "__init__.py").touch()

# Set up the source files. Only `lib.py` will actually be tested, but we still expect
# `random.py` to show up in the final report correctly.
(src_root / "lib.py").write_text(
dedent(
"""\
def add(x, y):
return x + y
def subtract(x, y):
return x - y
def multiply(x, y):
return x * y
"""
)
)
(src_root / "random.py").write_text(
dedent(
"""\
def capitalize(s):
return s.capitalize()
"""
)
)

sources = {
# Only `lib.py` will actually be tested, but we still expect `random.py` t`o show up in
# the final report correctly.
"src/python/project/__init__.py": "",
"src/python/project/lib.py": dedent(
"""\
def add(x, y):
return x + y
def subtract(x, y):
return x - y
def multiply(x, y):
return x * y
"""
),
"src/python/project/random.py": dedent(
"""\
def capitalize(s):
return s.capitalize()
"""
),
# Only test half of the library.
(src_root / "lib_test.py").write_text(
dedent(
"""\
from project.lib import add
def test_add():
assert add(2, 3) == 5
"""
"src/python/project/lib_test.py": dedent(
"""\
from project.lib import add
def test_add():
assert add(2, 3) == 5
"""
),
"src/python/project/BUILD": dedent(
"""\
python_library()
python_tests(
name="tests",
dependencies=[":project"],
)
)

(src_root / "BUILD").write_text(
dedent(
"""\
python_library()
python_tests(
name="tests",
dependencies=[":project"],
)
"""
"""
),
# Test that a `tests/` source root accurately gets coverage data for the `src/`
# root.
"tests/python/project_test/__init__.py": "",
"tests/python/project_test/test_multiply.py": dedent(
"""\
from project.lib import multiply
def test_multiply():
assert multiply(2, 3) == 6
"""
),
"tests/python/project_test/test_arithmetic.py": dedent(
"""\
from project.lib import add, subtract
def test_arithmetic():
assert add(4, 3) == 7 == subtract(10, 3)
"""
),
"tests/python/project_test/BUILD": dedent(
"""\
python_tests(
name="multiply",
sources=["test_multiply.py"],
dependencies=['{tmpdir}/src/python/project'],
)
)

# Test that a `tests/` source root accurately gets coverage data for the `src/` root.
test_root = Path(tmpdir, "tests", "python", "project_test")
test_root.mkdir(parents=True)
(test_root / "__init__.py").touch()
(test_root / "test_multiply.py").write_text(
dedent(
"""\
from project.lib import multiply
def test_multiply():
assert multiply(2, 3) == 6
"""
python_tests(
name="arithmetic",
sources=["test_arithmetic.py"],
dependencies=['{tmpdir}/src/python/project'],
)
)
(test_root / "test_arithmetic.py").write_text(
dedent(
"""\
from project.lib import add, subtract
def test_arithmetic():
assert add(4, 3) == 7 == subtract(10, 3)
"""
)
)
(test_root / "BUILD").write_text(
dedent(
f"""\
python_tests(
name="multiply",
sources=["test_multiply.py"],
dependencies=['{tmpdir_relative}/src/python/project'],
)
python_tests(
name="arithmetic",
sources=["test_arithmetic.py"],
dependencies=['{tmpdir_relative}/src/python/project'],
)
python_library()
"""
)
)
# Test a file that does not cover any src code. While this is unlikely to happen, this
# tests that we can properly handle the edge case. In particular, we want to make sure
# that coverage still works when we omit this test file through the option
# `--omit-test-sources`.
no_src_folder = Path(tmpdir, "tests", "python", "project_test", "no_src")
no_src_folder.mkdir()
(no_src_folder / "__init__.py").touch()
(no_src_folder / "test_no_src.py").write_text("def test_true():\n\tassert True is True\n")
(no_src_folder / "BUILD").write_text(
dedent(
"""\
python_tests()
python_library(name='lib')
"""
)
)
return tmpdir_relative

def _run_tests(self, tmpdir_relative, *more_args: str) -> PantsResult:
"""
),
# Test a file that does not cover any src code. While this is unlikely to happen,
# this tests that we can properly handle the edge case.
"tests/python/project_test/no_src/__init__.py": "",
"tests/python/project_test/no_src/test_no_src.py": dedent(
"""\
def test_true():
assert True is True
"""
),
"tests/python/project_test/no_src/BUILD.py": "python_tests()",
}

def run_coverage(self, tmpdir: str, *more_args: str) -> PantsResult:
command = [
"--backend-packages=pants.backend.python",
"test",
"--use-coverage",
f"{tmpdir_relative}/src/python/project:tests",
f"{tmpdir_relative}/tests/python/project_test:multiply",
f"{tmpdir_relative}/tests/python/project_test:arithmetic",
f"{tmpdir_relative}/tests/python/project_test/no_src",
f"{tmpdir}/src/python/project:tests",
f"{tmpdir}/tests/python/project_test:multiply",
f"{tmpdir}/tests/python/project_test:arithmetic",
f"{tmpdir}/tests/python/project_test/no_src",
*more_args,
]
command.extend(more_args)
result = self.run_pants(command)
self.assert_success(result)
# Regression test: make sure that individual tests do not complain about failing to
Expand All @@ -149,23 +117,21 @@ def _run_tests(self, tmpdir_relative, *more_args: str) -> PantsResult:
return result

def test_coverage(self) -> None:
build_root = get_buildroot()
with temporary_dir(root_dir=build_root) as tmpdir:
tmpdir_relative = self._prepare_sources(tmpdir, build_root)
result = self._run_tests(tmpdir_relative)
with self.setup_tmpdir(self.sources) as tmpdir:
result = self.run_coverage(tmpdir)
assert (
dedent(
f"""\
Name Stmts Miss Cover
---------------------------------------------------------------------------------
{tmpdir_relative}/src/python/project/__init__.py 0 0 100%
{tmpdir_relative}/src/python/project/lib.py 6 0 100%
{tmpdir_relative}/src/python/project/lib_test.py 3 0 100%
{tmpdir_relative}/tests/python/project_test/__init__.py 0 0 100%
{tmpdir_relative}/tests/python/project_test/no_src/__init__.py 0 0 100%
{tmpdir_relative}/tests/python/project_test/no_src/test_no_src.py 2 0 100%
{tmpdir_relative}/tests/python/project_test/test_arithmetic.py 3 0 100%
{tmpdir_relative}/tests/python/project_test/test_multiply.py 3 0 100%
{tmpdir}/src/python/project/__init__.py 0 0 100%
{tmpdir}/src/python/project/lib.py 6 0 100%
{tmpdir}/src/python/project/lib_test.py 3 0 100%
{tmpdir}/tests/python/project_test/__init__.py 0 0 100%
{tmpdir}/tests/python/project_test/no_src/__init__.py 0 0 100%
{tmpdir}/tests/python/project_test/no_src/test_no_src.py 2 0 100%
{tmpdir}/tests/python/project_test/test_arithmetic.py 3 0 100%
{tmpdir}/tests/python/project_test/test_multiply.py 3 0 100%
---------------------------------------------------------------------------------
TOTAL 17 0 100%
"""
Expand All @@ -174,30 +140,30 @@ def test_coverage(self) -> None:
)

def test_coverage_with_filter(self) -> None:
build_root = get_buildroot()
with temporary_dir(root_dir=build_root) as tmpdir:
tmpdir_relative = self._prepare_sources(tmpdir, build_root)
result = self._run_tests(
tmpdir_relative, "--coverage-py-filter=['project.lib', 'project_test.no_src']"
with self.setup_tmpdir(self.sources) as tmpdir:
result = self.run_coverage(
tmpdir, "--coverage-py-filter=['project.lib', 'project_test.no_src']"
)
assert (
dedent(
f"""\
Name Stmts Miss Cover
---------------------------------------------------------------------------------
{tmpdir_relative}/src/python/project/lib.py 6 0 100%
{tmpdir_relative}/tests/python/project_test/no_src/__init__.py 0 0 100%
{tmpdir_relative}/tests/python/project_test/no_src/test_no_src.py 2 0 100%
{tmpdir}/src/python/project/lib.py 6 0 100%
{tmpdir}/tests/python/project_test/no_src/__init__.py 0 0 100%
{tmpdir}/tests/python/project_test/no_src/test_no_src.py 2 0 100%
---------------------------------------------------------------------------------
TOTAL 8 0 100%
"""
)
in result.stderr
)

def _assert_raw_coverage(self, result: PantsResult, build_root: str) -> None:
def test_coverage_raw(self) -> None:
with self.setup_tmpdir(self.sources) as tmpdir:
result = self.run_coverage(tmpdir, "--coverage-py-report=raw")
assert "Wrote raw coverage report to `dist/coverage/python`" in result.stderr
coverage_data = Path(build_root, "dist", "coverage", "python", ".coverage")
coverage_data = Path(get_buildroot(), "dist", "coverage", "python", ".coverage")
assert coverage_data.exists() is True
conn = sqlite3.connect(coverage_data.as_posix())
cursor = conn.cursor()
Expand All @@ -212,32 +178,21 @@ def _assert_raw_coverage(self, result: PantsResult, build_root: str) -> None:
"tracer",
}

def test_coverage_raw(self) -> None:
build_root = get_buildroot()
with temporary_dir(root_dir=build_root) as tmpdir:
tmpdir_relative = self._prepare_sources(tmpdir, build_root)
result = self._run_tests(tmpdir_relative, "--coverage-py-report=raw")
self._assert_raw_coverage(result, build_root)

def test_coverage_html_xml_json(self) -> None:
build_root = get_buildroot()
with temporary_dir(root_dir=build_root) as tmpdir:
tmpdir_relative = self._prepare_sources(tmpdir, build_root)
result = self._run_tests(
tmpdir_relative, "--coverage-py-report=['xml', 'html', 'json']"
)
coverage_path = Path(build_root, "dist", "coverage", "python")
assert coverage_path.exists() is True

assert "Wrote xml coverage report to `dist/coverage/python`" in result.stderr
xml_coverage = coverage_path / "coverage.xml"
assert xml_coverage.exists() is True

assert "Wrote html coverage report to `dist/coverage/python`" in result.stderr
html_cov_dir = coverage_path / "htmlcov"
assert html_cov_dir.exists() is True
assert (html_cov_dir / "index.html").exists() is True

assert "Wrote json coverage report to `dist/coverage/python`" in result.stderr_data
json_coverage = coverage_path / "coverage.json"
assert json_coverage.exists() is True
with self.setup_tmpdir(self.sources) as tmpdir:
result = self.run_coverage(tmpdir, "--coverage-py-report=['xml', 'html', 'json']")
coverage_path = Path(get_buildroot(), "dist", "coverage", "python")
assert coverage_path.exists() is True

assert "Wrote xml coverage report to `dist/coverage/python`" in result.stderr
xml_coverage = coverage_path / "coverage.xml"
assert xml_coverage.exists() is True

assert "Wrote html coverage report to `dist/coverage/python`" in result.stderr
html_cov_dir = coverage_path / "htmlcov"
assert html_cov_dir.exists() is True
assert (html_cov_dir / "index.html").exists() is True

assert "Wrote json coverage report to `dist/coverage/python`" in result.stderr_data
json_coverage = coverage_path / "coverage.json"
assert json_coverage.exists() is True
Loading

0 comments on commit f7b260e

Please sign in to comment.