- changed to use --cleanslate when measuring individual test coverage,

  so as to detect issues before accepting a test that may later fail when
  measuring whole suite coverage;

src/coverup/coverup.py

@@ -514,6 +514,7 @@ async def improve_coverage(chatter: llm.Chatter, prompter: prompt.Prompter, seg:
             pytest_args = (f"--count={args.repeat_tests} " if args.repeat_tests else "") + args.pytest_args
             coverage = await measure_test_coverage(test=last_test, tests_dir=args.tests_dir,
                                                  pytest_args=pytest_args,
+                                                 isolate_tests=args.isolate_tests,
                                                  branch_coverage=args.branch_coverage,
                                                  log_write=lambda msg: log_write(seg, msg))
 

src/coverup/testrunner.py

@@ -9,7 +9,8 @@
 from .utils import subprocess_run
 
 
-async def measure_test_coverage(*, test: str, tests_dir: Path, pytest_args='', log_write=None, branch_coverage=True):
+async def measure_test_coverage(*, test: str, tests_dir: Path, pytest_args='',
+                                log_write=None, isolate_tests=False, branch_coverage=True):
     """Runs a given test and returns the coverage obtained."""
     with tempfile.NamedTemporaryFile(prefix="tmp_test_", suffix='.py', dir=str(tests_dir), mode="w") as t:
         t.write(test)
@@ -20,7 +21,9 @@ async def measure_test_coverage(*, test: str, tests_dir: Path, pytest_args='', l
                 # -qq to cut down on tokens
                 p = await subprocess_run([sys.executable, '-m', 'slipcover',  *(('--branch',) if branch_coverage else ()),
                                           '--json', '--out', j.name,
-                                          '-m', 'pytest', *pytest_args.split(), '-qq', '-x', '--disable-warnings', t.name],
+                                          '-m', 'pytest', *pytest_args.split(),
+                                          *(('--cleanslate',) if isolate_tests else ()),
+                                          '-qq', '-x', '--disable-warnings', t.name],
                                          check=True, timeout=60)
                 if log_write:
                     log_write(str(p.stdout, 'UTF-8', errors='ignore'))
@@ -37,7 +40,8 @@ async def measure_test_coverage(*, test: str, tests_dir: Path, pytest_args='', l
     return cov
 
 
-def measure_suite_coverage(*, tests_dir: Path, source_dir: Path, pytest_args='', trace=None, isolate_tests=False, branch_coverage=True):
+def measure_suite_coverage(*, tests_dir: Path, source_dir: Path, pytest_args='',
+                           trace=None, isolate_tests=False, branch_coverage=True):
     """Runs an entire test suite and returns the coverage obtained."""
 
     with tempfile.NamedTemporaryFile(suffix='.json', delete=False) as j:

tests/test_testrunner.py

@@ -3,6 +3,7 @@
 import coverup.testrunner as tr
 from pathlib import Path
 import tempfile
+import textwrap
 
 
 @pytest.mark.asyncio
@@ -11,6 +12,36 @@ async def test_measure_test_coverage_exit_1(tmpdir):
         await tr.measure_test_coverage(test="import os;\ndef test_foo(): os.exit(1)\n", tests_dir=Path(tmpdir))
 
 
+@pytest.mark.asyncio
+async def test_measure_test_coverage_uses_cleanslate(tmp_path, monkeypatch):
+    monkeypatch.chdir(tmp_path)
+
+    (tmp_path / "foo.py").write_text(textwrap.dedent("""\
+        answer = 42
+        """
+    ))
+    test = textwrap.dedent("""\
+        import os
+
+        def test_one():
+            from foo import answer
+            assert answer == 42
+
+        def test_two(monkeypatch):
+            def mock_stat(path):
+                if path == '/':
+                    return os.stat_result((0, 0, 0, 0, 0, 0, 0, 0, 0, 0))
+                raise Exception('mocked stat failure')
+
+            monkeypatch.setattr(os, 'stat', mock_stat)
+            from foo import answer      # should fail because of 'stat' if run in isolation
+            assert answer == 42
+        """)
+
+    with pytest.raises(subprocess.CalledProcessError) as einfo:
+        await tr.measure_test_coverage(test=test, tests_dir=tmp_path, isolate_tests=True)
+
+
 @pytest.mark.parametrize("absolute", [True, False])
 def test_measure_suite_coverage_empty_dir(absolute):
     with tempfile.TemporaryDirectory(dir=Path('.')) as tests_dir: