Reproduction of a teardown problem in pytest

pytest_teardown_reproduce.py

"""
Minimal reproduction of pytest plugin ordering bug where ResourceWarnings
from session-end gc.collect() are silently lost.

The unraisableexception plugin's cleanup (which forces GC) runs AFTER the
warnings plugin's cleanup (which tears down filterwarnings), so any
ResourceWarning emitted during session-end GC is never promoted to an error.

Usage:
    python reproduce.py

Runs two scenarios:
  1. Without fix — demonstrates the bug (test PASSES when it should FAIL)
  2. With fix   — a pytest_unconfigure hook forces GC while filters are
                   still active (test correctly FAILS)
"""

import subprocess
import sys
import textwrap
import tempfile
from pathlib import Path


TEST_FILE = textwrap.dedent("""\
    def test_leaks_resource():
        f = open("/dev/null")
        # Create a reference cycle so refcounting won't free the file
        # when the function returns — only the cyclic GC can collect it.
        cycle = [f]
        cycle.append(cycle)
""")

PYTEST_INI = textwrap.dedent("""\
    [pytest]
    filterwarnings =
        error::ResourceWarning
""")

# The fix: force GC during pytest_unconfigure (which runs BEFORE the
# cleanup stack tears down warning filters), and explicitly set an error
# filter for PytestUnraisableExceptionWarning so collect_unraisable
# actually raises instead of just printing to stderr.
CONFTEST_FIX = textwrap.dedent("""\
    import gc
    import warnings
    import pytest
    from _pytest.config import Config
    from _pytest.unraisableexception import collect_unraisable

    def pytest_unconfigure(config: Config) -> None:
        warnings.filterwarnings("error", category=pytest.PytestUnraisableExceptionWarning)
        for _ in range(5):
            gc.collect()
        collect_unraisable(config)
""")


def run_scenario(name, tmpdir, extra_files=None):
    tmpdir = Path(tmpdir)

    (tmpdir / "pytest.ini").write_text(PYTEST_INI)
    (tmpdir / "test_leak.py").write_text(TEST_FILE)

    if extra_files:
        for filename, content in extra_files.items():
            (tmpdir / filename).write_text(content)

    result = subprocess.run(
        [sys.executable, "-m", "pytest", "-v", "--tb=short", "--override-ini=addopts="],
        cwd=tmpdir,
        capture_output=True,
        text=True,
    )

    print(result.stdout)
    if result.stderr:
        if stderr_lines:
            print("STDERR:", "\n".join(stderr_lines))
            print()

    return result.returncode


def main():
    print("=" * 70)
    print("SCENARIO 1: Without fix (demonstrating the bug)")
    print("=" * 70)
    print()

    with tempfile.TemporaryDirectory() as tmpdir:
        rc1 = run_scenario("without fix", tmpdir)

    if rc1 == 0:
        print("=> pytest PASSED — bug confirmed, ResourceWarning was silently lost")
    else:
        print("=> pytest FAILED — bug not reproduced on this version")
    print()

    print("=" * 70)
    print("SCENARIO 2: With fix (pytest_unconfigure forces GC before teardown)")
    print("=" * 70)
    print()

    with tempfile.TemporaryDirectory() as tmpdir:
        rc2 = run_scenario("with fix", tmpdir, extra_files={
            "conftest.py": CONFTEST_FIX,
        })

    if rc2 != 0:
        print("=> pytest FAILED — fix works, ResourceWarning correctly caught")
    else:
        print("=> pytest PASSED — fix did not work")
    print()

    print("=" * 70)
    print("SUMMARY")
    print("=" * 70)
    print(f"  Without fix: exit code {rc1} ({'PASS (bug)' if rc1 == 0 else 'FAIL'})")
    print(f"  With fix:    exit code {rc2} ({'PASS' if rc2 == 0 else 'FAIL (correct)'})")
    print()
    if rc1 == 0 and rc2 != 0:
        print("  Bug reproduced and fix confirmed.")
    elif rc1 != 0:
        print("  Bug not reproduced — pytest may have fixed this upstream.")
    else:
        print("  Fix did not resolve the issue.")


if __name__ == "__main__":
    main()