Cleaner pytest tracebacks with collector plugin

[hauntsaninja]

Hello all!

I'm writing my first pytest plugin. It collects tests from some data files, not too unlike the YAML example in the documentation, except that the data files can be dynamically generated on demand.

I have something like the following and it mostly works great:

class Snapshot:
    def __init__(self, base_location, function):
        self.base_location = base_location
        self.function = function

    def data_paths(self):
        # some generation of files might happen around here
        for data_path in self.base_location.iterdir():
            yield data_path


class SnapshotItem(pytest.Item):
    def __init__(self, data_path, function, **kwargs) -> None:
        super().__init__(**kwargs)
        self.data_path = data_path
        self.function = function

    def runtest(self):
        with open(self.data_path, "rb") as f:
            data = json.load(f)
        value = data["value"]
        expected = data["expected"]
        actual = self.function(value)
        if actual != expected:
            expected_details, actual_details = get_details(expected, actual)
            self.add_report_section("call", "expected", expected_details)
            self.add_report_section("call", "actual", actual_details)
        assert actual == expected


class SnapshotCollector(pytest.Class):
    def collect(self):
        snapshot = self.obj
        for data_path in snapshot.data_paths():
            yield SnapshotItem.from_parent(
                parent=self, name=data_path.name, data_path=data_path, function=snapshot.function
            )


def pytest_pycollect_makeitem(collector, name: str, obj: object):
    if isinstance(obj, Snapshot):
        return SnapshotCollector.from_parent(parent=collector, name=name)
    return None

However, tracebacks now have a lot of pytest gunk in them, including from frames that have __tracebackhide__ = True .

Here's an example of the messy tracebacks:

=============================================================================== FAILURES ================================================================================
_____________________________________________________________________________ test session ______________________________________________________________________________

cls = <class '_pytest.runner.CallInfo'>, func = <function call_runtest_hook.. at 0x107071ee0>, when = 'call'
reraise = (<class '_pytest.outcomes.Exit'>, <class 'KeyboardInterrupt'>)

@classmethod
def from_call(
    cls,
    func: "Callable[[], TResult]",
    when: "Literal['collect', 'setup', 'call', 'teardown']",
    reraise: Optional[
        Union[Type[BaseException], Tuple[Type[BaseException], ...]]
    ] = None,
) -> "CallInfo[TResult]":
    excinfo = None
    start = timing.time()
    precise_start = timing.perf_counter()
    try:

      result: Optional[TResult] = func()

../../../.virtualenvs/cbqc/lib/python3.9/site-packages/_pytest/runner.py:311:

---

  lambda: ihook(item=item, **kwds), when=when, reraise=reraise

)

../../../.virtualenvs/cbqc/lib/python3.9/site-packages/_pytest/runner.py:255:

---

self = <_HookCaller 'pytest_runtest_call'>, args = (), kwargs = {'item': <SnapshotItem (redacted data file name)>}, argname = 'item', firstresult = False

def __call__(self, *args, **kwargs):
    if args:
        raise TypeError("hook calling supports only keyword arguments")
    assert not self.is_historic()

    # This is written to avoid expensive operations when not needed.
    if self.spec:
        for argname in self.spec.argnames:
            if argname not in kwargs:
                notincall = tuple(set(self.spec.argnames) - kwargs.keys())
                warnings.warn(
                    "Argument(s) {} which are declared in the hookspec "
                    "can not be found in this hook call".format(notincall),
                    stacklevel=2,
                )
                break

        firstresult = self.spec.opts.get("firstresult")
    else:
        firstresult = False

  return self._hookexec(self.name, self.get_hookimpls(), kwargs, firstresult)

../../../.virtualenvs/cbqc/lib/python3.9/site-packages/pluggy/_hooks.py:265:

---

self = <_pytest.config.PytestPluginManager object at 0x10474c6a0>, hook_name = 'pytest_runtest_call'
methods = [<HookImpl plugin_name='runner', plugin=<module '_pytest.runner' from '/Users/shantanu/.virtualenvs/cbqc/lib/py...ion' from '/Users/shantanu/.virtualenvs/cbqc/lib/python3.9/site-packages/_pytest/unraisableexception.py'>>, ...]
kwargs = {'item': <SnapshotItem (redacted data file name)>}, firstresult = False

def _hookexec(self, hook_name, methods, kwargs, firstresult):
    # called from all hookcaller instances.
    # enable_tracing will set its own wrapping function at self._inner_hookexec

  return self._inner_hookexec(hook_name, methods, kwargs, firstresult)

../../../.virtualenvs/cbqc/lib/python3.9/site-packages/pluggy/_manager.py:80:

---

hook_name = 'pytest_runtest_call'
hook_impls = [<HookImpl plugin_name='runner', plugin=<module '_pytest.runner' from '/Users/shantanu/.virtualenvs/cbqc/lib/py...ion' from '/Users/shantanu/.virtualenvs/cbqc/lib/python3.9/site-packages/_pytest/unraisableexception.py'>>, ...]
caller_kwargs = {'item': <SnapshotItem (redacted data file name)>}, firstresult = False

def _multicall(hook_name, hook_impls, caller_kwargs, firstresult):
    """Execute a call into multiple python functions/methods and return the
    result(s).

    ``caller_kwargs`` comes from _HookCaller.__call__().
    """
    __tracebackhide__ = True
    results = []
    excinfo = None
    try:  # run impl and wrapper setup functions in a loop
        teardowns = []
        try:
            for hook_impl in reversed(hook_impls):
                try:
                    args = [caller_kwargs[argname] for argname in hook_impl.argnames]
                except KeyError:
                    for argname in hook_impl.argnames:
                        if argname not in caller_kwargs:
                            raise HookCallError(
                                f"hook call must provide argument {argname!r}"
                            )

                if hook_impl.hookwrapper:
                    try:
                        gen = hook_impl.function(*args)
                        next(gen)  # first yield
                        teardowns.append(gen)
                    except StopIteration:
                        _raise_wrapfail(gen, "did not yield")
                else:
                    res = hook_impl.function(*args)
                    if res is not None:
                        results.append(res)
                        if firstresult:  # halt further impl calls
                            break
        except BaseException:
            excinfo = sys.exc_info()
    finally:
        if firstresult:  # first result hooks return a single value
            outcome = _Result(results[0] if results else None, excinfo)
        else:
            outcome = _Result(results, excinfo)

        # run all wrapper post-yield blocks
        for gen in reversed(teardowns):
            try:
                gen.send(outcome)
                _raise_wrapfail(gen, "has second yield")
            except StopIteration:
                pass

      return outcome.get_result()

../../../.virtualenvs/cbqc/lib/python3.9/site-packages/pluggy/_callers.py:60:

---

self = <pluggy._result._Result object at 0x107046af0>

def get_result(self):
    """Get the result(s) for this hook call.

    If the hook was marked as a ``firstresult`` only a single value
    will be returned otherwise a list of results.
    """
    __tracebackhide__ = True
    if self._excinfo is None:
        return self._result
    else:
        ex = self._excinfo

      raise ex[1].with_traceback(ex[2])

../../../.virtualenvs/cbqc/lib/python3.9/site-packages/pluggy/_result.py:60:

---

hook_name = 'pytest_runtest_call'
hook_impls = [<HookImpl plugin_name='runner', plugin=<module '_pytest.runner' from '/Users/shantanu/.virtualenvs/cbqc/lib/py...ion' from '/Users/shantanu/.virtualenvs/cbqc/lib/python3.9/site-packages/_pytest/unraisableexception.py'>>, ...]
caller_kwargs = {'item': <SnapshotItem (redacted data file name)>}, firstresult = False

def _multicall(hook_name, hook_impls, caller_kwargs, firstresult):
    """Execute a call into multiple python functions/methods and return the
    result(s).

    ``caller_kwargs`` comes from _HookCaller.__call__().
    """
    __tracebackhide__ = True
    results = []
    excinfo = None
    try:  # run impl and wrapper setup functions in a loop
        teardowns = []
        try:
            for hook_impl in reversed(hook_impls):
                try:
                    args = [caller_kwargs[argname] for argname in hook_impl.argnames]
                except KeyError:
                    for argname in hook_impl.argnames:
                        if argname not in caller_kwargs:
                            raise HookCallError(
                                f"hook call must provide argument {argname!r}"
                            )

                if hook_impl.hookwrapper:
                    try:
                        gen = hook_impl.function(*args)
                        next(gen)  # first yield
                        teardowns.append(gen)
                    except StopIteration:
                        _raise_wrapfail(gen, "did not yield")
                else:

                  res = hook_impl.function(*args)

../../../.virtualenvs/cbqc/lib/python3.9/site-packages/pluggy/_callers.py:39:

---

item = <SnapshotItem (redacted data file name)>

def pytest_runtest_call(item: Item) -> None:
    _update_current_test_var(item, "call")
    try:
        del sys.last_type
        del sys.last_value
        del sys.last_traceback
    except AttributeError:
        pass
    try:
        item.runtest()
    except Exception as e:
        # Store trace info to allow postmortem debugging
        sys.last_type = type(e)
        sys.last_value = e
        assert e.__traceback__ is not None
        # Skip *this* frame
        sys.last_traceback = e.__traceback__.tb_next

      raise e

../../../.virtualenvs/cbqc/lib/python3.9/site-packages/_pytest/runner.py:170:

---

item = <SnapshotItem (redacted data file name)>

def pytest_runtest_call(item: Item) -> None:
    _update_current_test_var(item, "call")
    try:
        del sys.last_type
        del sys.last_value
        del sys.last_traceback
    except AttributeError:
        pass
    try:

      item.runtest()

../../../.virtualenvs/cbqc/lib/python3.9/site-packages/_pytest/runner.py:162:

---

self = <SnapshotItem (redacted data file name)>

def runtest(self):
    with open(self.data_path, "rb") as f:
        data = json.load(f)
    # ...
    # basically the same as in the pastebin above
    # ...

  assert actual == expected

E AssertionError

[hauntsaninja]

Okay, I think I want something like:

    def repr_failure(self, excinfo):
        return self._repr_failure_py(excinfo, style="short")

and to make sure assert actual == expected gets rewritten.