Execution and Coverage Reporting#
See also
This notebook was rendered with myst-nb: tutorial_coverage.ipynb
The core component of the notebook execution API is the CoverageNotebookClient class,
which is a subclass of nbclient.client.NotebookClient,
that can additionally create code coverage analytics.
This class is called by execute_notebook(),
which returns an ExecuteResult object.
from pytest_notebook.execution import execute_notebook
from pytest_notebook.notebook import create_notebook, create_cell, dump_notebook
/home/docs/checkouts/readthedocs.org/user_builds/pytest-notebook/envs/latest/lib/python3.12/site-packages/pytest_notebook/utils.py:48: UserWarning: Field "pytest_notebook.execution.ExecuteResult.exec_error" type could not be converted.
warnings.warn( # noqa: B028
notebook = create_notebook(
metadata={
"kernelspec": {
"name": "python3",
"display_name": "Python 3",
}
},
cells=[
create_cell("""
from pytest_notebook import __version__
from pytest_notebook.notebook import create_notebook
print(__version__)
"""
)]
)
result = execute_notebook(notebook, with_coverage=True)
result.notebook
[IPKernelApp] WARNING | Kernel is running over TCP without encryption. All communication (including code and outputs) is sent in plain text and is susceptible to eavesdropping. Use IPC transport or launch with kernel manager-provisioned CurveZMQ keys to enable transport encryption.
{'nbformat': 4,
'nbformat_minor': 5,
'metadata': {'kernelspec': {'name': 'python3', 'display_name': 'Python 3'},
'language_info': {'name': 'python',
'version': '3.12.13',
'mimetype': 'text/x-python',
'codemirror_mode': {'name': 'ipython', 'version': 3},
'pygments_lexer': 'ipython3',
'nbconvert_exporter': 'python',
'file_extension': '.py'}},
'cells': [{'id': 'b599b090',
'cell_type': 'code',
'metadata': {},
'execution_count': 1,
'source': '\nfrom pytest_notebook import __version__\nfrom pytest_notebook.notebook import create_notebook\nprint(__version__)\n',
'outputs': [{'output_type': 'stream',
'name': 'stdout',
'text': '0.11.0\n'}]}]}
result.coverage_data()
<CoverageData @0x755795e2d3a0 _no_disk=True _basename='/home/docs/checkouts/readthedocs.org/user_builds/pytest-notebook/checkouts/latest/docs/source/user_guide/.coverage' _suffix=None _our_suffix=False _warn=None _filename='file:coverage-3c598f59-f484-4b44-9c66-6dad78220451?mode=memory&cache=shared' _file_map={'/tmp/ipykernel_893/1995098215.py': 1, '/tmp/ipykernel_893/1734066132.py': 2, '/tmp/ipykernel_893/1393556883.py': 3} _dbs={129019378039680: <SqliteDb @0x7557942dfb60 filename='file:coverage-3c598f59-f484-4b44-9c66-6dad78220451?mode=memory&cache=shared' no_disk=True nest=0 con=<sqlite3.Connection object at 0x7557942532e0>>} _pid=871 _lock=<unlocked _thread.RLock object owner=0 count=0 at 0x755794605880> _wrote_hash=False _hasher=<coverage.misc.Hasher object at 0x755794339b20> _have_used=True _has_lines=True _has_arcs=False _current_context=None _current_context_id=1 _query_context_ids=None>
result.coverage_dict
The coverage can be limited to particular files or modules, by setting cov_source.
result = execute_notebook(
notebook, with_coverage=True, cov_source=['pytest_notebook.notebook'])
result.coverage_dict
[IPKernelApp] WARNING | Kernel is running over TCP without encryption. All communication (including code and outputs) is sent in plain text and is susceptible to eavesdropping. Use IPC transport or launch with kernel manager-provisioned CurveZMQ keys to enable transport encryption.
{'/home/docs/checkouts/readthedocs.org/user_builds/pytest-notebook/envs/latest/lib/python3.12/site-packages/pytest_notebook/notebook.py': [128,
1,
3,
4,
5,
6,
7,
8,
9,
260,
11,
12,
13,
14,
15,
265,
17,
18,
19,
267,
21,
274,
23,
281,
26,
27,
28,
158,
159,
289,
167,
304,
305,
50,
51,
52,
306,
307,
309,
266,
189,
190,
191,
192,
194,
195,
196,
197,
322,
200,
201,
207,
208,
211,
212,
218,
91,
92,
93,
219,
220,
221,
223,
224,
225,
226,
230,
124,
125]}
Integration with pytest-cov#
If the pytest-cov plugin is installed,
the NBRegressionFixture will be initialised
with the settings and coverage.Coverage object, that
pytest-cov has created.
If the --nb-coverage flag is set, then nb_regression will run coverage introspection,
and merge the data back into the main Coverage object.
%load_ext pytest_notebook.ipy_magic
%%pytest --disable-warnings --color=yes --cov=pytest_notebook --nb-coverage --log-cli-level=info
import logging
try:
# Python <= 3.8
from importlib_resources import files
except ImportError:
from importlib.resources import files
from pytest_notebook import example_nbs
def test_notebook(nb_regression):
logging.getLogger(__name__).info(nb_regression)
with files(example_nbs).joinpath("coverage.ipynb") as path:
nb_regression.check(str(path))
/home/docs/checkouts/readthedocs.org/user_builds/pytest-notebook/envs/latest/lib/python3.12/site-packages/pytest_notebook/utils.py:48: UserWarning: Field "pytest_notebook.execution.ExecuteResult.exec_error" type could not be converted.
warnings.warn( # noqa: B028
/home/docs/checkouts/readthedocs.org/user_builds/pytest-notebook/envs/latest/lib/python3.12/site-packages/pytest_notebook/utils.py:48: UserWarning: Field "pytest_notebook.nb_regression.NBRegressionFixture.exec_cwd" type could not be converted.
warnings.warn( # noqa: B028
/home/docs/checkouts/readthedocs.org/user_builds/pytest-notebook/envs/latest/lib/python3.12/site-packages/pytest_notebook/utils.py:48: UserWarning: Field "pytest_notebook.nb_regression.NBRegressionFixture.exec_env" type could not be converted.
warnings.warn( # noqa: B028
/home/docs/checkouts/readthedocs.org/user_builds/pytest-notebook/envs/latest/lib/python3.12/site-packages/pytest_notebook/utils.py:48: UserWarning: Field "pytest_notebook.nb_regression.NBRegressionFixture.cov_config" type could not be converted.
warnings.warn( # noqa: B028
/home/docs/checkouts/readthedocs.org/user_builds/pytest-notebook/envs/latest/lib/python3.12/site-packages/pytest_notebook/utils.py:48: UserWarning: Field "pytest_notebook.nb_regression.NBRegressionFixture.cov_source" type could not be converted.
warnings.warn( # noqa: B028
/home/docs/checkouts/readthedocs.org/user_builds/pytest-notebook/envs/latest/lib/python3.12/site-packages/pytest_notebook/utils.py:48: UserWarning: Field "pytest_notebook.nb_regression.NBRegressionFixture.cov_merge" type could not be converted.
warnings.warn( # noqa: B028
============================= test session starts ==============================
platform linux -- Python 3.12.13, pytest-9.1.1, pluggy-1.6.0
rootdir: /tmp/tmppooeuxks
plugins: anyio-4.14.1, pytest_notebook-0.11.0, cov-7.1.0
collected 1 item
test_ipycell.py::test_notebook
-------------------------------- live log call ---------------------------------
INFO test_ipycell:test_ipycell.py:11 NBRegressionFixture(exec_notebook=True, exec_cwd=None, exec_allow_errors=False, exec_timeout=120, exec_env=None, coverage=True, cov_config='.coveragerc', cov_source=('pytest_notebook',), cov_merge=<Coverage @0x766ef090dc70 core=CTracer data_file='/tmp/tmppooeuxks/.coverage.build-33548040-project-492268-pytest-notebook.pid929.Xcb8g30x'>, post_processors=('coalesce_streams',), process_resources={}, diff_replace=(), diff_ignore=('/cells/*/outputs/*/traceback',), diff_use_color=True, diff_color_words=False, force_regen=False)
INFO pytest_notebook.execution:execution.py:102 Executing notebook with kernel:
FAILED [100%]/home/docs/checkouts/readthedocs.org/user_builds/pytest-notebook/envs/latest/lib/python3.12/site-packages/coverage/inorout.py:577: CoverageWarning: Module pytest_notebook was previously imported, but not measured (module-not-measured); see https://coverage.readthedocs.io/en/7.15.0/messages.html#warning-module-not-measured
self.warn(msg, slug="module-not-measured")
=================================== FAILURES ===================================
________________________________ test_notebook _________________________________
nb_regression = NBRegressionFixture(exec_notebook=True, exec_cwd=None, exec_allow_errors=False, exec_timeout=120, exec_env=None, cover...lace=(), diff_ignore=('/cells/*/outputs/*/traceback',), diff_use_color=True, diff_color_words=False, force_regen=False)
def test_notebook(nb_regression):
logging.getLogger(__name__).info(nb_regression)
with files(example_nbs).joinpath("coverage.ipynb") as path:
> nb_regression.check(str(path))
E pytest_notebook.nb_regression.NBRegressionError:
E --- expected
E +++ obtained
E ## modified /cells/0/outputs/0/text:
E @@ -1 +1 @@
E -0.6.0
E +0.11.0
E
E ## modified /metadata/language_info/version:
E - 3.6.7
E + 3.12.13
E
E
test_ipycell.py:13: NBRegressionError
----------------------------- Captured stderr call -----------------------------
[IPKernelApp] WARNING | Kernel is running over TCP without encryption. All communication (including code and outputs) is sent in plain text and is susceptible to eavesdropping. Use IPC transport or launch with kernel manager-provisioned CurveZMQ keys to enable transport encryption.
------------------------------ Captured log call -------------------------------
INFO test_ipycell:test_ipycell.py:11 NBRegressionFixture(exec_notebook=True, exec_cwd=None, exec_allow_errors=False, exec_timeout=120, exec_env=None, coverage=True, cov_config='.coveragerc', cov_source=('pytest_notebook',), cov_merge=<Coverage @0x766ef090dc70 core=CTracer data_file='/tmp/tmppooeuxks/.coverage.build-33548040-project-492268-pytest-notebook.pid929.Xcb8g30x'>, post_processors=('coalesce_streams',), process_resources={}, diff_replace=(), diff_ignore=('/cells/*/outputs/*/traceback',), diff_use_color=True, diff_color_words=False, force_regen=False)
INFO pytest_notebook.execution:execution.py:102 Executing notebook with kernel:
================================ tests coverage ================================
_______________ coverage: platform linux, python 3.12.13-final-0 _______________
Name Stmts Miss Cover
-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
/home/docs/checkouts/readthedocs.org/user_builds/pytest-notebook/envs/latest/lib/python3.12/site-packages/pytest_notebook/__init__.py 1 1 0%
/home/docs/checkouts/readthedocs.org/user_builds/pytest-notebook/envs/latest/lib/python3.12/site-packages/pytest_notebook/diffing.py 120 63 48%
/home/docs/checkouts/readthedocs.org/user_builds/pytest-notebook/envs/latest/lib/python3.12/site-packages/pytest_notebook/example_nbs/__init__.py 0 0 100%
/home/docs/checkouts/readthedocs.org/user_builds/pytest-notebook/envs/latest/lib/python3.12/site-packages/pytest_notebook/execution.py 135 106 21%
/home/docs/checkouts/readthedocs.org/user_builds/pytest-notebook/envs/latest/lib/python3.12/site-packages/pytest_notebook/ipy_magic.py 79 79 0%
/home/docs/checkouts/readthedocs.org/user_builds/pytest-notebook/envs/latest/lib/python3.12/site-packages/pytest_notebook/nb_regression.py 200 138 31%
/home/docs/checkouts/readthedocs.org/user_builds/pytest-notebook/envs/latest/lib/python3.12/site-packages/pytest_notebook/notebook.py 145 121 17%
/home/docs/checkouts/readthedocs.org/user_builds/pytest-notebook/envs/latest/lib/python3.12/site-packages/pytest_notebook/plugin.py 172 129 25%
/home/docs/checkouts/readthedocs.org/user_builds/pytest-notebook/envs/latest/lib/python3.12/site-packages/pytest_notebook/post_processors.py 96 70 27%
/home/docs/checkouts/readthedocs.org/user_builds/pytest-notebook/envs/latest/lib/python3.12/site-packages/pytest_notebook/resources/__init__.py 0 0 100%
/home/docs/checkouts/readthedocs.org/user_builds/pytest-notebook/envs/latest/lib/python3.12/site-packages/pytest_notebook/utils.py 58 58 0%
-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
TOTAL 1006 765 24%
=========================== short test summary info ============================
FAILED test_ipycell.py::test_notebook - pytest_notebook.nb_regression.NBRegressionError:
========================= 1 failed, 1 warning in 3.59s =========================
This is also the case, when using the pytest file collection approach.
%%pytest --disable-warnings --color=yes --cov=pytest_notebook --log-cli-level=info
---
[pytest]
nb_coverage = True
nb_test_files = True
nb_diff_ignore =
/metadata/language_info
---
***
(dump_notebook(notebook), "test_notebook1.ipynb")
***
/home/docs/checkouts/readthedocs.org/user_builds/pytest-notebook/envs/latest/lib/python3.12/site-packages/pytest_notebook/utils.py:48: UserWarning: Field "pytest_notebook.execution.ExecuteResult.exec_error" type could not be converted.
warnings.warn( # noqa: B028
/home/docs/checkouts/readthedocs.org/user_builds/pytest-notebook/envs/latest/lib/python3.12/site-packages/pytest_notebook/utils.py:48: UserWarning: Field "pytest_notebook.nb_regression.NBRegressionFixture.exec_cwd" type could not be converted.
warnings.warn( # noqa: B028
/home/docs/checkouts/readthedocs.org/user_builds/pytest-notebook/envs/latest/lib/python3.12/site-packages/pytest_notebook/utils.py:48: UserWarning: Field "pytest_notebook.nb_regression.NBRegressionFixture.exec_env" type could not be converted.
warnings.warn( # noqa: B028
/home/docs/checkouts/readthedocs.org/user_builds/pytest-notebook/envs/latest/lib/python3.12/site-packages/pytest_notebook/utils.py:48: UserWarning: Field "pytest_notebook.nb_regression.NBRegressionFixture.cov_config" type could not be converted.
warnings.warn( # noqa: B028
/home/docs/checkouts/readthedocs.org/user_builds/pytest-notebook/envs/latest/lib/python3.12/site-packages/pytest_notebook/utils.py:48: UserWarning: Field "pytest_notebook.nb_regression.NBRegressionFixture.cov_source" type could not be converted.
warnings.warn( # noqa: B028
/home/docs/checkouts/readthedocs.org/user_builds/pytest-notebook/envs/latest/lib/python3.12/site-packages/pytest_notebook/utils.py:48: UserWarning: Field "pytest_notebook.nb_regression.NBRegressionFixture.cov_merge" type could not be converted.
warnings.warn( # noqa: B028
============================= test session starts ==============================
platform linux -- Python 3.12.13, pytest-9.1.1, pluggy-1.6.0
rootdir: /tmp/tmpjasol7hn
configfile: pytest.ini
plugins: anyio-4.14.1, pytest_notebook-0.11.0, cov-7.1.0
collected 1 item
test_notebook1.ipynb::nbregression(test_notebook1)
-------------------------------- live log call ---------------------------------
INFO pytest_notebook.execution:execution.py:102 Executing notebook with kernel: python3
INFO pytest_notebook.execution:execution.py:131 Recording coverage for notebook
INFO pytest_notebook.execution:execution.py:149 Recording coverage for notebook
INFO pytest_notebook.nb_regression:nb_regression.py:319 Merging coverage.
FAILED [100%]/home/docs/checkouts/readthedocs.org/user_builds/pytest-notebook/envs/latest/lib/python3.12/site-packages/coverage/inorout.py:577: CoverageWarning: Module pytest_notebook was previously imported, but not measured (module-not-measured); see https://coverage.readthedocs.io/en/7.15.0/messages.html#warning-module-not-measured
self.warn(msg, slug="module-not-measured")
=================================== FAILURES ===================================
____________________ notebook: nbregression(test_notebook1) ____________________
pytest_notebook.nb_regression.NBRegressionError:
--- expected
+++ obtained
## replaced (type changed from NoneType to int) /cells/0/execution_count:
- None
+ 1
## inserted before /cells/0/outputs/0:
+ output:
+ output_type: stream
+ name: stdout
+ text:
+ 0.11.0
----------------------------- Captured stderr call -----------------------------
[IPKernelApp] WARNING | Kernel is running over TCP without encryption. All communication (including code and outputs) is sent in plain text and is susceptible to eavesdropping. Use IPC transport or launch with kernel manager-provisioned CurveZMQ keys to enable transport encryption.
------------------------------ Captured log call -------------------------------
INFO pytest_notebook.execution:execution.py:102 Executing notebook with kernel: python3
INFO pytest_notebook.execution:execution.py:131 Recording coverage for notebook
INFO pytest_notebook.execution:execution.py:149 Recording coverage for notebook
INFO pytest_notebook.nb_regression:nb_regression.py:319 Merging coverage.
================================ tests coverage ================================
_______________ coverage: platform linux, python 3.12.13-final-0 _______________
Name Stmts Miss Cover
-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
/home/docs/checkouts/readthedocs.org/user_builds/pytest-notebook/envs/latest/lib/python3.12/site-packages/pytest_notebook/__init__.py 1 0 100%
/home/docs/checkouts/readthedocs.org/user_builds/pytest-notebook/envs/latest/lib/python3.12/site-packages/pytest_notebook/diffing.py 120 39 68%
/home/docs/checkouts/readthedocs.org/user_builds/pytest-notebook/envs/latest/lib/python3.12/site-packages/pytest_notebook/example_nbs/__init__.py 0 0 100%
/home/docs/checkouts/readthedocs.org/user_builds/pytest-notebook/envs/latest/lib/python3.12/site-packages/pytest_notebook/execution.py 135 67 50%
/home/docs/checkouts/readthedocs.org/user_builds/pytest-notebook/envs/latest/lib/python3.12/site-packages/pytest_notebook/ipy_magic.py 79 79 0%
/home/docs/checkouts/readthedocs.org/user_builds/pytest-notebook/envs/latest/lib/python3.12/site-packages/pytest_notebook/nb_regression.py 200 130 35%
/home/docs/checkouts/readthedocs.org/user_builds/pytest-notebook/envs/latest/lib/python3.12/site-packages/pytest_notebook/notebook.py 145 74 49%
/home/docs/checkouts/readthedocs.org/user_builds/pytest-notebook/envs/latest/lib/python3.12/site-packages/pytest_notebook/plugin.py 172 115 33%
/home/docs/checkouts/readthedocs.org/user_builds/pytest-notebook/envs/latest/lib/python3.12/site-packages/pytest_notebook/post_processors.py 96 70 27%
/home/docs/checkouts/readthedocs.org/user_builds/pytest-notebook/envs/latest/lib/python3.12/site-packages/pytest_notebook/resources/__init__.py 0 0 100%
/home/docs/checkouts/readthedocs.org/user_builds/pytest-notebook/envs/latest/lib/python3.12/site-packages/pytest_notebook/utils.py 58 23 60%
-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
TOTAL 1006 597 41%
=========================== short test summary info ============================
FAILED test_notebook1.ipynb::nbregression(test_notebook1) - pytest_notebook.nb_regression.NBRegressionError:
========================= 1 failed, 1 warning in 5.21s =========================