pytest: use subtests for various layout options, add coverage

Makefile

@@ -14,7 +14,7 @@ SEG_MODEL := https://qurator-data.de/eynollah/2022-04-05/models_eynollah.tar.gz
 
 BIN_MODEL := https://github.com/qurator-spk/sbb_binarization/releases/download/v0.0.11/saved_model_2021_03_09.zip
 
-PYTEST_ARGS ?= 
+PYTEST_ARGS ?= -vv
 
 # BEGIN-EVAL makefile-parser --make-help Makefile
 
@@ -90,6 +90,7 @@ smoke-test: tests/resources/kant_aufklaerung_1784_0020.tif
 	@set -x; test "$$(identify -format '%w %h' $<)" = "$$(identify -format '%w %h' $(TMPDIR)/$(<F))"
 	$(RM) -r $(TMPDIR)
 
+ocrd-test: export OCRD_MISSING_OUTPUT := ABORT
 ocrd-test: TMPDIR != mktemp -d
 ocrd-test: tests/resources/kant_aufklaerung_1784_0020.tif
 	cp $< $(TMPDIR)
@@ -107,6 +108,12 @@ ocrd-test: tests/resources/kant_aufklaerung_1784_0020.tif
 test:
 	EYNOLLAH_MODELS=$(CURDIR)/models_eynollah $(PYTHON) -m pytest tests  --durations=0 --continue-on-collection-errors $(PYTEST_ARGS)
 
+coverage:
+	coverage erase
+	$(MAKE) test PYTHON="coverage run"
+	coverage report
+	coverage html
+
 # Build docker image
 docker:
 	docker build \
@@ -115,4 +122,4 @@ docker:
 	--build-arg BUILD_DATE=$$(date -u +"%Y-%m-%dT%H:%M:%SZ") \
 	-t $(DOCKER_TAG) .
 
-.PHONY: models build install install-dev test smoke-test ocrd-test docker help
+.PHONY: models build install install-dev test smoke-test ocrd-test coverage docker help

pyproject.toml

@@ -47,3 +47,9 @@ where = ["src"]
 
 [tool.setuptools.package-data]
 "*" = ["*.json", '*.yml', '*.xml', '*.xsd']
+
+[tool.coverage.run]
+branch = true
+source = [
+    "src/eynollah"
+]

requirements-test.txt

@@ -1,2 +1,4 @@
 pytest
+pytest-subtests
+coverage
 black

tests/base.py

@@ -1,54 +0,0 @@
-# pylint: disable=unused-import
-
-from os.path import dirname, realpath
-from os import chdir
-import sys
-import logging
-import io
-import collections
-from unittest import TestCase as VanillaTestCase, skip, main as unittests_main
-import pytest
-from ocrd_utils import disableLogging, initLogging
-
-def main(fn=None):
-    if fn:
-        sys.exit(pytest.main([fn]))
-    else:
-        unittests_main()
-
-class TestCase(VanillaTestCase):
-
-    @classmethod
-    def setUpClass(cls):
-        chdir(dirname(realpath(__file__)) + '/..')
-
-    def setUp(self):
-        disableLogging()
-        initLogging()
-
-class CapturingTestCase(TestCase):
-    """
-    A TestCase that needs to capture stderr/stdout and invoke click CLI.
-    """
-
-    @pytest.fixture(autouse=True)
-    def _setup_pytest_capfd(self, capfd):
-        self.capfd = capfd
-
-    def invoke_cli(self, cli, args):
-        """
-        Substitution for click.CliRunner.invooke that works together nicely
-        with unittests/pytest capturing stdout/stderr.
-        """
-        self.capture_out_err()  # XXX snapshot just before executing the CLI
-        code = 0
-        sys.argv[1:] = args # XXX necessary because sys.argv reflects pytest args not cli args
-        try:
-            cli.main(args=args)
-        except SystemExit as e:
-            code = e.code
-        out, err = self.capture_out_err()
-        return code, out, err
-
-    def capture_out_err(self):
-        return self.capfd.readouterr()

tests/test_counter.py

@@ -1,4 +1,3 @@
-from tests.base import main
 from eynollah.utils.counter import EynollahIdCounter
 
 def test_counter_string():
@@ -29,5 +28,3 @@ def test_counter_methods():
     c.inc('region', -9)
     assert c.get('region') == 1
 
-if __name__ == '__main__':
-    main(__file__)

tests/test_dpi.py

@@ -1,11 +1,8 @@
 import cv2
 from pathlib import Path
 from eynollah.utils.pil_cv2 import check_dpi
-from tests.base import main
 
 def test_dpi():
     fpath = str(Path(__file__).parent.joinpath('resources', 'kant_aufklaerung_1784_0020.tif'))
     assert 230 == check_dpi(cv2.imread(fpath))
 
-if __name__ == '__main__':
-    main(__file__)

tests/test_run.py

@@ -1,24 +1,39 @@
 from os import environ
 from pathlib import Path
-from ocrd_utils import pushd_popd
-from tests.base import CapturingTestCase as TestCase, main
 from eynollah.cli import layout as eynollah_cli
+from click.testing import CliRunner
 
 testdir = Path(__file__).parent.resolve()
 
 EYNOLLAH_MODELS = environ.get('EYNOLLAH_MODELS', str(testdir.joinpath('..', 'models_eynollah').resolve()))
 
-class TestEynollahRun(TestCase):
-
-    def test_full_run(self):
-        with pushd_popd(tempdir=True) as tempdir:
-            code, out, err = self.invoke_cli(eynollah_cli, [
-                '-m', EYNOLLAH_MODELS,
-                '-i', str(testdir.joinpath('resources/kant_aufklaerung_1784_0020.tif')),
-                '-o', tempdir
-            ])
-            print(code, out, err)
-            assert not code
-
-if __name__ == '__main__':
-    main(__file__)
+def test_full_run(tmpdir, subtests, pytestconfig):
+    args = [
+        '-m', EYNOLLAH_MODELS,
+        '-i', str(testdir.joinpath('resources/kant_aufklaerung_1784_0020.tif')),
+        '-o', tmpdir,
+        # subtests write to same location
+        '--overwrite',
+    ]
+    if pytestconfig.getoption('verbose') > 0:
+        args.extend(['-l', 'DEBUG'])
+    runner = CliRunner()
+    for options in [
+            [], # defaults
+            ["--allow_scaling", "--curved-line"],
+            ["--allow_scaling", "--curved-line", "--full-layout"],
+            ["--allow_scaling", "--curved-line", "--full-layout", "--reading_order_machine_based"],
+            ["--allow_scaling", "--curved-line", "--full-layout", "--reading_order_machine_based",
+             "--textline_light", "--light_version"],
+            # -ep ...
+            # -eoi ...
+            # --do_ocr
+            # --skip_layout_and_reading_order
+    ]:
+        with subtests.test(#msg="test CLI",
+                           options=options):
+            result = runner.invoke(eynollah_cli, args + options)
+            print(result)
+            print(result.output)
+            assert result.exit_code == 0
+            assert 'kant_aufklaerung_1784_0020.tif' in result.output

tests/test_xml.py

@@ -1,4 +1,3 @@
-from pytest import main
 from eynollah.utils.xml import create_page_xml
 from ocrd_models.ocrd_page import to_xml
 
@@ -9,6 +8,3 @@ def test_create_xml():
     xmlstr = to_xml(pcgts)
     assert 'xmlns:pc="%s"' % PAGE_2019 in xmlstr
     assert 'Metadata' in xmlstr
-
-if __name__ == '__main__':
-    main([__file__])