Merge branch 'cli-logging' into model-zoo

src/eynollah/cli.py

@@ -1,17 +1,22 @@
 from dataclasses import dataclass
-import os
-import sys
-import click
 import logging
-from ocrd_utils import initLogging, getLevelName, getLogger
+import sys
+import os
+from typing import Union
+
+import click
 
 from .model_zoo import EynollahModelZoo
-
 from .cli_models import models_cli
 
 @dataclass()
 class EynollahCliCtx:
+    """
+    Holds options relevant for all eynollah subcommands
+    """
     model_zoo: EynollahModelZoo
+    log_level : Union[str, None] = 'INFO'
+
 
 @click.group()
 @click.option(
@@ -28,10 +33,31 @@ class EynollahCliCtx:
     type=(str, str, str),
     multiple=True,
 )
+@click.option(
+    "--log_level",
+    "-l",
+    type=click.Choice(['OFF', 'DEBUG', 'INFO', 'WARN', 'ERROR']),
+    help="Override log level globally to this",
+)
 @click.pass_context
-def main(ctx, model_basedir, model_overrides):
+def main(ctx, model_basedir, model_overrides, log_level):
+    """
+    eynollah - Document Layout Analysis, Image Enhancement, OCR
+    """
+    # Initialize logging
+    console_handler = logging.StreamHandler(sys.stdout)
+    console_handler.setLevel(logging.NOTSET)
+    formatter = logging.Formatter('%(asctime)s.%(msecs)03d %(levelname)s %(name)s - %(message)s', datefmt='%H:%M:%S')
+    console_handler.setFormatter(formatter)
+    logging.getLogger('eynollah').addHandler(console_handler)
+    logging.getLogger('eynollah').setLevel(log_level or logging.INFO)
     # Initialize model zoo
-    ctx.obj = EynollahCliCtx(model_zoo=EynollahModelZoo(basedir=model_basedir, model_overrides=model_overrides))
+    model_zoo = EynollahModelZoo(basedir=model_basedir, model_overrides=model_overrides)
+    # Initialize CLI context
+    ctx.obj = EynollahCliCtx(
+        model_zoo=model_zoo,
+        log_level=log_level,
+    )
 
 main.add_command(models_cli, 'models')
 
@@ -55,20 +81,14 @@ main.add_command(models_cli, 'models')
     type=click.Path(exists=True, file_okay=False),
     required=True,
 )
-@click.option(
-    "--log_level",
-    "-l",
-    type=click.Choice(['OFF', 'DEBUG', 'INFO', 'WARN', 'ERROR']),
-    help="Override log level globally to this",
-)
 @click.pass_context
-def machine_based_reading_order(ctx, input, dir_in, out, log_level):
+def machine_based_reading_order(ctx, input, dir_in, out):
+    """
+    Generate ReadingOrder with a ML model
+    """
     from eynollah.mb_ro_on_layout import machine_based_reading_order_on_layout
     assert bool(input) != bool(dir_in), "Either -i (single input) or -di (directory) must be provided, but not both."
     orderer = machine_based_reading_order_on_layout(model_zoo=ctx.obj.model_zoo)
-    if log_level:
-        orderer.logger.setLevel(getLevelName(log_level))
-
     orderer.run(xml_filename=input,
                 dir_in=dir_in,
                 dir_out=out,
@@ -103,12 +123,6 @@ def machine_based_reading_order(ctx, input, dir_in, out, log_level):
     default='single',
     help="Whether to use the (newer and faster) single-model binarization or the (slightly better) multi-model binarization"
 )
-@click.option(
-    "--log_level",
-    "-l",
-    type=click.Choice(['OFF', 'DEBUG', 'INFO', 'WARN', 'ERROR']),
-    help="Override log level globally to this",
-)
 @click.pass_context
 def binarization(
     ctx,
@@ -117,13 +131,13 @@ def binarization(
     mode,
     dir_in,
     output,
-    log_level,
 ):
+    """
+    Binarize images with a ML model
+    """
     from eynollah.sbb_binarize import SbbBinarizer
     assert bool(input_image) != bool(dir_in), "Either -i (single input) or -di (directory) must be provided, but not both."
     binarizer = SbbBinarizer(model_zoo=ctx.obj.model_zoo, mode=mode)
-    if log_level:
-        binarizer.log.setLevel(getLevelName(log_level))
     binarizer.run(
         image_path=input_image,
         use_patches=patches,
@@ -175,25 +189,19 @@ def binarization(
     is_flag=True,
     help="if this parameter set to true, this tool will save the enhanced image in org scale.",
 )
-@click.option(
-    "--log_level",
-    "-l",
-    type=click.Choice(['OFF', 'DEBUG', 'INFO', 'WARN', 'ERROR']),
-    help="Override log level globally to this",
-)
 @click.pass_context
-def enhancement(ctx, image, out, overwrite, dir_in, num_col_upper, num_col_lower, save_org_scale,  log_level):
+def enhancement(ctx, image, out, overwrite, dir_in, num_col_upper, num_col_lower, save_org_scale):
-    from eynollah.image_enhancer import Enhancer
+    """
+    Enhance image
+    """
     assert bool(image) != bool(dir_in), "Either -i (single input) or -di (directory) must be provided, but not both."
-    initLogging()
+    from .image_enhancer import Enhancer
     enhancer = Enhancer(
         model_zoo=ctx.obj.model_zoo,
         num_col_upper=num_col_upper,
         num_col_lower=num_col_lower,
         save_org_scale=save_org_scale,
     )
-    if log_level:
-        enhancer.logger.setLevel(getLevelName(log_level))
     enhancer.run(overwrite=overwrite,
                  dir_in=dir_in,
                  image_filename=image,
@@ -384,19 +392,6 @@ def enhancement(ctx, image, out, overwrite, dir_in, num_col_upper, num_col_lower
     is_flag=True,
     help="if this parameter set to true, this tool will ignore layout detection and reading order. It means that textline detection will be done within printspace and contours of textline will be written in xml output file.",
 )
-# TODO move to top-level CLI context
-@click.option(
-    "--log_level",
-    "-l",
-    type=click.Choice(['OFF', 'DEBUG', 'INFO', 'WARN', 'ERROR']),
-    help="Override 'eynollah' log level globally to this",
-)
-# 
-@click.option(
-    "--setup-logging",
-    is_flag=True,
-    help="Setup a basic console logger",
-)
 @click.pass_context
 def layout(
     ctx,
@@ -434,16 +429,10 @@ def layout(
     log_level,
     setup_logging,
 ):
+    """
+    Detect Layout (with optional image enhancement and reading order detection)
+    """
     from eynollah.eynollah import Eynollah
-    if setup_logging:
-        console_handler = logging.StreamHandler(sys.stdout)
-        console_handler.setLevel(logging.INFO)
-        formatter = logging.Formatter('%(message)s')
-        console_handler.setFormatter(formatter)
-        getLogger('eynollah').addHandler(console_handler)
-        getLogger('eynollah').setLevel(logging.INFO)
-    else:
-        initLogging()
     assert enable_plotting or not save_layout, "Plotting with -sl also requires -ep"
     assert enable_plotting or not save_deskewed, "Plotting with -sd also requires -ep"
     assert enable_plotting or not save_all, "Plotting with -sa also requires -ep"
@@ -463,6 +452,7 @@ def layout(
     assert not extract_only_images or not right2left, "Image extraction -eoi can not be set alongside right2left -r2l"
     assert not extract_only_images or not headers_off, "Image extraction -eoi can not be set alongside headers_off -ho"
     assert bool(image) != bool(dir_in), "Either -i (single input) or -di (directory) must be provided, but not both."
+    from .eynollah import Eynollah
     eynollah = Eynollah(
         model_zoo=ctx.obj.model_zoo,
         extract_only_images=extract_only_images,
@@ -488,8 +478,6 @@ def layout(
         threshold_art_class_textline=threshold_art_class_textline,
         threshold_art_class_layout=threshold_art_class_layout,
     )
-    if log_level:
-        eynollah.logger.setLevel(getLevelName(log_level))
     eynollah.run(overwrite=overwrite,
                  image_filename=image,
                  dir_in=dir_in,
@@ -579,12 +567,6 @@ def layout(
     "-min_conf",
     help="minimum OCR confidence value. Text lines with a confidence value lower than this threshold will not be included in the output XML file.",
 )
-@click.option(
-    "--log_level",
-    "-l",
-    type=click.Choice(['OFF', 'DEBUG', 'INFO', 'WARN', 'ERROR']),
-    help="Override log level globally to this",
-)
 @click.pass_context
 def ocr(
     ctx,
@@ -601,11 +583,10 @@ def ocr(
     batch_size,
     dataset_abbrevation,
     min_conf_value_of_textline_text,
-    log_level,
 ):
-    from eynollah.eynollah_ocr import Eynollah_ocr
+    """
-    initLogging()
+    Recognize text with a CNN/RNN or transformer ML model.
-
+    """
     assert not export_textline_images_and_text or not tr_ocr, "Exporting textline and text  -etit can not be set alongside transformer ocr -tr_ocr"
     # FIXME: refactor: move export_textline_images_and_text out of eynollah.py
     # assert not export_textline_images_and_text or not model, "Exporting textline and text  -etit can not be set alongside model -m"
@@ -613,6 +594,7 @@ def ocr(
     assert not export_textline_images_and_text or not dir_in_bin, "Exporting textline and text  -etit can not be set alongside directory of bin images -dib"
     assert not export_textline_images_and_text or not dir_out_image_text, "Exporting textline and text  -etit can not be set alongside directory of images with predicted text -doit"
     assert bool(image) != bool(dir_in), "Either -i (single image) or -di (directory) must be provided, but not both."
+    from .eynollah_ocr import Eynollah_ocr
     eynollah_ocr = Eynollah_ocr(
         model_zoo=ctx.obj.model_zoo,
         tr_ocr=tr_ocr,
@@ -620,10 +602,7 @@ def ocr(
         do_not_mask_with_textline_contour=do_not_mask_with_textline_contour,
         batch_size=batch_size,
         pref_of_dataset=dataset_abbrevation,
-        min_conf_value_of_textline_text=min_conf_value_of_textline_text,
+        min_conf_value_of_textline_text=min_conf_value_of_textline_text)
-    )
-    if log_level:
-        eynollah_ocr.logger.setLevel(getLevelName(log_level))
     eynollah_ocr.run(overwrite=overwrite,
                      dir_in=dir_in,
                      dir_in_bin=dir_in_bin,

src/eynollah/eynollah.py

@@ -8,6 +8,15 @@
 document layout analysis (segmentation) with output in PAGE-XML
 """
 
+import logging
+import sys
+
+# cannot use importlib.resources until we move to 3.9+ forimportlib.resources.files
+if sys.version_info < (3, 10):
+    import importlib_resources
+else:
+    import importlib.resources as importlib_resources
+
 from difflib import SequenceMatcher as sq
 import math
 import os
@@ -27,7 +36,7 @@ import shapely.affinity
 from scipy.signal import find_peaks
 from scipy.ndimage import gaussian_filter1d
 from skimage.morphology import skeletonize
-from ocrd_utils import getLogger, tf_disable_interactive_logs
+from ocrd_utils import tf_disable_interactive_logs
 import statistics
 
 try:
@@ -42,10 +51,15 @@ except ImportError:
 #os.environ['CUDA_VISIBLE_DEVICES'] = '-1'
 tf_disable_interactive_logs()
 import tensorflow as tf
-tf.get_logger().setLevel("ERROR")
+# warnings.filterwarnings("ignore")
-warnings.filterwarnings("ignore")
+from tensorflow.python.keras import backend as K
+from tensorflow.keras.models import load_model
+# use tf1 compatibility for keras backend
+from tensorflow.compat.v1.keras.backend import set_session
+from tensorflow.keras import layers
+from tensorflow.keras.layers import StringLookup
 
-from .model_zoo import (EynollahModelZoo, KerasModel, TrOCRProcessor)
+from .model_zoo import EynollahModelZoo
 from .utils.contour import (
     filter_contours_area_of_image,
     filter_contours_area_of_image_tables,
@@ -162,8 +176,9 @@ class Eynollah:
         threshold_art_class_layout: Optional[float] = None,
         threshold_art_class_textline: Optional[float] = None,
         skip_layout_and_reading_order : bool = False,
+        logger : Optional[logging.Logger] = None,
     ):
-        self.logger = getLogger('eynollah')
+        self.logger = logger or logging.getLogger('eynollah')
         self.model_zoo = model_zoo
         self.plotter = None
 
@@ -4724,5 +4739,3 @@ class Eynollah:
                 conf_contours_textregions=conf_contours_textregions)
             
         return pcgts
-
-

src/eynollah/image_enhancer.py

@@ -2,7 +2,7 @@
 Image enhancer. The output can be written as same scale of input or in new predicted scale.
 """
 
-from logging import Logger
+import logging
 import os
 import time
 from typing import Dict, Optional
@@ -12,7 +12,6 @@ import gc
 import cv2
 from keras.models import Model
 import numpy as np
-from ocrd_utils import getLogger, tf_disable_interactive_logs
 import tensorflow as tf
 from skimage.morphology import skeletonize
 
@@ -37,7 +36,6 @@ class Enhancer:
         num_col_upper : Optional[int] = None,
         num_col_lower : Optional[int] = None,
         save_org_scale : bool = False,
-        logger : Optional[Logger] = None,
     ):
         self.input_binary = False
         self.light_version = False
@@ -51,7 +49,7 @@ class Enhancer:
         else:
             self.num_col_lower = num_col_lower
             
-        self.logger = logger if logger else getLogger('eynollah.enhance')
+        self.logger = logging.getLogger('eynollah.enhance')
         self.model_zoo = model_zoo
         for v in ['binarization', 'enhancement', 'col_classifier', 'page']:
             self.model_zoo.load_model(v)

src/eynollah/mb_ro_on_layout.py

@@ -1,8 +1,8 @@
 """
-Image enhancer. The output can be written as same scale of input or in new predicted scale.
+Machine learning based reading order detection
 """
 
-from logging import Logger
+import logging 
 import os
 import time
 from typing import Optional
@@ -12,7 +12,6 @@ import xml.etree.ElementTree as ET
 import cv2
 from keras.models import Model
 import numpy as np
-from ocrd_utils import getLogger
 import statistics
 import tensorflow as tf
 
@@ -34,9 +33,9 @@ class machine_based_reading_order_on_layout:
         self,
         *,
         model_zoo: EynollahModelZoo,
-        logger : Optional[Logger] = None,
+        logger : Optional[logging.Logger] = None,
     ):
-        self.logger = logger if logger else getLogger('mbreorder')
+        self.logger = logger or logging.getLogger('eynollah.mbreorder')
         self.model_zoo = model_zoo
         
         try:

src/eynollah/ocrd_cli_binarization.py

@@ -34,6 +34,7 @@ class SbbBinarizeProcessor(Processor):
         Set up the model prior to processing.
         """
         # resolve relative path via OCR-D ResourceManager
+        assert isinstance(self.parameter, dict)
         model_path = self.resolve_resource(self.parameter['model'])
         self.binarizer = SbbBinarizer(model_dir=model_path, logger=self.logger)
 

src/eynollah/processor.py

@@ -32,8 +32,8 @@ class EynollahProcessor(Processor):
             allow_scaling=self.parameter['allow_scaling'],
             headers_off=self.parameter['headers_off'],
             tables=self.parameter['tables'],
+            logger=self.logger
         )
-        self.eynollah.logger = self.logger
         self.eynollah.plotter = None
 
     def shutdown(self):

src/eynollah/sbb_binarize.py

@@ -5,14 +5,14 @@ Tool to load model and binarize a given image.
 import os
 import logging
 from pathlib import Path
-from typing import Dict, List
+from typing import Dict, List, Optional
 
-from keras.models import Model
 import numpy as np
 import cv2
 from ocrd_utils import tf_disable_interactive_logs
 
 from eynollah.model_zoo import EynollahModelZoo
+from eynollah.model_zoo.types import AnyModel
 tf_disable_interactive_logs()
 import tensorflow as tf
 from tensorflow.python.keras import backend as tensorflow_backend
@@ -24,10 +24,14 @@ def resize_image(img_in, input_height, input_width):
 
 class SbbBinarizer:
 
-    def __init__(self, *, model_zoo: EynollahModelZoo, mode: str, logger=None):
+    def __init__(
-        if mode not in ('single', 'multi'):
+        self,
-            raise ValueError(f"'mode' must be either 'multi' or 'single', not {mode}")
+        *,
-        self.log = logger if logger else logging.getLogger('eynollah.binarization')
+        model_zoo: EynollahModelZoo,
+        mode: str,
+        logger: Optional[logging.Logger] = None,
+    ):
+        self.logger = logger if logger else logging.getLogger('eynollah.binarization')
         self.model_zoo = model_zoo
         self.models = self.setup_models(mode)
         self.session = self.start_new_session()
@@ -40,7 +44,7 @@ class SbbBinarizer:
         tensorflow_backend.set_session(session)
         return session
     
-    def setup_models(self, mode: str) -> Dict[Path, Model]:
+    def setup_models(self, mode: str) -> Dict[Path, AnyModel]:
         return {
             self.model_zoo.model_path(v): self.model_zoo.load_model(v)
             for v in (['binarization'] if mode == 'single' else [f'binarization_multi_{i}' for i in range(1, 5)])
@@ -341,17 +345,19 @@ class SbbBinarizer:
             img_last[:, :][img_last[:, :] > 0] = 255
             img_last = (img_last[:, :] == 0) * 255
             if output:
+                self.logger.info('Writing binarized image to %s', output)
                 cv2.imwrite(output, img_last)
             return img_last
         else:
             ls_imgs = list(filter(is_image_filename, os.listdir(dir_in)))
-            for image_name in ls_imgs:
+            self.logger.info("Found %d image files to binarize in %s", len(ls_imgs), dir_in)
+            for i, image_name in enumerate(ls_imgs):
                 image_stem = image_name.split('.')[0]
-                # print(image_name,'image_name')
+                self.logger.info('Binarizing [%3d/%d] %s', i + 1, len(ls_imgs), image_name)
                 image = cv2.imread(os.path.join(dir_in,image_name) )
                 img_last = 0
                 for n, (model_file, model) in enumerate(self.models.items()):
-                    self.log.info('Predicting %s with model %s [%s/%s]', image_name, model_file, n + 1, len(self.models.keys()))
+                    self.logger.info('Predicting %s with model %s [%s/%s]', image_name, model_file, n + 1, len(self.models.keys()))
 
                     res = self.predict(model, image, use_patches)
 
@@ -371,4 +377,6 @@ class SbbBinarizer:
                 img_last[:, :][img_last[:, :] > 0] = 255
                 img_last = (img_last[:, :] == 0) * 255
                 
-                cv2.imwrite(os.path.join(output, image_stem + '.png'), img_last)
+                output_filename = os.path.join(output, image_stem + '.png')
+                self.logger.info('Writing binarized image to %s', output_filename)
+                cv2.imwrite(output_filename, img_last)

src/eynollah/utils/__init__.py

@@ -19,7 +19,6 @@ from .contour import (contours_in_same_horizon,
                       find_new_features_of_contours,
                       return_contours_of_image,
                       return_parent_contours)
-
 def pairwise(iterable):
     # pairwise('ABCDEFG') → AB BC CD DE EF FG
 

src/eynollah/writer.py

@@ -3,10 +3,11 @@
 from pathlib import Path
 import os.path
 from typing import Optional
+import logging
+import xml.etree.ElementTree as ET
 from .utils.xml import create_page_xml, xml_reading_order
 from .utils.counter import EynollahIdCounter
 
-from ocrd_utils import getLogger
 from ocrd_models.ocrd_page import (
         BorderType,
         CoordsType,
@@ -23,7 +24,7 @@ import numpy as np
 class EynollahXmlWriter:
 
     def __init__(self, *, dir_out, image_filename, curved_line,textline_light, pcgts=None):
-        self.logger = getLogger('eynollah.writer')
+        self.logger = logging.getLogger('eynollah.writer')
         self.counter = EynollahIdCounter()
         self.dir_out = dir_out
         self.image_filename = image_filename

tests/cli_tests/conftest.py

@@ -32,7 +32,7 @@ def run_eynollah_ok_and_check_logs(
                 *args
         ]
         if pytestconfig.getoption('verbose') > 0:
-            args.extend(['-l', 'DEBUG'])
+            args = ['-l', 'DEBUG'] + args
         caplog.set_level(logging.INFO)
         runner = CliRunner()
         with caplog.filtering(eynollah_log_filter):