diff --git a/src/eynollah/cli.py b/src/eynollah/cli.py index bd2d807..af1b805 100644 --- a/src/eynollah/cli.py +++ b/src/eynollah/cli.py @@ -1,17 +1,22 @@ from dataclasses import dataclass -import os -import sys -import click import logging -from ocrd_utils import initLogging, getLevelName, getLogger +import sys +import os +from typing import Union + +import click from .model_zoo import EynollahModelZoo - from .cli_models import models_cli @dataclass() class EynollahCliCtx: + """ + Holds options relevant for all eynollah subcommands + """ model_zoo: EynollahModelZoo + log_level : Union[str, None] = 'INFO' + @click.group() @click.option( @@ -28,10 +33,31 @@ class EynollahCliCtx: type=(str, str, str), multiple=True, ) +@click.option( + "--log_level", + "-l", + type=click.Choice(['OFF', 'DEBUG', 'INFO', 'WARN', 'ERROR']), + help="Override log level globally to this", +) @click.pass_context -def main(ctx, model_basedir, model_overrides): +def main(ctx, model_basedir, model_overrides, log_level): + """ + eynollah - Document Layout Analysis, Image Enhancement, OCR + """ + # Initialize logging + console_handler = logging.StreamHandler(sys.stdout) + console_handler.setLevel(logging.NOTSET) + formatter = logging.Formatter('%(asctime)s.%(msecs)03d %(levelname)s %(name)s - %(message)s', datefmt='%H:%M:%S') + console_handler.setFormatter(formatter) + logging.getLogger('eynollah').addHandler(console_handler) + logging.getLogger('eynollah').setLevel(log_level or logging.INFO) # Initialize model zoo - ctx.obj = EynollahCliCtx(model_zoo=EynollahModelZoo(basedir=model_basedir, model_overrides=model_overrides)) + model_zoo = EynollahModelZoo(basedir=model_basedir, model_overrides=model_overrides) + # Initialize CLI context + ctx.obj = EynollahCliCtx( + model_zoo=model_zoo, + log_level=log_level, + ) main.add_command(models_cli, 'models') @@ -55,20 +81,14 @@ main.add_command(models_cli, 'models') type=click.Path(exists=True, file_okay=False), required=True, ) -@click.option( - "--log_level", - "-l", - type=click.Choice(['OFF', 'DEBUG', 'INFO', 'WARN', 'ERROR']), - help="Override log level globally to this", -) @click.pass_context -def machine_based_reading_order(ctx, input, dir_in, out, log_level): +def machine_based_reading_order(ctx, input, dir_in, out): + """ + Generate ReadingOrder with a ML model + """ from eynollah.mb_ro_on_layout import machine_based_reading_order_on_layout assert bool(input) != bool(dir_in), "Either -i (single input) or -di (directory) must be provided, but not both." orderer = machine_based_reading_order_on_layout(model_zoo=ctx.obj.model_zoo) - if log_level: - orderer.logger.setLevel(getLevelName(log_level)) - orderer.run(xml_filename=input, dir_in=dir_in, dir_out=out, @@ -103,12 +123,6 @@ def machine_based_reading_order(ctx, input, dir_in, out, log_level): default='single', help="Whether to use the (newer and faster) single-model binarization or the (slightly better) multi-model binarization" ) -@click.option( - "--log_level", - "-l", - type=click.Choice(['OFF', 'DEBUG', 'INFO', 'WARN', 'ERROR']), - help="Override log level globally to this", -) @click.pass_context def binarization( ctx, @@ -117,13 +131,13 @@ def binarization( mode, dir_in, output, - log_level, ): + """ + Binarize images with a ML model + """ from eynollah.sbb_binarize import SbbBinarizer assert bool(input_image) != bool(dir_in), "Either -i (single input) or -di (directory) must be provided, but not both." binarizer = SbbBinarizer(model_zoo=ctx.obj.model_zoo, mode=mode) - if log_level: - binarizer.log.setLevel(getLevelName(log_level)) binarizer.run( image_path=input_image, use_patches=patches, @@ -175,25 +189,19 @@ def binarization( is_flag=True, help="if this parameter set to true, this tool will save the enhanced image in org scale.", ) -@click.option( - "--log_level", - "-l", - type=click.Choice(['OFF', 'DEBUG', 'INFO', 'WARN', 'ERROR']), - help="Override log level globally to this", -) @click.pass_context -def enhancement(ctx, image, out, overwrite, dir_in, num_col_upper, num_col_lower, save_org_scale, log_level): - from eynollah.image_enhancer import Enhancer +def enhancement(ctx, image, out, overwrite, dir_in, num_col_upper, num_col_lower, save_org_scale): + """ + Enhance image + """ assert bool(image) != bool(dir_in), "Either -i (single input) or -di (directory) must be provided, but not both." - initLogging() + from .image_enhancer import Enhancer enhancer = Enhancer( model_zoo=ctx.obj.model_zoo, num_col_upper=num_col_upper, num_col_lower=num_col_lower, save_org_scale=save_org_scale, ) - if log_level: - enhancer.logger.setLevel(getLevelName(log_level)) enhancer.run(overwrite=overwrite, dir_in=dir_in, image_filename=image, @@ -384,19 +392,6 @@ def enhancement(ctx, image, out, overwrite, dir_in, num_col_upper, num_col_lower is_flag=True, help="if this parameter set to true, this tool will ignore layout detection and reading order. It means that textline detection will be done within printspace and contours of textline will be written in xml output file.", ) -# TODO move to top-level CLI context -@click.option( - "--log_level", - "-l", - type=click.Choice(['OFF', 'DEBUG', 'INFO', 'WARN', 'ERROR']), - help="Override 'eynollah' log level globally to this", -) -# -@click.option( - "--setup-logging", - is_flag=True, - help="Setup a basic console logger", -) @click.pass_context def layout( ctx, @@ -434,16 +429,10 @@ def layout( log_level, setup_logging, ): + """ + Detect Layout (with optional image enhancement and reading order detection) + """ from eynollah.eynollah import Eynollah - if setup_logging: - console_handler = logging.StreamHandler(sys.stdout) - console_handler.setLevel(logging.INFO) - formatter = logging.Formatter('%(message)s') - console_handler.setFormatter(formatter) - getLogger('eynollah').addHandler(console_handler) - getLogger('eynollah').setLevel(logging.INFO) - else: - initLogging() assert enable_plotting or not save_layout, "Plotting with -sl also requires -ep" assert enable_plotting or not save_deskewed, "Plotting with -sd also requires -ep" assert enable_plotting or not save_all, "Plotting with -sa also requires -ep" @@ -463,6 +452,7 @@ def layout( assert not extract_only_images or not right2left, "Image extraction -eoi can not be set alongside right2left -r2l" assert not extract_only_images or not headers_off, "Image extraction -eoi can not be set alongside headers_off -ho" assert bool(image) != bool(dir_in), "Either -i (single input) or -di (directory) must be provided, but not both." + from .eynollah import Eynollah eynollah = Eynollah( model_zoo=ctx.obj.model_zoo, extract_only_images=extract_only_images, @@ -488,8 +478,6 @@ def layout( threshold_art_class_textline=threshold_art_class_textline, threshold_art_class_layout=threshold_art_class_layout, ) - if log_level: - eynollah.logger.setLevel(getLevelName(log_level)) eynollah.run(overwrite=overwrite, image_filename=image, dir_in=dir_in, @@ -579,12 +567,6 @@ def layout( "-min_conf", help="minimum OCR confidence value. Text lines with a confidence value lower than this threshold will not be included in the output XML file.", ) -@click.option( - "--log_level", - "-l", - type=click.Choice(['OFF', 'DEBUG', 'INFO', 'WARN', 'ERROR']), - help="Override log level globally to this", -) @click.pass_context def ocr( ctx, @@ -601,11 +583,10 @@ def ocr( batch_size, dataset_abbrevation, min_conf_value_of_textline_text, - log_level, ): - from eynollah.eynollah_ocr import Eynollah_ocr - initLogging() - + """ + Recognize text with a CNN/RNN or transformer ML model. + """ assert not export_textline_images_and_text or not tr_ocr, "Exporting textline and text -etit can not be set alongside transformer ocr -tr_ocr" # FIXME: refactor: move export_textline_images_and_text out of eynollah.py # assert not export_textline_images_and_text or not model, "Exporting textline and text -etit can not be set alongside model -m" @@ -613,6 +594,7 @@ def ocr( assert not export_textline_images_and_text or not dir_in_bin, "Exporting textline and text -etit can not be set alongside directory of bin images -dib" assert not export_textline_images_and_text or not dir_out_image_text, "Exporting textline and text -etit can not be set alongside directory of images with predicted text -doit" assert bool(image) != bool(dir_in), "Either -i (single image) or -di (directory) must be provided, but not both." + from .eynollah_ocr import Eynollah_ocr eynollah_ocr = Eynollah_ocr( model_zoo=ctx.obj.model_zoo, tr_ocr=tr_ocr, @@ -620,10 +602,7 @@ def ocr( do_not_mask_with_textline_contour=do_not_mask_with_textline_contour, batch_size=batch_size, pref_of_dataset=dataset_abbrevation, - min_conf_value_of_textline_text=min_conf_value_of_textline_text, - ) - if log_level: - eynollah_ocr.logger.setLevel(getLevelName(log_level)) + min_conf_value_of_textline_text=min_conf_value_of_textline_text) eynollah_ocr.run(overwrite=overwrite, dir_in=dir_in, dir_in_bin=dir_in_bin, diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 867d86b..ab04a67 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -8,6 +8,15 @@ document layout analysis (segmentation) with output in PAGE-XML """ +import logging +import sys + +# cannot use importlib.resources until we move to 3.9+ forimportlib.resources.files +if sys.version_info < (3, 10): + import importlib_resources +else: + import importlib.resources as importlib_resources + from difflib import SequenceMatcher as sq import math import os @@ -27,7 +36,7 @@ import shapely.affinity from scipy.signal import find_peaks from scipy.ndimage import gaussian_filter1d from skimage.morphology import skeletonize -from ocrd_utils import getLogger, tf_disable_interactive_logs +from ocrd_utils import tf_disable_interactive_logs import statistics try: @@ -42,10 +51,15 @@ except ImportError: #os.environ['CUDA_VISIBLE_DEVICES'] = '-1' tf_disable_interactive_logs() import tensorflow as tf -tf.get_logger().setLevel("ERROR") -warnings.filterwarnings("ignore") +# warnings.filterwarnings("ignore") +from tensorflow.python.keras import backend as K +from tensorflow.keras.models import load_model +# use tf1 compatibility for keras backend +from tensorflow.compat.v1.keras.backend import set_session +from tensorflow.keras import layers +from tensorflow.keras.layers import StringLookup -from .model_zoo import (EynollahModelZoo, KerasModel, TrOCRProcessor) +from .model_zoo import EynollahModelZoo from .utils.contour import ( filter_contours_area_of_image, filter_contours_area_of_image_tables, @@ -162,8 +176,9 @@ class Eynollah: threshold_art_class_layout: Optional[float] = None, threshold_art_class_textline: Optional[float] = None, skip_layout_and_reading_order : bool = False, + logger : Optional[logging.Logger] = None, ): - self.logger = getLogger('eynollah') + self.logger = logger or logging.getLogger('eynollah') self.model_zoo = model_zoo self.plotter = None @@ -4724,5 +4739,3 @@ class Eynollah: conf_contours_textregions=conf_contours_textregions) return pcgts - - diff --git a/src/eynollah/image_enhancer.py b/src/eynollah/image_enhancer.py index 08d3d90..00aedec 100644 --- a/src/eynollah/image_enhancer.py +++ b/src/eynollah/image_enhancer.py @@ -2,7 +2,7 @@ Image enhancer. The output can be written as same scale of input or in new predicted scale. """ -from logging import Logger +import logging import os import time from typing import Dict, Optional @@ -12,7 +12,6 @@ import gc import cv2 from keras.models import Model import numpy as np -from ocrd_utils import getLogger, tf_disable_interactive_logs import tensorflow as tf from skimage.morphology import skeletonize @@ -37,7 +36,6 @@ class Enhancer: num_col_upper : Optional[int] = None, num_col_lower : Optional[int] = None, save_org_scale : bool = False, - logger : Optional[Logger] = None, ): self.input_binary = False self.light_version = False @@ -51,7 +49,7 @@ class Enhancer: else: self.num_col_lower = num_col_lower - self.logger = logger if logger else getLogger('eynollah.enhance') + self.logger = logging.getLogger('eynollah.enhance') self.model_zoo = model_zoo for v in ['binarization', 'enhancement', 'col_classifier', 'page']: self.model_zoo.load_model(v) diff --git a/src/eynollah/mb_ro_on_layout.py b/src/eynollah/mb_ro_on_layout.py index 620d6c0..3527103 100644 --- a/src/eynollah/mb_ro_on_layout.py +++ b/src/eynollah/mb_ro_on_layout.py @@ -1,8 +1,8 @@ """ -Image enhancer. The output can be written as same scale of input or in new predicted scale. +Machine learning based reading order detection """ -from logging import Logger +import logging import os import time from typing import Optional @@ -12,7 +12,6 @@ import xml.etree.ElementTree as ET import cv2 from keras.models import Model import numpy as np -from ocrd_utils import getLogger import statistics import tensorflow as tf @@ -34,9 +33,9 @@ class machine_based_reading_order_on_layout: self, *, model_zoo: EynollahModelZoo, - logger : Optional[Logger] = None, + logger : Optional[logging.Logger] = None, ): - self.logger = logger if logger else getLogger('mbreorder') + self.logger = logger or logging.getLogger('eynollah.mbreorder') self.model_zoo = model_zoo try: diff --git a/src/eynollah/ocrd_cli_binarization.py b/src/eynollah/ocrd_cli_binarization.py index 848bbac..e5f85b1 100644 --- a/src/eynollah/ocrd_cli_binarization.py +++ b/src/eynollah/ocrd_cli_binarization.py @@ -34,6 +34,7 @@ class SbbBinarizeProcessor(Processor): Set up the model prior to processing. """ # resolve relative path via OCR-D ResourceManager + assert isinstance(self.parameter, dict) model_path = self.resolve_resource(self.parameter['model']) self.binarizer = SbbBinarizer(model_dir=model_path, logger=self.logger) diff --git a/src/eynollah/processor.py b/src/eynollah/processor.py index 12c7356..60c136c 100644 --- a/src/eynollah/processor.py +++ b/src/eynollah/processor.py @@ -32,8 +32,8 @@ class EynollahProcessor(Processor): allow_scaling=self.parameter['allow_scaling'], headers_off=self.parameter['headers_off'], tables=self.parameter['tables'], + logger=self.logger ) - self.eynollah.logger = self.logger self.eynollah.plotter = None def shutdown(self): diff --git a/src/eynollah/sbb_binarize.py b/src/eynollah/sbb_binarize.py index a8a05fa..753a626 100644 --- a/src/eynollah/sbb_binarize.py +++ b/src/eynollah/sbb_binarize.py @@ -5,14 +5,14 @@ Tool to load model and binarize a given image. import os import logging from pathlib import Path -from typing import Dict, List +from typing import Dict, List, Optional -from keras.models import Model import numpy as np import cv2 from ocrd_utils import tf_disable_interactive_logs from eynollah.model_zoo import EynollahModelZoo +from eynollah.model_zoo.types import AnyModel tf_disable_interactive_logs() import tensorflow as tf from tensorflow.python.keras import backend as tensorflow_backend @@ -24,10 +24,14 @@ def resize_image(img_in, input_height, input_width): class SbbBinarizer: - def __init__(self, *, model_zoo: EynollahModelZoo, mode: str, logger=None): - if mode not in ('single', 'multi'): - raise ValueError(f"'mode' must be either 'multi' or 'single', not {mode}") - self.log = logger if logger else logging.getLogger('eynollah.binarization') + def __init__( + self, + *, + model_zoo: EynollahModelZoo, + mode: str, + logger: Optional[logging.Logger] = None, + ): + self.logger = logger if logger else logging.getLogger('eynollah.binarization') self.model_zoo = model_zoo self.models = self.setup_models(mode) self.session = self.start_new_session() @@ -40,7 +44,7 @@ class SbbBinarizer: tensorflow_backend.set_session(session) return session - def setup_models(self, mode: str) -> Dict[Path, Model]: + def setup_models(self, mode: str) -> Dict[Path, AnyModel]: return { self.model_zoo.model_path(v): self.model_zoo.load_model(v) for v in (['binarization'] if mode == 'single' else [f'binarization_multi_{i}' for i in range(1, 5)]) @@ -341,17 +345,19 @@ class SbbBinarizer: img_last[:, :][img_last[:, :] > 0] = 255 img_last = (img_last[:, :] == 0) * 255 if output: + self.logger.info('Writing binarized image to %s', output) cv2.imwrite(output, img_last) return img_last else: ls_imgs = list(filter(is_image_filename, os.listdir(dir_in))) - for image_name in ls_imgs: + self.logger.info("Found %d image files to binarize in %s", len(ls_imgs), dir_in) + for i, image_name in enumerate(ls_imgs): image_stem = image_name.split('.')[0] - # print(image_name,'image_name') + self.logger.info('Binarizing [%3d/%d] %s', i + 1, len(ls_imgs), image_name) image = cv2.imread(os.path.join(dir_in,image_name) ) img_last = 0 for n, (model_file, model) in enumerate(self.models.items()): - self.log.info('Predicting %s with model %s [%s/%s]', image_name, model_file, n + 1, len(self.models.keys())) + self.logger.info('Predicting %s with model %s [%s/%s]', image_name, model_file, n + 1, len(self.models.keys())) res = self.predict(model, image, use_patches) @@ -371,4 +377,6 @@ class SbbBinarizer: img_last[:, :][img_last[:, :] > 0] = 255 img_last = (img_last[:, :] == 0) * 255 - cv2.imwrite(os.path.join(output, image_stem + '.png'), img_last) + output_filename = os.path.join(output, image_stem + '.png') + self.logger.info('Writing binarized image to %s', output_filename) + cv2.imwrite(output_filename, img_last) diff --git a/src/eynollah/utils/__init__.py b/src/eynollah/utils/__init__.py index 94f6983..29359eb 100644 --- a/src/eynollah/utils/__init__.py +++ b/src/eynollah/utils/__init__.py @@ -19,7 +19,6 @@ from .contour import (contours_in_same_horizon, find_new_features_of_contours, return_contours_of_image, return_parent_contours) - def pairwise(iterable): # pairwise('ABCDEFG') → AB BC CD DE EF FG diff --git a/src/eynollah/writer.py b/src/eynollah/writer.py index a0ec077..38b7b9e 100644 --- a/src/eynollah/writer.py +++ b/src/eynollah/writer.py @@ -3,10 +3,11 @@ from pathlib import Path import os.path from typing import Optional +import logging +import xml.etree.ElementTree as ET from .utils.xml import create_page_xml, xml_reading_order from .utils.counter import EynollahIdCounter -from ocrd_utils import getLogger from ocrd_models.ocrd_page import ( BorderType, CoordsType, @@ -23,7 +24,7 @@ import numpy as np class EynollahXmlWriter: def __init__(self, *, dir_out, image_filename, curved_line,textline_light, pcgts=None): - self.logger = getLogger('eynollah.writer') + self.logger = logging.getLogger('eynollah.writer') self.counter = EynollahIdCounter() self.dir_out = dir_out self.image_filename = image_filename diff --git a/tests/cli_tests/conftest.py b/tests/cli_tests/conftest.py index 223cc85..601d76b 100644 --- a/tests/cli_tests/conftest.py +++ b/tests/cli_tests/conftest.py @@ -32,7 +32,7 @@ def run_eynollah_ok_and_check_logs( *args ] if pytestconfig.getoption('verbose') > 0: - args.extend(['-l', 'DEBUG']) + args = ['-l', 'DEBUG'] + args caplog.set_level(logging.INFO) runner = CliRunner() with caplog.filtering(eynollah_log_filter):