qurator-spk/eynollah
1
0
Fork 0
mirror of https://github.com/qurator-spk/eynollah.git synced 2025-11-09 22:24:13 +01:00
Code Issues Projects Releases Wiki Activity

Merge branch 'cli-logging' into model-zoo

Browse source
This commit is contained in:
kba 2025-10-29 19:41:01 +01:00
commit de76eabc1d
10 changed files with 105 additions and 107 deletions
Download patch file Download diff file Expand all files Collapse all files

129
src/eynollah/cli.py
View file

@ -1,17 +1,22 @@
from dataclasses import dataclass
import os
import sys
import click
import logging
from ocrd_utils import initLogging, getLevelName, getLogger
import sys
import os
from typing import Union
import click
from .model_zoo import EynollahModelZoo
from .cli_models import models_cli
@dataclass()
class EynollahCliCtx:
"""
Holds options relevant for all eynollah subcommands
"""
model_zoo: EynollahModelZoo
log_level : Union[str, None] = 'INFO'
@click.group()
@click.option(
@ -28,10 +33,31 @@ class EynollahCliCtx:
type=(str, str, str),
multiple=True,
)
@click.option(
"--log_level",
"-l",
type=click.Choice(['OFF', 'DEBUG', 'INFO', 'WARN', 'ERROR']),
help="Override log level globally to this",
)
@click.pass_context
def main(ctx, model_basedir, model_overrides):
def main(ctx, model_basedir, model_overrides, log_level):
"""
eynollah - Document Layout Analysis, Image Enhancement, OCR
"""
# Initialize logging
console_handler = logging.StreamHandler(sys.stdout)
console_handler.setLevel(logging.NOTSET)
formatter = logging.Formatter('%(asctime)s.%(msecs)03d %(levelname)s %(name)s - %(message)s', datefmt='%H:%M:%S')
console_handler.setFormatter(formatter)
logging.getLogger('eynollah').addHandler(console_handler)
logging.getLogger('eynollah').setLevel(log_level or logging.INFO)
# Initialize model zoo
ctx.obj = EynollahCliCtx(model_zoo=EynollahModelZoo(basedir=model_basedir, model_overrides=model_overrides))
model_zoo = EynollahModelZoo(basedir=model_basedir, model_overrides=model_overrides)
# Initialize CLI context
ctx.obj = EynollahCliCtx(
model_zoo=model_zoo,
log_level=log_level,
)
main.add_command(models_cli, 'models')
@ -55,20 +81,14 @@ main.add_command(models_cli, 'models')
type=click.Path(exists=True, file_okay=False),
required=True,
)
@click.option(
"--log_level",
"-l",
type=click.Choice(['OFF', 'DEBUG', 'INFO', 'WARN', 'ERROR']),
help="Override log level globally to this",
)
@click.pass_context
def machine_based_reading_order(ctx, input, dir_in, out, log_level):
def machine_based_reading_order(ctx, input, dir_in, out):
"""
Generate ReadingOrder with a ML model
"""
from eynollah.mb_ro_on_layout import machine_based_reading_order_on_layout
assert bool(input) != bool(dir_in), "Either -i (single input) or -di (directory) must be provided, but not both."
orderer = machine_based_reading_order_on_layout(model_zoo=ctx.obj.model_zoo)
if log_level:
orderer.logger.setLevel(getLevelName(log_level))
orderer.run(xml_filename=input,
dir_in=dir_in,
dir_out=out,
@ -103,12 +123,6 @@ def machine_based_reading_order(ctx, input, dir_in, out, log_level):
default='single',
help="Whether to use the (newer and faster) single-model binarization or the (slightly better) multi-model binarization"
)
@click.option(
"--log_level",
"-l",
type=click.Choice(['OFF', 'DEBUG', 'INFO', 'WARN', 'ERROR']),
help="Override log level globally to this",
)
@click.pass_context
def binarization(
ctx,
@ -117,13 +131,13 @@ def binarization(
mode,
dir_in,
output,
log_level,
):
"""
Binarize images with a ML model
"""
from eynollah.sbb_binarize import SbbBinarizer
assert bool(input_image) != bool(dir_in), "Either -i (single input) or -di (directory) must be provided, but not both."
binarizer = SbbBinarizer(model_zoo=ctx.obj.model_zoo, mode=mode)
if log_level:
binarizer.log.setLevel(getLevelName(log_level))
binarizer.run(
image_path=input_image,
use_patches=patches,
@ -175,25 +189,19 @@ def binarization(
is_flag=True,
help="if this parameter set to true, this tool will save the enhanced image in org scale.",
)
@click.option(
"--log_level",
"-l",
type=click.Choice(['OFF', 'DEBUG', 'INFO', 'WARN', 'ERROR']),
help="Override log level globally to this",
)
@click.pass_context
def enhancement(ctx, image, out, overwrite, dir_in, num_col_upper, num_col_lower, save_org_scale, log_level):
from eynollah.image_enhancer import Enhancer
def enhancement(ctx, image, out, overwrite, dir_in, num_col_upper, num_col_lower, save_org_scale):
"""
Enhance image
"""
assert bool(image) != bool(dir_in), "Either -i (single input) or -di (directory) must be provided, but not both."
initLogging()
from .image_enhancer import Enhancer
enhancer = Enhancer(
model_zoo=ctx.obj.model_zoo,
num_col_upper=num_col_upper,
num_col_lower=num_col_lower,
save_org_scale=save_org_scale,
)
if log_level:
enhancer.logger.setLevel(getLevelName(log_level))
enhancer.run(overwrite=overwrite,
dir_in=dir_in,
image_filename=image,
@ -384,19 +392,6 @@ def enhancement(ctx, image, out, overwrite, dir_in, num_col_upper, num_col_lower
is_flag=True,
help="if this parameter set to true, this tool will ignore layout detection and reading order. It means that textline detection will be done within printspace and contours of textline will be written in xml output file.",
)
# TODO move to top-level CLI context
@click.option(
"--log_level",
"-l",
type=click.Choice(['OFF', 'DEBUG', 'INFO', 'WARN', 'ERROR']),
help="Override 'eynollah' log level globally to this",
)
#
@click.option(
"--setup-logging",
is_flag=True,
help="Setup a basic console logger",
)
@click.pass_context
def layout(
ctx,
@ -434,16 +429,10 @@ def layout(
log_level,
setup_logging,
):
"""
Detect Layout (with optional image enhancement and reading order detection)
"""
from eynollah.eynollah import Eynollah
if setup_logging:
console_handler = logging.StreamHandler(sys.stdout)
console_handler.setLevel(logging.INFO)
formatter = logging.Formatter('%(message)s')
console_handler.setFormatter(formatter)
getLogger('eynollah').addHandler(console_handler)
getLogger('eynollah').setLevel(logging.INFO)
else:
initLogging()
assert enable_plotting or not save_layout, "Plotting with -sl also requires -ep"
assert enable_plotting or not save_deskewed, "Plotting with -sd also requires -ep"
assert enable_plotting or not save_all, "Plotting with -sa also requires -ep"
@ -463,6 +452,7 @@ def layout(
assert not extract_only_images or not right2left, "Image extraction -eoi can not be set alongside right2left -r2l"
assert not extract_only_images or not headers_off, "Image extraction -eoi can not be set alongside headers_off -ho"
assert bool(image) != bool(dir_in), "Either -i (single input) or -di (directory) must be provided, but not both."
from .eynollah import Eynollah
eynollah = Eynollah(
model_zoo=ctx.obj.model_zoo,
extract_only_images=extract_only_images,
@ -488,8 +478,6 @@ def layout(
threshold_art_class_textline=threshold_art_class_textline,
threshold_art_class_layout=threshold_art_class_layout,
)
if log_level:
eynollah.logger.setLevel(getLevelName(log_level))
eynollah.run(overwrite=overwrite,
image_filename=image,
dir_in=dir_in,
@ -579,12 +567,6 @@ def layout(
"-min_conf",
help="minimum OCR confidence value. Text lines with a confidence value lower than this threshold will not be included in the output XML file.",
)
@click.option(
"--log_level",
"-l",
type=click.Choice(['OFF', 'DEBUG', 'INFO', 'WARN', 'ERROR']),
help="Override log level globally to this",
)
@click.pass_context
def ocr(
ctx,
@ -601,11 +583,10 @@ def ocr(
batch_size,
dataset_abbrevation,
min_conf_value_of_textline_text,
log_level,
):
from eynollah.eynollah_ocr import Eynollah_ocr
initLogging()
"""
Recognize text with a CNN/RNN or transformer ML model.
"""
assert not export_textline_images_and_text or not tr_ocr, "Exporting textline and text -etit can not be set alongside transformer ocr -tr_ocr"
# FIXME: refactor: move export_textline_images_and_text out of eynollah.py
# assert not export_textline_images_and_text or not model, "Exporting textline and text -etit can not be set alongside model -m"
@ -613,6 +594,7 @@ def ocr(
assert not export_textline_images_and_text or not dir_in_bin, "Exporting textline and text -etit can not be set alongside directory of bin images -dib"
assert not export_textline_images_and_text or not dir_out_image_text, "Exporting textline and text -etit can not be set alongside directory of images with predicted text -doit"
assert bool(image) != bool(dir_in), "Either -i (single image) or -di (directory) must be provided, but not both."
from .eynollah_ocr import Eynollah_ocr
eynollah_ocr = Eynollah_ocr(
model_zoo=ctx.obj.model_zoo,
tr_ocr=tr_ocr,
@ -620,10 +602,7 @@ def ocr(
do_not_mask_with_textline_contour=do_not_mask_with_textline_contour,
batch_size=batch_size,
pref_of_dataset=dataset_abbrevation,
min_conf_value_of_textline_text=min_conf_value_of_textline_text,
)
if log_level:
eynollah_ocr.logger.setLevel(getLevelName(log_level))
min_conf_value_of_textline_text=min_conf_value_of_textline_text)
eynollah_ocr.run(overwrite=overwrite,
dir_in=dir_in,
dir_in_bin=dir_in_bin,

27
src/eynollah/eynollah.py
View file

@ -8,6 +8,15 @@
document layout analysis (segmentation) with output in PAGE-XML
"""
import logging
import sys
# cannot use importlib.resources until we move to 3.9+ forimportlib.resources.files
if sys.version_info < (3, 10):
import importlib_resources
else:
import importlib.resources as importlib_resources
from difflib import SequenceMatcher as sq
import math
import os
@ -27,7 +36,7 @@ import shapely.affinity
from scipy.signal import find_peaks
from scipy.ndimage import gaussian_filter1d
from skimage.morphology import skeletonize
from ocrd_utils import getLogger, tf_disable_interactive_logs
from ocrd_utils import tf_disable_interactive_logs
import statistics
try:
@ -42,10 +51,15 @@ except ImportError:
#os.environ['CUDA_VISIBLE_DEVICES'] = '-1'
tf_disable_interactive_logs()
import tensorflow as tf
tf.get_logger().setLevel("ERROR")
warnings.filterwarnings("ignore")
# warnings.filterwarnings("ignore")
from tensorflow.python.keras import backend as K
from tensorflow.keras.models import load_model
# use tf1 compatibility for keras backend
from tensorflow.compat.v1.keras.backend import set_session
from tensorflow.keras import layers
from tensorflow.keras.layers import StringLookup
from .model_zoo import (EynollahModelZoo, KerasModel, TrOCRProcessor)
from .model_zoo import EynollahModelZoo
from .utils.contour import (
filter_contours_area_of_image,
filter_contours_area_of_image_tables,
@ -162,8 +176,9 @@ class Eynollah:
threshold_art_class_layout: Optional[float] = None,
threshold_art_class_textline: Optional[float] = None,
skip_layout_and_reading_order : bool = False,
logger : Optional[logging.Logger] = None,
):
self.logger = getLogger('eynollah')
self.logger = logger or logging.getLogger('eynollah')
self.model_zoo = model_zoo
self.plotter = None
@ -4724,5 +4739,3 @@ class Eynollah:
conf_contours_textregions=conf_contours_textregions)
return pcgts

6
src/eynollah/image_enhancer.py
View file

@ -2,7 +2,7 @@
Image enhancer. The output can be written as same scale of input or in new predicted scale.
"""
from logging import Logger
import logging
import os
import time
from typing import Dict, Optional
@ -12,7 +12,6 @@ import gc
import cv2
from keras.models import Model
import numpy as np
from ocrd_utils import getLogger, tf_disable_interactive_logs
import tensorflow as tf
from skimage.morphology import skeletonize
@ -37,7 +36,6 @@ class Enhancer:
num_col_upper : Optional[int] = None,
num_col_lower : Optional[int] = None,
save_org_scale : bool = False,
logger : Optional[Logger] = None,
):
self.input_binary = False
self.light_version = False
@ -51,7 +49,7 @@ class Enhancer:
else:
self.num_col_lower = num_col_lower
self.logger = logger if logger else getLogger('eynollah.enhance')
self.logger = logging.getLogger('eynollah.enhance')
self.model_zoo = model_zoo
for v in ['binarization', 'enhancement', 'col_classifier', 'page']:
self.model_zoo.load_model(v)

9
src/eynollah/mb_ro_on_layout.py
View file

@ -1,8 +1,8 @@
"""
Image enhancer. The output can be written as same scale of input or in new predicted scale.
Machine learning based reading order detection
"""
from logging import Logger
import logging
import os
import time
from typing import Optional
@ -12,7 +12,6 @@ import xml.etree.ElementTree as ET
import cv2
from keras.models import Model
import numpy as np
from ocrd_utils import getLogger
import statistics
import tensorflow as tf
@ -34,9 +33,9 @@ class machine_based_reading_order_on_layout:
self,
*,
model_zoo: EynollahModelZoo,
logger : Optional[Logger] = None,
logger : Optional[logging.Logger] = None,
):
self.logger = logger if logger else getLogger('mbreorder')
self.logger = logger or logging.getLogger('eynollah.mbreorder')
self.model_zoo = model_zoo
try:

1
src/eynollah/ocrd_cli_binarization.py
View file

@ -34,6 +34,7 @@ class SbbBinarizeProcessor(Processor):
Set up the model prior to processing.
"""
# resolve relative path via OCR-D ResourceManager
assert isinstance(self.parameter, dict)
model_path = self.resolve_resource(self.parameter['model'])
self.binarizer = SbbBinarizer(model_dir=model_path, logger=self.logger)

2
src/eynollah/processor.py
View file

@ -32,8 +32,8 @@ class EynollahProcessor(Processor):
allow_scaling=self.parameter['allow_scaling'],
headers_off=self.parameter['headers_off'],
tables=self.parameter['tables'],
logger=self.logger
)
self.eynollah.logger = self.logger
self.eynollah.plotter = None
def shutdown(self):

30
src/eynollah/sbb_binarize.py
View file

@ -5,14 +5,14 @@ Tool to load model and binarize a given image.
import os
import logging
from pathlib import Path
from typing import Dict, List
from typing import Dict, List, Optional
from keras.models import Model
import numpy as np
import cv2
from ocrd_utils import tf_disable_interactive_logs
from eynollah.model_zoo import EynollahModelZoo
from eynollah.model_zoo.types import AnyModel
tf_disable_interactive_logs()
import tensorflow as tf
from tensorflow.python.keras import backend as tensorflow_backend
@ -24,10 +24,14 @@ def resize_image(img_in, input_height, input_width):
class SbbBinarizer:
def __init__(self, *, model_zoo: EynollahModelZoo, mode: str, logger=None):
if mode not in ('single', 'multi'):
raise ValueError(f"'mode' must be either 'multi' or 'single', not {mode}")
self.log = logger if logger else logging.getLogger('eynollah.binarization')
def __init__(
self,
*,
model_zoo: EynollahModelZoo,
mode: str,
logger: Optional[logging.Logger] = None,
):
self.logger = logger if logger else logging.getLogger('eynollah.binarization')
self.model_zoo = model_zoo
self.models = self.setup_models(mode)
self.session = self.start_new_session()
@ -40,7 +44,7 @@ class SbbBinarizer:
tensorflow_backend.set_session(session)
return session
def setup_models(self, mode: str) -> Dict[Path, Model]:
def setup_models(self, mode: str) -> Dict[Path, AnyModel]:
return {
self.model_zoo.model_path(v): self.model_zoo.load_model(v)
for v in (['binarization'] if mode == 'single' else [f'binarization_multi_{i}' for i in range(1, 5)])
@ -341,17 +345,19 @@ class SbbBinarizer:
img_last[:, :][img_last[:, :] > 0] = 255
img_last = (img_last[:, :] == 0) * 255
if output:
self.logger.info('Writing binarized image to %s', output)
cv2.imwrite(output, img_last)
return img_last
else:
ls_imgs = list(filter(is_image_filename, os.listdir(dir_in)))
for image_name in ls_imgs:
self.logger.info("Found %d image files to binarize in %s", len(ls_imgs), dir_in)
for i, image_name in enumerate(ls_imgs):
image_stem = image_name.split('.')[0]
# print(image_name,'image_name')
self.logger.info('Binarizing [%3d/%d] %s', i + 1, len(ls_imgs), image_name)
image = cv2.imread(os.path.join(dir_in,image_name) )
img_last = 0
for n, (model_file, model) in enumerate(self.models.items()):
self.log.info('Predicting %s with model %s [%s/%s]', image_name, model_file, n + 1, len(self.models.keys()))
self.logger.info('Predicting %s with model %s [%s/%s]', image_name, model_file, n + 1, len(self.models.keys()))
res = self.predict(model, image, use_patches)
@ -371,4 +377,6 @@ class SbbBinarizer:
img_last[:, :][img_last[:, :] > 0] = 255
img_last = (img_last[:, :] == 0) * 255
cv2.imwrite(os.path.join(output, image_stem + '.png'), img_last)
output_filename = os.path.join(output, image_stem + '.png')
self.logger.info('Writing binarized image to %s', output_filename)
cv2.imwrite(output_filename, img_last)

1
src/eynollah/utils/__init__.py
View file

@ -19,7 +19,6 @@ from .contour import (contours_in_same_horizon,
find_new_features_of_contours,
return_contours_of_image,
return_parent_contours)
def pairwise(iterable):
# pairwise('ABCDEFG') → AB BC CD DE EF FG

5
src/eynollah/writer.py
View file

@ -3,10 +3,11 @@
from pathlib import Path
import os.path
from typing import Optional
import logging
import xml.etree.ElementTree as ET
from .utils.xml import create_page_xml, xml_reading_order
from .utils.counter import EynollahIdCounter
from ocrd_utils import getLogger
from ocrd_models.ocrd_page import (
BorderType,
CoordsType,
@ -23,7 +24,7 @@ import numpy as np
class EynollahXmlWriter:
def __init__(self, *, dir_out, image_filename, curved_line,textline_light, pcgts=None):
self.logger = getLogger('eynollah.writer')
self.logger = logging.getLogger('eynollah.writer')
self.counter = EynollahIdCounter()
self.dir_out = dir_out
self.image_filename = image_filename

2
tests/cli_tests/conftest.py
View file

@ -32,7 +32,7 @@ def run_eynollah_ok_and_check_logs(
*args
]
if pytestconfig.getoption('verbose') > 0:
args.extend(['-l', 'DEBUG'])
args = ['-l', 'DEBUG'] + args
caplog.set_level(logging.INFO)
runner = CliRunner()
with caplog.filtering(eynollah_log_filter):