From dfa651ef8a1589aebd28357d95436beb74f37186 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Fri, 12 Jun 2026 22:21:06 +0200 Subject: [PATCH 01/21] predictor: show full stacktrace before passing the exception over --- src/eynollah/predictor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/eynollah/predictor.py b/src/eynollah/predictor.py index 6790676..526223a 100644 --- a/src/eynollah/predictor.py +++ b/src/eynollah/predictor.py @@ -214,7 +214,7 @@ class Predictor(mp.context.SpawnProcess): self.resultq.put((jobid, result)) #self.logger.debug("sent result for '%d': %s", jobid, result) except Exception as e: - self.logger.error("prediction for %s failed: %s", self.name, e.__class__.__name__) + self.logger.exception("prediction for %s failed: %s", self.name, e.__class__.__name__) result = e self.resultq.put((jobid, result)) close_all() From eb4cae9dee0ab625e08df3a23cd931dcebfae67f Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Tue, 16 Jun 2026 17:28:10 +0200 Subject: [PATCH 02/21] training.models for cnn-rnn-ocr: avoid `Conv1D(..channels_first..)` --- src/eynollah/training/models.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/eynollah/training/models.py b/src/eynollah/training/models.py index b3d811a..9111225 100644 --- a/src/eynollah/training/models.py +++ b/src/eynollah/training/models.py @@ -21,6 +21,7 @@ from tensorflow.keras.layers import ( LSTM, MaxPooling2D, MultiHeadAttention, + Permute, Reshape, UpSampling2D, ZeroPadding2D, @@ -70,7 +71,7 @@ class CTCDecoder(Layer): ## but Keras greedy sometimes removes arbitrary letters # outputs, logits = tf.keras.backend.ctc_decode(inputs, # lengths, - # beam_width=20 + # beam_width=20, # greedy=False, # True, # # backend does not allow these kwargs # #merge_repeated=False, @@ -530,10 +531,12 @@ def cnn_rnn_ocr_model(input_height=None, input_width=None, n_classes=None, max_l addition_rnn = Bidirectional(LSTM(input_width, return_sequences=True, dropout=0.25))(addition) - out = Conv1D(max_len, 1, data_format="channels_first")(addition_rnn) + #out = Conv1D(max_len, 1, data_format="channels_first")(addition_rnn) + out = Permute((2, 1))(addition_rnn) + out = Conv1D(max_len, 1, data_format="channels_last")(out) + out = Permute((2, 1))(out) out = BatchNormalization(name="bn9")(out) out = Activation("relu", name="relu9")(out) - #out = Conv1D(n_classes, 1, activation='relu', data_format="channels_last")(out) out = Dense(n_classes, activation="softmax", name="dense2")(out) From 0bfbbfdc801de3ea339f1a187f42c9625037ee5f Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Fri, 19 Jun 2026 22:01:02 +0200 Subject: [PATCH 03/21] training.metrics: allow module init without TFA --- src/eynollah/training/metrics.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/eynollah/training/metrics.py b/src/eynollah/training/metrics.py index caa0e65..2b4fc4f 100644 --- a/src/eynollah/training/metrics.py +++ b/src/eynollah/training/metrics.py @@ -5,7 +5,10 @@ import tensorflow as tf from tensorflow.keras import backend as K from tensorflow.keras.metrics import Metric, MeanMetricWrapper, get from tensorflow.keras.initializers import Zeros -from tensorflow_addons.image import connected_components +try: + from tensorflow_addons.image import connected_components +except ModuleNotFoundError: + pass # n/a beyond TF 2.15 (and only needed for training) ... import numpy as np From 42a3751e6392f295c79fc3e6a1d8112a0d909df7 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Fri, 19 Jun 2026 22:01:39 +0200 Subject: [PATCH 04/21] ModelZoo ONNX: avoid verbose logging --- src/eynollah/model_zoo/model_zoo.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/eynollah/model_zoo/model_zoo.py b/src/eynollah/model_zoo/model_zoo.py index 13656f4..232a28a 100644 --- a/src/eynollah/model_zoo/model_zoo.py +++ b/src/eynollah/model_zoo/model_zoo.py @@ -307,6 +307,8 @@ class EynollahModelZoo: import onnxruntime as ort import numpy as np + ort.set_default_logger_severity(3) + providers = ort.get_available_providers() if device: if ':' in device: From ef47f0ef09f2fa7993bc7251ad12250215d2a253 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Fri, 26 Jun 2026 02:11:59 +0200 Subject: [PATCH 05/21] training convert: only add characters_org.txt if it exists --- src/eynollah/training/convert.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/eynollah/training/convert.py b/src/eynollah/training/convert.py index d2d7b49..5bf30a1 100644 --- a/src/eynollah/training/convert.py +++ b/src/eynollah/training/convert.py @@ -68,12 +68,13 @@ def convert_cli(rebuild, format_, in_, out): ex.add_config(str(config_path)) # some models deviate between training and inference ex.add_config(inference=True) - # make sure the local vocab file gets re-used + # OCR models: make sure the local vocab file gets re-used, if available characters_txt_file = model_path / "characters_org.txt" - with open(characters_txt_file, "r") as voc_file: - voc = json.load(voc_file) - ex.add_config(characters_txt_file=characters_txt_file) - ex.add_config(n_classes=len(voc) + 3) + if characters_txt_file.exists(): + with open(characters_txt_file, "r") as voc_file: + voc = json.load(voc_file) + ex.add_config(characters_txt_file=characters_txt_file) + ex.add_config(n_classes=len(voc) + 3) # just retrieve final config (via pseudo-run) ex.main(lambda: 0) config = ex.run(options={'--loglevel': 'ERROR'}).config From 45168178dcb6bce14c0d65ccfaebf4cfa74bd899 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Fri, 26 Jun 2026 02:13:39 +0200 Subject: [PATCH 06/21] ModelZoo ONNX backend: log configured top provider (backend) --- src/eynollah/model_zoo/model_zoo.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/eynollah/model_zoo/model_zoo.py b/src/eynollah/model_zoo/model_zoo.py index 232a28a..d6b5e38 100644 --- a/src/eynollah/model_zoo/model_zoo.py +++ b/src/eynollah/model_zoo/model_zoo.py @@ -351,10 +351,15 @@ class EynollahModelZoo: # 'trt_timing_cache_enable': True, # ... })] + providers + provider0 = providers[0] + if isinstance(provider0, tuple): + provider0 = provider0[0] + self.logger.info("using %s with ONNX provider %s for model %s", + "GPU %d" % gpu if gpu >= 0 else "CPU", + provider0[:-17], model_category) model = ort.InferenceSession( model_path, providers=providers) - # FIXME: notify about selected provider/device model_inputs = [model_input.name for model_input in model.get_inputs()] model_outputs = [model_output.name From 948d841a7d58d7737b2abd58bbdced4a9f9d222d Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Fri, 26 Jun 2026 02:21:47 +0200 Subject: [PATCH 07/21] =?UTF-8?q?training.models=20for=20cnn-rnn-ocr:=20ma?= =?UTF-8?q?ke=20ONNX=20convertible=E2=80=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - `training.models.CTCDecoder`: switch back from `tf.nn.ctc_beam_search_decoder()` to `tf.nn.ctc_greedy_decoder()` (because ONNX only implements `CTCGreedyDecoder`) - `training.models.cnn_rnn_ocr_model(inference=True)` and `training.models.cnn_rnn_ocr_model4inference`: drop layer `tf.io.decode_raw()` (because ONNX does not implement `DecodePaddedRaw`) - `Eynollah_ocr.run_cnn()`: expect bytes arrays from predictor instead of uint8 - `predictor`: to prevent segfaults when sending `tf.string` results via `shared_memory`, convert `np.object` to `np.bytes_` directly --- src/eynollah/eynollah_ocr.py | 10 ++++------ src/eynollah/predictor.py | 12 +++++++++--- src/eynollah/training/models.py | 15 +++++---------- src/eynollah/training/reload-models-v0.8.mk | 2 -- 4 files changed, 18 insertions(+), 21 deletions(-) diff --git a/src/eynollah/eynollah_ocr.py b/src/eynollah/eynollah_ocr.py index 9ed02d4..9cc2c9a 100644 --- a/src/eynollah/eynollah_ocr.py +++ b/src/eynollah/eynollah_ocr.py @@ -296,13 +296,11 @@ class Eynollah_ocr(Eynollah): probs[ver_index > 0][flipped_ver_is_better] = probs_ver[flipped_ver_is_better] def nooov(x): - return x != b'[UNK]' + if x == b'[UNK]': + return b'' + return x for pred, prob in zip(preds, probs): - text = b''.join( - filter(nooov, - map(bytes, - (filter(None, char) - for char in pred.tolist())))).decode('utf-8') + text = b''.join(map(nooov, pred.tolist())).decode('utf-8') extracted_texts.append(text) extracted_confs.append(prob) del cropped_lines_rgb diff --git a/src/eynollah/predictor.py b/src/eynollah/predictor.py index 526223a..6b7fe96 100644 --- a/src/eynollah/predictor.py +++ b/src/eynollah/predictor.py @@ -191,13 +191,19 @@ class Predictor(mp.context.SpawnProcess): #result = self.model.predict(data, verbose=0) # faster, less VRAM result = self.model.predict_on_batch(data) - if isinstance(result, tuple): + def make_shareable(x): + # convert tf.string/np.object to fixed-length bytes + # (because object segfaults in shm) + if x.dtype is np.dtype(object): + return x.astype(bytes) + return x + if isinstance(result, (list, tuple)): multi_output = True - results = zip(*(np.split(result0, len(jobs)) + results = zip(*(np.split(make_shareable(result0), len(jobs)) for result0 in result)) else: multi_output = False - results = np.split(result, len(jobs)) + results = np.split(make_shareable(result), len(jobs)) #self.logger.debug("sharing result array for '%d'", jobid) with ExitStack() as stack: for jobid, result in zip(jobs, results): diff --git a/src/eynollah/training/models.py b/src/eynollah/training/models.py index 9111225..eda6b0f 100644 --- a/src/eynollah/training/models.py +++ b/src/eynollah/training/models.py @@ -82,18 +82,17 @@ class CTCDecoder(Layer): inputs = tf.math.log( tf.transpose(inputs, perm=[1, 0, 2]) + tf.keras.backend.epsilon() ) - # tf.nn.ctc_greedy_decoder() is not as precise # tf.compat.v1.nn.ctc_beam_search_decoder() also needs merge_repeated=False - decoded, logits = tf.nn.ctc_beam_search_decoder( + # tf.nn.ctc_beam_search_decoder() is not supported by ONNX, yet + # tf.nn.ctc_greedy_decoder() is not as precise, though: + decoded, logits = tf.nn.ctc_greedy_decoder( inputs, lengths, - beam_width=10, - top_paths=2, ) # get top path for all sequences in batch decoded = decoded[0] - logits = logits[:, 0] - logits[:, 1] - probs = tf.exp(-logits) + logits = logits[:, 0] + probs = tf.exp(-logits / n_steps) # convert to dense outputs = tf.SparseTensor(decoded.indices, decoded.values, (n_samples, n_steps)) @@ -555,8 +554,6 @@ def cnn_rnn_ocr_model(input_height=None, input_width=None, n_classes=None, max_l voc = char2num.get_vocabulary() num2char = StringLookup(vocabulary=voc, invert=True) output = num2char(out) - # avoid output tf.dtype=string → np.dtype=object (which cannot be shm-ed) - output = tf.io.decode_raw(output, tf.uint8, fixed_length=max(map(len, voc))) return Model((inputs, inputs_bin), (output, prob)) @@ -585,8 +582,6 @@ def cnn_rnn_ocr_model4inference(model, model_path): voc = char2num.get_vocabulary() num2char = StringLookup(vocabulary=voc, invert=True) output = num2char(output) - # avoid output tf.dtype=string → np.dtype=object (which cannot be shm-ed) - output = tf.io.decode_raw(output, tf.uint8, fixed_length=max(map(len, voc))) inputs = (inputs, inputs_bin) outputs = (output, prob) return Model(inputs, outputs) diff --git a/src/eynollah/training/reload-models-v0.8.mk b/src/eynollah/training/reload-models-v0.8.mk index ef74bdc..022603e 100644 --- a/src/eynollah/training/reload-models-v0.8.mk +++ b/src/eynollah/training/reload-models-v0.8.mk @@ -59,8 +59,6 @@ $(MODELS_DST)/%.h5: $(MODELS_SRC)/% $(MODELS_DST)/%.onnx: $(MODELS_SRC)/% if jq -e '.task == "segmentation" and .backbone_type == "transformer"' $/dev/null; then \ echo skipping $@: vision transformer architecture currently does not work with ONNX; \ - elif jq -e '.task == "cnn-rnn-ocr"' $/dev/null || test x$(findstring _ocr,$@) = x_ocr; then \ - echo skipping $@: OCR CTC decoder does not work with ONNX; \ else \ eynollah-training convert \ $(and $(wildcard $ Date: Fri, 26 Jun 2026 02:42:51 +0200 Subject: [PATCH 08/21] =?UTF-8?q?predictor=20for=20OCR=20models:=20work=20?= =?UTF-8?q?around=20ONNX=20bug=20=E2=80=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (ONNX converted models already return `np.dtype=object` arrays of `np.str_` instead of `np.bytes_`; so undo this) --- src/eynollah/predictor.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/eynollah/predictor.py b/src/eynollah/predictor.py index 6b7fe96..9121ffe 100644 --- a/src/eynollah/predictor.py +++ b/src/eynollah/predictor.py @@ -195,6 +195,10 @@ class Predictor(mp.context.SpawnProcess): # convert tf.string/np.object to fixed-length bytes # (because object segfaults in shm) if x.dtype is np.dtype(object): + # ONNX conversion for some reason decodes bytes into str already + # so here we undo this, too + if x[0].dtype is np.dtype(object) and isinstance(x[0, 0], str): + x = np.char.encode(x.astype(str), 'utf-8') return x.astype(bytes) return x if isinstance(result, (list, tuple)): From 5e531ab00619688d0020834d8e159a74c7866da3 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Wed, 1 Jul 2026 18:27:46 +0200 Subject: [PATCH 09/21] get_textlines_of_textregion_sorted: fix be61875 --- src/eynollah/eynollah.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 9db47ce..8cc17a1 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -901,7 +901,7 @@ class Eynollah: if N > 1: mean_y_diff = np.median(diff_cy) mean_x_diff = np.median(diff_cx) - count_hor = np.count_nonzero(np.diff(w_h_textline) > 0) + count_hor = np.count_nonzero(np.diff(w_h_textline, axis=0) > 0) count_ver = N - count_hor else: mean_y_diff = 0 From 1b27c7390f7311492bd746dfb83185f58dfb63db Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Thu, 2 Jul 2026 20:56:55 +0200 Subject: [PATCH 10/21] training.convert/ONNX: run strict shape inference and check model --- src/eynollah/training/convert.py | 5 +++++ src/eynollah/training/reload-models-v0.8.mk | 11 ++++------- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/eynollah/training/convert.py b/src/eynollah/training/convert.py index 5bf30a1..7351356 100644 --- a/src/eynollah/training/convert.py +++ b/src/eynollah/training/convert.py @@ -99,7 +99,12 @@ def convert_cli(rebuild, format_, in_, out): model.export(out) elif format_ == "onnx": import tf2onnx + import onnx tf2onnx.convert.from_keras(model, opset=18, output_path=out) + model = onnx.load(out) + model = onnx.shape_inference.infer_shapes(model, strict_mode=True) + onnx.checker.check_model(model, full_check=True) + onnx.save(model, out) else: raise ValueError("unknown output format '%s'" % format_) diff --git a/src/eynollah/training/reload-models-v0.8.mk b/src/eynollah/training/reload-models-v0.8.mk index 022603e..342d211 100644 --- a/src/eynollah/training/reload-models-v0.8.mk +++ b/src/eynollah/training/reload-models-v0.8.mk @@ -38,7 +38,7 @@ $(MODELS_DST)/%: $(MODELS_SRC)/% --in $< \ --format $(FORMAT) \ --out $@ \ - 2>&1 | tee $(notdir $<).$(FORMAT).log + > $(notdir $<).$(FORMAT).log 2>&1 || { cat $(notdir $<).$(FORMAT).log; false; } $(MODELS_DST)/%.keras: $(MODELS_SRC)/% eynollah-training convert \ @@ -46,7 +46,7 @@ $(MODELS_DST)/%.keras: $(MODELS_SRC)/% --in $< \ --format keras \ --out $@ \ - 2>&1 | tee $(notdir $<).keras.log + > $(notdir $<).keras.log 2>&1 || { cat $(notdir $<).keras.log; false; } $(MODELS_DST)/%.h5: $(MODELS_SRC)/% eynollah-training convert \ @@ -54,18 +54,15 @@ $(MODELS_DST)/%.h5: $(MODELS_SRC)/% --in $< \ --format hdf5 \ --out $@ \ - 2>&1 | tee $(notdir $<).hdf5.log + > $(notdir $<).hdf5.log 2>&1 || { cat $(notdir $<).hdf5.log; false; } $(MODELS_DST)/%.onnx: $(MODELS_SRC)/% - if jq -e '.task == "segmentation" and .backbone_type == "transformer"' $/dev/null; then \ - echo skipping $@: vision transformer architecture currently does not work with ONNX; \ - else \ eynollah-training convert \ $(and $(wildcard $&1 | tee $(notdir $<).onnx.log; fi + > $(notdir $<).onnx.log 2>&1 || { cat $(notdir $<).onnx.log; false; } compare: for i in `find $(MODELS_DST) -mindepth 2`;do \ From 16943f70b4c08db914b64a768a8820cee79e7370 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Thu, 2 Jul 2026 20:58:22 +0200 Subject: [PATCH 11/21] =?UTF-8?q?models=20(ViT=20backbone)=20iterate=20`ex?= =?UTF-8?q?tract=5Fpatches`=20over=20batch=20dim=E2=80=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `Patches.call`: use `tf.map_fn` instead of running entire batch through `tf.image.extract_patches` (faster, less VRAM, allows ONNX conversion to work) --- src/eynollah/model_zoo/model_zoo.py | 33 ++++++++++++++++++++++++++--- src/eynollah/patch_encoder.py | 13 +++++++++--- 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/src/eynollah/model_zoo/model_zoo.py b/src/eynollah/model_zoo/model_zoo.py index d6b5e38..3fca088 100644 --- a/src/eynollah/model_zoo/model_zoo.py +++ b/src/eynollah/model_zoo/model_zoo.py @@ -19,7 +19,7 @@ MODEL_VRAM_LIMITS = { "enhancement": 980, # due to bs 3 "col_classifier": 210, "page": 618, - "textline": 1680, # 954 for bs 1 + "textline": 1880, # 954 for bs 1 "region_1_2": 1580, "region_fl_np": 1756, "table": 1818, @@ -338,6 +338,14 @@ class EynollahModelZoo: # 'cudnn_conv_algo_search': 'EXHAUSTIVE', #'cudnn_conv_use_max_workspace': 0, # 'do_copy_in_default_stream': True, + # enable_cuda_graph + # cudnn_conv1d_pad_to_nc1d + # prefer_nhwc + # tunable_op_enable + # tunable_op_tuning_enable + # tunable_op_max_tuning_duration_ms + # use_ep_level_unified_stream + # enable_skip_layer_norm_strict_mode # ... })] + providers if 'TensorrtExecutionProvider' in providers: @@ -347,9 +355,28 @@ class EynollahModelZoo: 'device_id': gpu, 'trt_max_workspace_size': MODEL_VRAM_LIMITS[model_category] * 1024 * 1024, # 'trt_fp16_enable': True, - # 'trt_engine_cache_enable': True, - # 'trt_timing_cache_enable': True, + # trt_bf16_enable + 'trt_engine_cache_enable': True, + 'trt_timing_cache_enable': True, # ... + # trt_engine_hw_compatible + # trt_engine_cache_path + # trt_engine_cache_prefix + # trt_timing_cache_path + # trt_onnx_model_folder_path + # trt_ep_context_file_path + # trt_cuda_graph_enable + # trt_profile_opt_shapes + # trt_profile_min_shapes + # trt_profile_max_shapes + # trt_builder_optimization_level + # trt_build_heuristics_enable + # trt_sparsity_enable + # trt_weight_stripped_engine_enable + # trt_dla_core + # trt_dla_enable + # trt_min_subgraph_size + # trt_ep_context_embed_mode })] + providers provider0 = providers[0] if isinstance(provider0, tuple): diff --git a/src/eynollah/patch_encoder.py b/src/eynollah/patch_encoder.py index 610f0b4..556a4c9 100644 --- a/src/eynollah/patch_encoder.py +++ b/src/eynollah/patch_encoder.py @@ -29,7 +29,13 @@ class Patches(layers.Layer): self.patch_size_y = patch_size_y def call(self, images): - batch_size = tf.shape(images)[0] + #batch_size = tf.shape(images)[0] + return tf.map_fn(self.call_single, images) + + def call_single(self, image): + # avoid batched extract_patches: too much memory, + # and variable batch dim not supported by ONNX implementation + images = tf.expand_dims(image, axis=0) patches = tf.image.extract_patches( images=images, sizes=[1, self.patch_size_y, self.patch_size_x, 1], @@ -37,8 +43,9 @@ class Patches(layers.Layer): rates=[1, 1, 1, 1], padding="VALID", ) - patch_dims = patches.shape[-1] - return tf.reshape(patches, [batch_size, -1, patch_dims]) + _, n_rows, n_cols, patch_dims = patches.shape + n_tiles = patches.shape[1] * patches.shape[2] #-1 + return tf.reshape(patches, [1, n_tiles, patch_dims]) def get_config(self): return dict(patch_size_x=self.patch_size_x, From 1567df1379b457c1247568d5a7bfeb703ed82a8a Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Tue, 7 Jul 2026 22:09:46 +0200 Subject: [PATCH 12/21] remove numba dependency (previously used to free CUDA memory) --- requirements.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index d79853f..5bfb3b9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,7 @@ # ocrd includes opencv, numpy, shapely, click ocrd >= 3.3.0 -numpy < 2.0 scikit-learn >= 0.23.2 tensorflow tf-keras # avoid keras 3 (also needs TF_USE_LEGACY_KERAS=1) -numba <= 0.58.1 scikit-image tabulate From 171a8a3161c13ff4747a39286b175db3de3a1d62 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Tue, 7 Jul 2026 22:56:36 +0200 Subject: [PATCH 13/21] move TF+Keras dependencies to `training` extra, replace by ONNX+TRT --- requirements.txt | 4 ++-- train/requirements.txt | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 5bfb3b9..a707407 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ # ocrd includes opencv, numpy, shapely, click ocrd >= 3.3.0 +onnxruntime-gpu[cuda,cudnn] # w/ .onnx models +tensorrt_cu12 < 11 # 11 incompatible with CUDA libs from onnxruntime-gpu[cuda,cudnn] scikit-learn >= 0.23.2 -tensorflow -tf-keras # avoid keras 3 (also needs TF_USE_LEGACY_KERAS=1) scikit-image tabulate diff --git a/train/requirements.txt b/train/requirements.txt index 090bc50..1734b67 100644 --- a/train/requirements.txt +++ b/train/requirements.txt @@ -6,5 +6,6 @@ imutils scipy tensorflow-addons # for connected_components, depublished and only compatible with tensorflow < 2.16 tensorflow < 2.16 # for tensorflow-addons, so only needed in training -tf_data < 2.16 # for tensorflow-addons, so only needed in training +tf-keras # avoid keras 3 (also needs TF_USE_LEGACY_KERAS=1) +tf-data < 2.16 # for tensorflow-addons, so only needed in training protobuf < 5 # for tensorflow-addons, so only needed in training From 32568a590f928a41c0a3c749d25945918d8a83a2 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Wed, 8 Jul 2026 02:54:34 +0200 Subject: [PATCH 14/21] Docker: update to ONNX base image --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index a15776e..f267b85 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,7 @@ LABEL \ org.opencontainers.image.documentation="https://github.com/qurator-spk/eynollah/blob/${VCS_REF}/README.md" \ org.opencontainers.image.revision=$VCS_REF \ org.opencontainers.image.created=$BUILD_DATE \ - org.opencontainers.image.base.name=ocrd/core-cuda-tf2 + org.opencontainers.image.base.name=ocrd/core-cuda-onnx ENV DEBIAN_FRONTEND=noninteractive # set proper locales @@ -40,8 +40,8 @@ RUN ocrd ocrd-tool ocrd-tool.json dump-tools > $(dirname $(ocrd bashlib filename RUN ocrd ocrd-tool ocrd-tool.json dump-module-dirs > $(dirname $(ocrd bashlib filename))/ocrd-all-module-dir.json # install everything and reduce image size RUN make install EXTRAS=OCR && rm -rf /build/eynollah -# fixup for broken cuDNN installation (Torch pulls in 8.5.0, which is incompatible with Tensorflow) -RUN pip install nvidia-cudnn-cu11==8.6.0.163 +# fixup for broken cuDNN installation (Torch may pull in version which is incompatible with Tensorflow) +# but most recent Torch versions pull cu13 variants, which are not conflicting here # smoke test RUN eynollah --help From 8cc8c2847167c33585815f1e3fb9d1ca7c12710d Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Wed, 8 Jul 2026 02:55:25 +0200 Subject: [PATCH 15/21] ModelZoo for ONNX backend: configure TRT cache path from XDG env --- src/eynollah/model_zoo/model_zoo.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/eynollah/model_zoo/model_zoo.py b/src/eynollah/model_zoo/model_zoo.py index 3fca088..a0e6052 100644 --- a/src/eynollah/model_zoo/model_zoo.py +++ b/src/eynollah/model_zoo/model_zoo.py @@ -306,6 +306,7 @@ class EynollahModelZoo: def _load_onnx_model(self, model_category, model_path, device=''): import onnxruntime as ort import numpy as np + from ocrd_utils import config ort.set_default_logger_severity(3) @@ -358,11 +359,11 @@ class EynollahModelZoo: # trt_bf16_enable 'trt_engine_cache_enable': True, 'trt_timing_cache_enable': True, + 'trt_engine_cache_path': config.XDG_CONFIG_HOME, + 'trt_timing_cache_path': config.XDG_CONFIG_HOME, # ... # trt_engine_hw_compatible - # trt_engine_cache_path # trt_engine_cache_prefix - # trt_timing_cache_path # trt_onnx_model_folder_path # trt_ep_context_file_path # trt_cuda_graph_enable From 5354583913a87ffc157ea82bb6f70b17c72af2a0 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Wed, 8 Jul 2026 02:56:11 +0200 Subject: [PATCH 16/21] update readme --- README.md | 99 ++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 68 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index 7e9a90a..bf80bca 100644 --- a/README.md +++ b/README.md @@ -25,9 +25,12 @@ documents using a combination of multiple deep learning models and heuristics; therefore processing can be slow. ## Installation -Python `3.8-3.11` with Tensorflow `<2.13` on Linux are currently supported. -For (limited) GPU support the CUDA toolkit needs to be installed. -A working config is CUDA `11.8` with cuDNN `8.6`. + +Python `3.8-3.11` with ONNX Runtime on Linux are currently supported. + +For GPU support, NVidia drivers supporting CUDA 12 must be installed. +The runtime dependencies will pull in ONNX, TensorRT and CUDA runtime +libraries (including cuDNN) from PyPI. You can either install from PyPI @@ -52,6 +55,13 @@ pip install "eynollah[OCR]" make install EXTRAS=OCR ``` +> **Note**: Requirements for OCR are more involved, +> as they may need Tensorflow (with tf-keras) and/or +> Torch (with transformers). Those two frameworks may +> also have conflicting CUDA dependencies. An ONNX +> conversion for these models may be achieved soon. +> :construction: + ### Docker Use @@ -64,7 +74,12 @@ When using Eynollah with Docker, see [`docker.md`](https://github.com/qurator-sp ## Models -Pretrained models can be downloaded from [Zenodo](https://zenodo.org/records/17727267) or [Hugging Face](https://huggingface.co/SBB?search_models=eynollah). +Pretrained models can be downloaded from [Zenodo](https://zenodo.org/records/17727267) or [Hugging Face](https://huggingface.co/SBB?search_models=eynollah). + +For fast runtime inference, download the ONNX models. + +For finetuning training, download the original (Tensorflow / Torch) models +(and install the `[training]` extra). For model documentation and model cards, see [`models.md`](https://github.com/qurator-spk/eynollah/tree/main/docs/models.md). @@ -83,18 +98,26 @@ Eynollah supports five use cases: Some example outputs can be found in [`examples.md`](https://github.com/qurator-spk/eynollah/tree/main/docs/examples.md). +The **generic options** shared by all subcommands are: +```sh + -m + -mv + -D + -l +``` + ### Layout Analysis -The layout analysis module is responsible for detecting layout elements, identifying text lines, and determining reading -order using heuristic methods or a [pretrained model](https://github.com/qurator-spk/eynollah#machine-based-reading-order). +Detects layout elements, i.e. regions of various types and text lines, +and determines their reading order using either heuristic methods or a +[pretrained model](https://github.com/qurator-spk/eynollah#machine-based-reading-order). The command-line interface for layout analysis can be called like this: ```sh -eynollah layout \ +eynollah [GENERIC_OPTIONS] layout \ -i | -di \ -o \ - -m \ [OPTIONS] ``` @@ -106,7 +129,7 @@ The following options can be used to further configure the processing: | `-tab` | apply table detection | | `-ae` | apply enhancement (the resulting image is saved to the output directory) | | `-as` | apply scaling | -| `-cl` | apply contour detection for curved text lines instead of bounding boxes | +| `-cl` | apply contour detection for curved text lines, deskewing all regions independently | | `-ib` | apply binarization (the resulting image is saved to the output directory) | | `-ep` | enable plotting (MUST always be used with `-sl`, `-sd`, `-sa`, `-si` or `-ae`) | | `-ho` | ignore headers for reading order dectection | @@ -115,79 +138,93 @@ The following options can be used to further configure the processing: | `-sl ` | save layout prediction as plot to this directory | | `-sp ` | save cropped page image to this directory | | `-sa ` | save all (plot, enhanced/binary image, layout) to this directory | -| `-thart` | threshold of artifical class in the case of textline detection. The default value is 0.1 | -| `-tharl` | threshold of artifical class in the case of layout detection. The default value is 0.1 | +| `-thart` | confidence threshold of artifical boundary class during textline detection | +| `-tharl` | confidence threshold of artifical boundary class during region detection | | `-ncu` | upper limit of columns in document image | | `-ncl` | lower limit of columns in document image | | `-slro` | skip layout detection and reading order | | `-romb` | apply machine based reading order detection | | `-ipe` | ignore page extraction | +| `-j` | number of CPU jobs to run parallel (useful with -di) | +| `-H` | when to halt when some jobs fail | +The default is to only perform layout detection of main regions +(background, text, images, separators and marginals). -If no further option is set, the tool performs layout detection of main regions (background, text, images, separators -and marginals). -The best output quality is achieved when RGB images are used as input rather than greyscale or binarized images. +The best output quality is achieved when RGB images are used as input +rather than greyscale or binarized images. -Additional documentation can be found in [`usage.md`](https://github.com/qurator-spk/eynollah/tree/main/docs/usage.md). +Additional documentation can be found in +[`usage.md`](https://github.com/qurator-spk/eynollah/tree/main/docs/usage.md). ### Binarization -The binarization module performs document image binarization using pretrained pixelwise segmentation models. +Performs document image binarization (thresholding) +using pretrained pixelwise segmentation models. The command-line interface for binarization can be called like this: ```sh -eynollah binarization \ +eynollah [GENERIC_OPTIONS] binarization \ -i | -di \ -o \ - -m + [OPTIONS] ``` ### Image Enhancement -TODO + +This enlarges and enhances images. Useful in case the scan quality is low. + +```sh +eynollah [GENERIC_OPTIONS] enhancement \ + -i | -di \ + -o \ + [OPTIONS] +``` + +| option | description | +|-------------------|:--------------------------------------------------------------------------------------------| +| `-sos` | save the enhanced image in original image size | +| `-ncu` | upper limit of columns in document image | +| `-ncl` | lower limit of columns in document image | ### OCR -The OCR module performs text recognition using either a CNN-RNN model or a Transformer model. +Performs text recognition using either a CNN-RNN model or a Transformer model. +Needs a PAGE-XML input file. The command-line interface for OCR can be called like this: ```sh -eynollah ocr \ +eynollah [GENERIC_OPTIONS] ocr \ -i | -di \ -dx \ -o \ - -m | --model_name ``` The following options can be used to further configure the ocr processing: | option | description | |-------------------|:-------------------------------------------------------------------------------------------| +| `-trocr` | use transformer OCR model instead of CNN-RNN model | | `-dib` | directory of binarized images (file type must be '.png'), prediction with both RGB and bin | | `-doit` | directory for output images rendered with the predicted text | -| `--model_name` | file path to use specific model for OCR | -| `-trocr` | use transformer ocr model (otherwise cnn_rnn model is used) | -| `-etit` | export textline images and text in xml to output dir (OCR training data) | | `-nmtc` | cropped textline images will not be masked with textline contour | | `-bs` | ocr inference batch size. Default batch size is 2 for trocr and 8 for cnn_rnn models | -| `-ds_pref` | add an abbrevation of dataset name to generated training data | | `-min_conf` | minimum OCR confidence value. OCR with textline conf lower than this will be ignored | ### Reading Order Detection -Reading order detection can be performed either as part of layout analysis based on image input, or, currently under -development, based on pre-existing layout analysis data in PAGE-XML format as input. -The reading order detection module employs a pretrained model to identify the reading order from layouts represented in PAGE-XML files. +Reading order can be detected either during layout analysis, +or as a separate module, which requires a PAGE-XML input file. The command-line interface for machine based reading order can be called like this: ```sh -eynollah machine-based-reading-order \ +eynollah [GENERIC_OPTIONS] machine-based-reading-order \ -i | -di \ -xml | -dx \ - -m \ -o ``` From 98e28ca9f43d73fc8b1423ea5095ecd34c2accd8 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Wed, 8 Jul 2026 03:05:07 +0200 Subject: [PATCH 17/21] Docker: fixup cuDNN installation after OCR extra --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index f267b85..d0b41a5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -41,7 +41,7 @@ RUN ocrd ocrd-tool ocrd-tool.json dump-module-dirs > $(dirname $(ocrd bashlib fi # install everything and reduce image size RUN make install EXTRAS=OCR && rm -rf /build/eynollah # fixup for broken cuDNN installation (Torch may pull in version which is incompatible with Tensorflow) -# but most recent Torch versions pull cu13 variants, which are not conflicting here +RUN pip install "nvidia-cudnn-cu12<9.10.2.21" # smoke test RUN eynollah --help From c9c14ed83d48b5d75e4c014b553e68a81b691349 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Wed, 8 Jul 2026 14:04:19 +0200 Subject: [PATCH 18/21] Docker: update to ONNX base image (in Makefile, too) --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index f54cf5b..e4ac5c6 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ PYTHON ?= python3 PIP ?= pip3 EXTRAS ?= -DOCKER_BASE_IMAGE ?= docker.io/ocrd/core-cuda-tf2:v3.13.0 +DOCKER_BASE_IMAGE ?= docker.io/ocrd/core-cuda-onnx:v3.13.1 DOCKER_TAG ?= ocrd/eynollah DOCKER ?= docker WGET = wget -O From b9ba43b44499bd19609e78f7026c159c99daf13e Mon Sep 17 00:00:00 2001 From: kba Date: Thu, 9 Jul 2026 14:37:51 +0200 Subject: [PATCH 19/21] training: require tf2onnx and pin ml_dtypes >= 0.5 --- .gitignore | 2 ++ train/requirements.txt | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 49835a7..e3356ea 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,5 @@ output.html *.sw? TAGS uv.lock +/ignore +*.log diff --git a/train/requirements.txt b/train/requirements.txt index 1734b67..dbc4f76 100644 --- a/train/requirements.txt +++ b/train/requirements.txt @@ -6,6 +6,8 @@ imutils scipy tensorflow-addons # for connected_components, depublished and only compatible with tensorflow < 2.16 tensorflow < 2.16 # for tensorflow-addons, so only needed in training -tf-keras # avoid keras 3 (also needs TF_USE_LEGACY_KERAS=1) +tf-keras < 2.16 # avoid keras 3 (also needs TF_USE_LEGACY_KERAS=1) tf-data < 2.16 # for tensorflow-addons, so only needed in training protobuf < 5 # for tensorflow-addons, so only needed in training +tf2onnx +ml_dtypes >= 0.5 From 492fcbacb758fd954af7cdfad821fb977d10483b Mon Sep 17 00:00:00 2001 From: kba Date: Thu, 9 Jul 2026 15:17:11 +0200 Subject: [PATCH 20/21] switch to fork of tf2onnx --- train/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/train/requirements.txt b/train/requirements.txt index dbc4f76..02cba9f 100644 --- a/train/requirements.txt +++ b/train/requirements.txt @@ -9,5 +9,5 @@ tensorflow < 2.16 # for tensorflow-addons, so only needed in training tf-keras < 2.16 # avoid keras 3 (also needs TF_USE_LEGACY_KERAS=1) tf-data < 2.16 # for tensorflow-addons, so only needed in training protobuf < 5 # for tensorflow-addons, so only needed in training -tf2onnx +eynollah-fork-tf2onnx == 1.17.0.post1 ml_dtypes >= 0.5 From b1f2f430519567b60e9837ce79597d84eca451bc Mon Sep 17 00:00:00 2001 From: kba Date: Thu, 9 Jul 2026 17:30:48 +0200 Subject: [PATCH 21/21] upgrade tf2onnx fork dep, remove spurious tf-data dep --- train/requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/train/requirements.txt b/train/requirements.txt index 02cba9f..392581c 100644 --- a/train/requirements.txt +++ b/train/requirements.txt @@ -7,7 +7,6 @@ scipy tensorflow-addons # for connected_components, depublished and only compatible with tensorflow < 2.16 tensorflow < 2.16 # for tensorflow-addons, so only needed in training tf-keras < 2.16 # avoid keras 3 (also needs TF_USE_LEGACY_KERAS=1) -tf-data < 2.16 # for tensorflow-addons, so only needed in training protobuf < 5 # for tensorflow-addons, so only needed in training -eynollah-fork-tf2onnx == 1.17.0.post1 +eynollah-fork-tf2onnx == 1.17.0.post2 ml_dtypes >= 0.5