Merge branch 'fix-0.8-modelzoo-and-predictor-kba0709' into integrating_trocr_and_torch_ensembling_and_updating_characters_list-refactor

# Conflicts:
#	train/requirements.txt

.gitignore

@@ -12,3 +12,5 @@ output.html
 *.sw?
 TAGS
 uv.lock
+/ignore
+*.log

Dockerfile

@@ -15,7 +15,7 @@ LABEL \
     org.opencontainers.image.documentation="https://github.com/qurator-spk/eynollah/blob/${VCS_REF}/README.md" \
     org.opencontainers.image.revision=$VCS_REF \
     org.opencontainers.image.created=$BUILD_DATE \
-    org.opencontainers.image.base.name=ocrd/core-cuda-tf2
+    org.opencontainers.image.base.name=ocrd/core-cuda-onnx
 
 ENV DEBIAN_FRONTEND=noninteractive
 # set proper locales
@@ -40,8 +40,8 @@ RUN ocrd ocrd-tool ocrd-tool.json dump-tools > $(dirname $(ocrd bashlib filename
 RUN ocrd ocrd-tool ocrd-tool.json dump-module-dirs > $(dirname $(ocrd bashlib filename))/ocrd-all-module-dir.json
 # install everything and reduce image size
 RUN make install EXTRAS=OCR && rm -rf /build/eynollah
-# fixup for broken cuDNN installation (Torch pulls in 8.5.0, which is incompatible with Tensorflow)
+# fixup for broken cuDNN installation (Torch may pull in version which is incompatible with Tensorflow)
-RUN pip install nvidia-cudnn-cu11==8.6.0.163
+RUN pip install "nvidia-cudnn-cu12<9.10.2.21"
 # smoke test
 RUN eynollah --help
 

Makefile

@@ -2,7 +2,7 @@ PYTHON ?= python3
 PIP ?= pip3
 EXTRAS ?=
 
-DOCKER_BASE_IMAGE ?= docker.io/ocrd/core-cuda-tf2:v3.13.0
+DOCKER_BASE_IMAGE ?= docker.io/ocrd/core-cuda-onnx:v3.13.1
 DOCKER_TAG ?= ocrd/eynollah
 DOCKER ?= docker
 WGET = wget -O

README.md

@@ -25,9 +25,12 @@
 documents using a combination of multiple deep learning models and heuristics; therefore processing can be slow.
 
 ## Installation
-Python `3.8-3.11` with Tensorflow `<2.13` on Linux are currently supported.
+
-For (limited) GPU support the CUDA toolkit needs to be installed. 
+Python `3.8-3.11` with ONNX Runtime on Linux are currently supported.
-A working config is CUDA `11.8` with cuDNN `8.6`.
+
+For GPU support, NVidia drivers supporting CUDA 12 must be installed.
+The runtime dependencies will pull in ONNX, TensorRT and CUDA runtime
+libraries (including cuDNN) from PyPI.
 
 You can either install from PyPI
 
@@ -52,6 +55,13 @@ pip install "eynollah[OCR]"
 make install EXTRAS=OCR
 ```
 
+> **Note**: Requirements for OCR are more involved, 
+> as they may need Tensorflow (with tf-keras) and/or
+> Torch (with transformers). Those two frameworks may
+> also have conflicting CUDA dependencies. An ONNX
+> conversion for these models may be achieved soon.
+> :construction:
+
 ### Docker
 
 Use
@@ -66,6 +76,11 @@ When using Eynollah with Docker, see [`docker.md`](https://github.com/qurator-sp
 
 Pretrained models can be downloaded from [Zenodo](https://zenodo.org/records/17727267) or [Hugging Face](https://huggingface.co/SBB?search_models=eynollah).
 
+For fast runtime inference, download the ONNX models. 
+
+For finetuning training, download the original (Tensorflow / Torch) models
+(and install the `[training]` extra).
+
 For model documentation and model cards, see [`models.md`](https://github.com/qurator-spk/eynollah/tree/main/docs/models.md).
 
 ## Training
@@ -83,18 +98,26 @@ Eynollah supports five use cases:
 
 Some example outputs can be found in [`examples.md`](https://github.com/qurator-spk/eynollah/tree/main/docs/examples.md).
 
+The **generic options** shared by all subcommands are:
+```sh
+  -m <directory containing model files>
+  -mv <model category> <model variant> <model path>
+  -D <device specifier>
+  -l <log level>
+```
+
 ### Layout Analysis
 
-The layout analysis module is responsible for detecting layout elements, identifying text lines, and determining reading 
+Detects layout elements, i.e. regions of various types and text lines,
-order using heuristic methods or a [pretrained model](https://github.com/qurator-spk/eynollah#machine-based-reading-order).
+and determines their reading order using either heuristic methods or a
+[pretrained model](https://github.com/qurator-spk/eynollah#machine-based-reading-order).
 
 The command-line interface for layout analysis can be called like this:
 
 ```sh
-eynollah layout \
+eynollah [GENERIC_OPTIONS] layout \
   -i <single image file> | -di <directory containing image files> \
   -o <output directory> \
-  -m <directory containing model files> \
      [OPTIONS]
 ```
 
@@ -106,7 +129,7 @@ The following options can be used to further configure the processing:
 | `-tab`            | apply table detection                                                                       |
 | `-ae`             | apply enhancement (the resulting image is saved to the output directory)                    |
 | `-as`             | apply scaling                                                                               |
-| `-cl`             | apply contour detection for curved text lines instead of bounding boxes                     |
+| `-cl`             | apply contour detection for curved text lines, deskewing all regions independently          |
 | `-ib`             | apply binarization (the resulting image is saved to the output directory)                   |
 | `-ep`             | enable plotting (MUST always be used with `-sl`, `-sd`, `-sa`, `-si` or `-ae`)              |
 | `-ho`             | ignore headers for reading order dectection                                                 |
@@ -115,79 +138,93 @@ The following options can be used to further configure the processing:
 | `-sl <directory>` | save layout prediction as plot to this directory                                            |
 | `-sp <directory>` | save cropped page image to this directory                                                   |
 | `-sa <directory>` | save all (plot, enhanced/binary image, layout) to this directory                            |
-| `-thart`          | threshold of artifical class in the case of textline detection. The default value is 0.1    |
+| `-thart`          | confidence threshold of artifical boundary class during textline detection                  |
-| `-tharl`          | threshold of artifical class in the case of layout detection. The default value is 0.1      |
+| `-tharl`          | confidence threshold of artifical boundary class during region detection                    |
 | `-ncu`            | upper limit of columns in document image                                                    |
 | `-ncl`            | lower limit of columns in document image                                                    |
 | `-slro`           | skip layout detection and reading order                                                     |
 | `-romb`           | apply machine based reading order detection                                                 |
 | `-ipe`            | ignore page extraction                                                                      |
+| `-j`              | number of CPU jobs to run parallel (useful with -di)                                        |
+| `-H`              | when to halt when some jobs fail                                                            |
 
+The default is to only perform layout detection of main regions
+(background, text, images, separators and marginals).
 
-If no further option is set, the tool performs layout detection of main regions (background, text, images, separators 
+The best output quality is achieved when RGB images are used as input
-and marginals).
+rather than greyscale or binarized images.
-The best output quality is achieved when RGB images are used as input rather than greyscale or binarized images.
 
-Additional documentation can be found in [`usage.md`](https://github.com/qurator-spk/eynollah/tree/main/docs/usage.md).
+Additional documentation can be found in
+[`usage.md`](https://github.com/qurator-spk/eynollah/tree/main/docs/usage.md).
 
 ### Binarization
 
-The binarization module performs document image binarization using pretrained pixelwise segmentation models. 
+Performs document image binarization (thresholding)
+using pretrained pixelwise segmentation models. 
 
 The command-line interface for binarization can be called like this:
 
 ```sh
-eynollah binarization \
+eynollah [GENERIC_OPTIONS] binarization \
   -i <single image file> | -di <directory containing image files> \
   -o <output directory> \
-  -m <directory containing model files> 
+  [OPTIONS]
 ```
 
 ### Image Enhancement
-TODO
+
+This enlarges and enhances images. Useful in case the scan quality is low.
+
+```sh
+eynollah [GENERIC_OPTIONS] enhancement \
+  -i <single image file> | -di <directory containing image files> \
+  -o <output directory> \
+  [OPTIONS]
+```
+
+| option            | description                                                                                 |
+|-------------------|:--------------------------------------------------------------------------------------------|
+| `-sos`            | save the enhanced image in original image size                                              |
+| `-ncu`            | upper limit of columns in document image                                                    |
+| `-ncl`            | lower limit of columns in document image                                                    |
 
 ### OCR
 
-The OCR module performs text recognition using either a CNN-RNN model or a Transformer model.
+Performs text recognition using either a CNN-RNN model or a Transformer model.
+Needs a PAGE-XML input file.
 
 The command-line interface for OCR can be called like this:
 
 ```sh
-eynollah ocr \
+eynollah [GENERIC_OPTIONS] ocr \
   -i <single image file> | -di <directory containing image files> \
   -dx <directory of xmls> \
   -o <output directory> \
-  -m <directory containing model files> | --model_name <path to specific model>
 ```
 
 The following options can be used to further configure the ocr processing:
 
 | option            | description                                                                                |
 |-------------------|:-------------------------------------------------------------------------------------------|
+| `-trocr`          | use transformer OCR model instead of CNN-RNN model                                         |
 | `-dib`            | directory of binarized images (file type must be '.png'), prediction with both RGB and bin |
 | `-doit`           | directory for output images rendered with the predicted text                               |
-| `--model_name`    | file path to use specific model for OCR                                                    |
-| `-trocr`          | use transformer ocr model (otherwise cnn_rnn model is used)                                |
-| `-etit`           | export textline images and text in xml to output dir (OCR training data)                   |
 | `-nmtc`           | cropped textline images will not be masked with textline contour                           |
 | `-bs`             | ocr inference batch size. Default batch size is 2 for trocr and 8 for cnn_rnn models       |
-| `-ds_pref`        | add an abbrevation of dataset name to generated training data                              |
 | `-min_conf`       | minimum OCR confidence value. OCR with textline conf lower than this will be ignored       |
 
 
 ### Reading Order Detection
-Reading order detection can be performed either as part of layout analysis based on image input, or, currently under 
-development, based on pre-existing layout analysis data in PAGE-XML format as input.
 
-The reading order detection module employs a pretrained model to identify the reading order from layouts represented in PAGE-XML files.
+Reading order can be detected either during layout analysis,
+or as a separate module, which requires a PAGE-XML input file.
 
 The command-line interface for machine based reading order can be called like this:
 
 ```sh
-eynollah machine-based-reading-order \
+eynollah [GENERIC_OPTIONS] machine-based-reading-order \
   -i <single image file> | -di <directory containing image files> \
   -xml <xml file name> | -dx <directory containing xml files> \
-  -m <path to directory containing model files> \
   -o <output directory> 
 ```
 

requirements.txt

@@ -1,9 +1,7 @@
 # ocrd includes opencv, numpy, shapely, click
 ocrd >= 3.3.0
-numpy < 2.0
+onnxruntime-gpu[cuda,cudnn] # w/ .onnx models
+tensorrt_cu12 < 11 # 11 incompatible with CUDA libs from onnxruntime-gpu[cuda,cudnn]
 scikit-learn >= 0.23.2
-tensorflow
-tf-keras # avoid keras 3 (also needs TF_USE_LEGACY_KERAS=1)
-numba <= 0.58.1
 scikit-image
 tabulate

src/eynollah/eynollah.py

@@ -901,7 +901,7 @@ class Eynollah:
         if N > 1:
             mean_y_diff = np.median(diff_cy)
             mean_x_diff = np.median(diff_cx)
-            count_hor = np.count_nonzero(np.diff(w_h_textline) > 0)
+            count_hor = np.count_nonzero(np.diff(w_h_textline, axis=0) > 0)
             count_ver = N - count_hor
         else:
             mean_y_diff = 0

src/eynollah/eynollah_ocr.py

@@ -296,13 +296,11 @@ class Eynollah_ocr(Eynollah):
                     probs[ver_index > 0][flipped_ver_is_better] = probs_ver[flipped_ver_is_better]
 
             def nooov(x):
-                return x != b'[UNK]'
+                if x == b'[UNK]':
+                    return b''
+                return x
             for pred, prob in zip(preds, probs):
-                text = b''.join(
+                text = b''.join(map(nooov, pred.tolist())).decode('utf-8')
-                    filter(nooov,
-                           map(bytes,
-                               (filter(None, char)
-                                for char in pred.tolist())))).decode('utf-8')
                 extracted_texts.append(text)
                 extracted_confs.append(prob)
         del cropped_lines_rgb

src/eynollah/model_zoo/model_zoo.py

@@ -19,7 +19,7 @@ MODEL_VRAM_LIMITS = {
     "enhancement": 980, # due to bs 3
     "col_classifier": 210,
     "page": 618,
-    "textline": 1680, # 954 for bs 1
+    "textline": 1880, # 954 for bs 1
     "region_1_2": 1580,
     "region_fl_np": 1756,
     "table": 1818,
@@ -306,6 +306,9 @@ class EynollahModelZoo:
     def _load_onnx_model(self, model_category, model_path, device=''):
         import onnxruntime as ort
         import numpy as np
+        from ocrd_utils import config
+
+        ort.set_default_logger_severity(3)
 
         providers = ort.get_available_providers()
         if device:
@@ -336,6 +339,14 @@ class EynollahModelZoo:
                     # 'cudnn_conv_algo_search': 'EXHAUSTIVE',
                     #'cudnn_conv_use_max_workspace': 0,
                     # 'do_copy_in_default_stream': True,
+                    # enable_cuda_graph
+                    # cudnn_conv1d_pad_to_nc1d
+                    # prefer_nhwc
+                    # tunable_op_enable
+                    # tunable_op_tuning_enable
+                    # tunable_op_max_tuning_duration_ms
+                    # use_ep_level_unified_stream
+                    # enable_skip_layer_norm_strict_mode
                     # ...
                 })] + providers
         if 'TensorrtExecutionProvider' in providers:
@@ -345,14 +356,38 @@ class EynollahModelZoo:
                     'device_id': gpu,
                     'trt_max_workspace_size': MODEL_VRAM_LIMITS[model_category] * 1024 * 1024,
                     # 'trt_fp16_enable': True,
-                    # 'trt_engine_cache_enable': True,
+                    # trt_bf16_enable
-                    # 'trt_timing_cache_enable': True,
+                    'trt_engine_cache_enable': True,
+                    'trt_timing_cache_enable': True,
+                    'trt_engine_cache_path': config.XDG_CONFIG_HOME,
+                    'trt_timing_cache_path': config.XDG_CONFIG_HOME,
                     # ...
+                    # trt_engine_hw_compatible
+                    # trt_engine_cache_prefix
+                    # trt_onnx_model_folder_path
+                    # trt_ep_context_file_path
+                    # trt_cuda_graph_enable
+                    # trt_profile_opt_shapes
+                    # trt_profile_min_shapes
+                    # trt_profile_max_shapes
+                    # trt_builder_optimization_level
+                    # trt_build_heuristics_enable
+                    # trt_sparsity_enable
+                    # trt_weight_stripped_engine_enable
+                    # trt_dla_core
+                    # trt_dla_enable
+                    # trt_min_subgraph_size
+                    # trt_ep_context_embed_mode
                 })] + providers
+        provider0 = providers[0]
+        if isinstance(provider0, tuple):
+            provider0 = provider0[0]
+        self.logger.info("using %s with ONNX provider %s for model %s",
+                         "GPU %d" % gpu if gpu >= 0 else "CPU",
+                         provider0[:-17], model_category)
         model = ort.InferenceSession(
             model_path,
             providers=providers)
-        # FIXME: notify about selected provider/device
         model_inputs = [model_input.name
                         for model_input in model.get_inputs()]
         model_outputs = [model_output.name

src/eynollah/patch_encoder.py

@@ -29,7 +29,13 @@ class Patches(layers.Layer):
         self.patch_size_y = patch_size_y
 
     def call(self, images):
-        batch_size = tf.shape(images)[0]
+        #batch_size = tf.shape(images)[0]
+        return tf.map_fn(self.call_single, images)
+
+    def call_single(self, image):
+        # avoid batched extract_patches: too much memory,
+        # and variable batch dim not supported by ONNX implementation
+        images = tf.expand_dims(image, axis=0)
         patches = tf.image.extract_patches(
             images=images,
             sizes=[1, self.patch_size_y, self.patch_size_x, 1],
@@ -37,8 +43,9 @@ class Patches(layers.Layer):
             rates=[1, 1, 1, 1],
             padding="VALID",
         )
-        patch_dims = patches.shape[-1]
+        _, n_rows, n_cols, patch_dims = patches.shape
-        return tf.reshape(patches, [batch_size, -1, patch_dims])
+        n_tiles = patches.shape[1] * patches.shape[2] #-1
+        return tf.reshape(patches, [1, n_tiles, patch_dims])
 
     def get_config(self):
         return dict(patch_size_x=self.patch_size_x,

src/eynollah/predictor.py

@@ -191,13 +191,23 @@ class Predictor(mp.context.SpawnProcess):
                         #result = self.model.predict(data, verbose=0)
                         # faster, less VRAM
                         result = self.model.predict_on_batch(data)
-                    if isinstance(result, tuple):
+                    def make_shareable(x):
+                        # convert tf.string/np.object to fixed-length bytes
+                        # (because object segfaults in shm)
+                        if x.dtype is np.dtype(object):
+                            # ONNX conversion for some reason decodes bytes into str already
+                            # so here we undo this, too
+                            if x[0].dtype is np.dtype(object) and isinstance(x[0, 0], str):
+                                x = np.char.encode(x.astype(str), 'utf-8')
+                            return x.astype(bytes)
+                        return x
+                    if isinstance(result, (list, tuple)):
                         multi_output = True
-                        results = zip(*(np.split(result0, len(jobs))
+                        results = zip(*(np.split(make_shareable(result0), len(jobs))
                                         for result0 in result))
                     else:
                         multi_output = False
-                        results = np.split(result, len(jobs))
+                        results = np.split(make_shareable(result), len(jobs))
                     #self.logger.debug("sharing result array for '%d'", jobid)
                     with ExitStack() as stack:
                         for jobid, result in zip(jobs, results):
@@ -214,7 +224,7 @@ class Predictor(mp.context.SpawnProcess):
                             self.resultq.put((jobid, result))
                             #self.logger.debug("sent result for '%d': %s", jobid, result)
             except Exception as e:
-                self.logger.error("prediction for %s failed: %s", self.name, e.__class__.__name__)
+                self.logger.exception("prediction for %s failed: %s", self.name, e.__class__.__name__)
                 result = e
                 self.resultq.put((jobid, result))
         close_all()

src/eynollah/training/convert.py

@@ -68,12 +68,13 @@ def convert_cli(rebuild, format_, in_, out):
         ex.add_config(str(config_path))
         # some models deviate between training and inference
         ex.add_config(inference=True)
-        # make sure the local vocab file gets re-used
+        # OCR models: make sure the local vocab file gets re-used, if available
         characters_txt_file = model_path / "characters_org.txt"
-        with open(characters_txt_file, "r") as voc_file:
+        if characters_txt_file.exists():
-            voc = json.load(voc_file)
+            with open(characters_txt_file, "r") as voc_file:
-        ex.add_config(characters_txt_file=characters_txt_file)
+                voc = json.load(voc_file)
-        ex.add_config(n_classes=len(voc) + 3)
+            ex.add_config(characters_txt_file=characters_txt_file)
+            ex.add_config(n_classes=len(voc) + 3)
         # just retrieve final config (via pseudo-run)
         ex.main(lambda: 0)
         config = ex.run(options={'--loglevel': 'ERROR'}).config
@@ -98,7 +99,12 @@ def convert_cli(rebuild, format_, in_, out):
         model.export(out)
     elif format_ == "onnx":
         import tf2onnx
+        import onnx
         tf2onnx.convert.from_keras(model, opset=18, output_path=out)
+        model = onnx.load(out)
+        model = onnx.shape_inference.infer_shapes(model, strict_mode=True)
+        onnx.checker.check_model(model, full_check=True)
+        onnx.save(model, out)
     else:
         raise ValueError("unknown output format '%s'" % format_)
 

src/eynollah/training/metrics.py

@@ -5,7 +5,10 @@ import tensorflow as tf
 from tensorflow.keras import backend as K
 from tensorflow.keras.metrics import Metric, MeanMetricWrapper, get
 from tensorflow.keras.initializers import Zeros
-from tensorflow_addons.image import connected_components
+try:
+    from tensorflow_addons.image import connected_components
+except ModuleNotFoundError:
+    pass # n/a beyond TF 2.15 (and only needed for training) ...
 import numpy as np
 
 

src/eynollah/training/models.py

@@ -21,6 +21,7 @@ from tensorflow.keras.layers import (
     LSTM,
     MaxPooling2D,
     MultiHeadAttention,
+    Permute,
     Reshape,
     UpSampling2D,
     ZeroPadding2D,
@@ -70,7 +71,7 @@ class CTCDecoder(Layer):
         ## but Keras greedy sometimes removes arbitrary letters
         # outputs, logits = tf.keras.backend.ctc_decode(inputs,
         #                                               lengths,
-        #                                               beam_width=20
+        #                                               beam_width=20,
         #                                               greedy=False, # True,
         #                                               # backend does not allow these kwargs
         #                                               #merge_repeated=False,
@@ -81,18 +82,17 @@ class CTCDecoder(Layer):
         inputs = tf.math.log(
             tf.transpose(inputs, perm=[1, 0, 2]) + tf.keras.backend.epsilon()
         )
-        # tf.nn.ctc_greedy_decoder() is not as precise
         # tf.compat.v1.nn.ctc_beam_search_decoder() also needs merge_repeated=False
-        decoded, logits = tf.nn.ctc_beam_search_decoder(
+        # tf.nn.ctc_beam_search_decoder() is not supported by ONNX, yet
+        # tf.nn.ctc_greedy_decoder() is not as precise, though:
+        decoded, logits = tf.nn.ctc_greedy_decoder(
             inputs,
             lengths,
-            beam_width=10,
-            top_paths=2,
         )
         # get top path for all sequences in batch
         decoded = decoded[0]
-        logits = logits[:, 0] - logits[:, 1]
+        logits = logits[:, 0]
-        probs = tf.exp(-logits)
+        probs = tf.exp(-logits / n_steps)
         # convert to dense
         outputs = tf.SparseTensor(decoded.indices, decoded.values,
                                   (n_samples, n_steps))
@@ -530,10 +530,12 @@ def cnn_rnn_ocr_model(input_height=None, input_width=None, n_classes=None, max_l
 
     addition_rnn = Bidirectional(LSTM(input_width, return_sequences=True, dropout=0.25))(addition)
 
-    out = Conv1D(max_len, 1, data_format="channels_first")(addition_rnn)
+    #out = Conv1D(max_len, 1, data_format="channels_first")(addition_rnn)
+    out = Permute((2, 1))(addition_rnn)
+    out = Conv1D(max_len, 1, data_format="channels_last")(out)
+    out = Permute((2, 1))(out)
     out = BatchNormalization(name="bn9")(out)
     out = Activation("relu", name="relu9")(out)
-    #out = Conv1D(n_classes, 1, activation='relu', data_format="channels_last")(out)
 
     out = Dense(n_classes, activation="softmax", name="dense2")(out)
 
@@ -552,8 +554,6 @@ def cnn_rnn_ocr_model(input_height=None, input_width=None, n_classes=None, max_l
         voc = char2num.get_vocabulary()
         num2char = StringLookup(vocabulary=voc, invert=True)
         output = num2char(out)
-        # avoid output tf.dtype=string → np.dtype=object (which cannot be shm-ed)
-        output = tf.io.decode_raw(output, tf.uint8, fixed_length=max(map(len, voc)))
 
         return Model((inputs, inputs_bin), (output, prob))
 
@@ -582,8 +582,6 @@ def cnn_rnn_ocr_model4inference(model, model_path):
         voc = char2num.get_vocabulary()
         num2char = StringLookup(vocabulary=voc, invert=True)
         output = num2char(output)
-        # avoid output tf.dtype=string → np.dtype=object (which cannot be shm-ed)
-        output = tf.io.decode_raw(output, tf.uint8, fixed_length=max(map(len, voc)))
         inputs = (inputs, inputs_bin)
         outputs = (output, prob)
         return Model(inputs, outputs)

src/eynollah/training/reload-models-v0.8.mk

@@ -38,7 +38,7 @@ $(MODELS_DST)/%: $(MODELS_SRC)/%
 		--in $< \
 		--format $(FORMAT) \
 		--out $@ \
-	2>&1 | tee $(notdir $<).$(FORMAT).log
+	> $(notdir $<).$(FORMAT).log 2>&1 || { cat $(notdir $<).$(FORMAT).log; false; }
 
 $(MODELS_DST)/%.keras: $(MODELS_SRC)/%
 	eynollah-training convert \
@@ -46,7 +46,7 @@ $(MODELS_DST)/%.keras: $(MODELS_SRC)/%
 		--in $< \
 		--format keras \
 		--out $@ \
-	2>&1 | tee $(notdir $<).keras.log
+	> $(notdir $<).keras.log 2>&1 || { cat $(notdir $<).keras.log; false; }
 
 $(MODELS_DST)/%.h5: $(MODELS_SRC)/%
 	eynollah-training convert \
@@ -54,20 +54,15 @@ $(MODELS_DST)/%.h5: $(MODELS_SRC)/%
 		--in $< \
 		--format hdf5 \
 		--out $@ \
-	2>&1 | tee $(notdir $<).hdf5.log
+	> $(notdir $<).hdf5.log 2>&1 || { cat $(notdir $<).hdf5.log; false; }
 
 $(MODELS_DST)/%.onnx: $(MODELS_SRC)/%
-	if jq -e '.task == "segmentation" and .backbone_type == "transformer"' $</config.json &>/dev/null; then \
-	echo skipping $@: vision transformer architecture currently does not work with ONNX; \
-	elif jq -e '.task == "cnn-rnn-ocr"' $</config.json &>/dev/null || test x$(findstring _ocr,$@) = x_ocr; then \
-	echo skipping $@: OCR CTC decoder does not work with ONNX; \
-	else \
 	eynollah-training convert \
 		$(and $(wildcard $</config.json),--rebuild) \
 		--in $< \
 		--format onnx \
 		--out $@ \
-	2>&1 | tee $(notdir $<).onnx.log; fi
+	> $(notdir $<).onnx.log 2>&1 || { cat $(notdir $<).onnx.log; false; }
 
 compare: 
 	for i in `find $(MODELS_DST) -mindepth 2`;do \

train/requirements.txt

@@ -6,8 +6,10 @@ imutils
 scipy
 tensorflow-addons # for connected_components, depublished and only compatible with tensorflow < 2.16
 tensorflow < 2.16 # for tensorflow-addons, so only needed in training
-tf_data < 2.16 # for tensorflow-addons, so only needed in training
+tf-keras < 2.16 # avoid keras 3 (also needs TF_USE_LEGACY_KERAS=1)
 protobuf < 5 # for tensorflow-addons, so only needed in training
 torch
 transformers <= 4.30.2 ; python_version < '3.10'
 transformers >= 5 ; python_version >= '3.10'
+eynollah-fork-tf2onnx == 1.17.0.post2
+ml_dtypes >= 0.5