Merge branch 'fix-0.8-modelzoo-and-predictor' of https://github.com/bertsky/eynollah into fix-0.8-modelzoo-and-predictor

.github/workflows/test-eynollah.yml

@@ -64,6 +64,9 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
+        # preempt CUDA dependencies (which need core's recipe)
+        pip install onnxruntime tensorflow tf-keras "torch<2.11"
+        sed -i '/onnxruntime-gpu/d;/tensorrt/d;/torch/d;/tensorflow/d' requirements*.txt
         make install-dev EXTRAS=OCR,plotting
         make deps-test EXTRAS=OCR,plotting
 

pyproject.toml

@@ -67,6 +67,7 @@ source = ["eynollah"]
 
 [tool.ruff]
 line-length = 120
+include = ["pyproject.toml", "src/eynollah/**/*.py"]
 
 [tool.ruff.lint]
 ignore = [

requirements-ocr.txt

@@ -1,5 +1,5 @@
-torch
+torch < 2.11 # avoid pull CUDA 13 (which will clash with 12)
 transformers <= 4.30.2 ; python_version < '3.10'
 transformers >= 5 ; python_version >= '3.10'
-tensorflow < 2.16 # for tensorflow-addons, so only needed in training
+tensorflow[and-cuda]
-tf-keras < 2.16 # avoid keras 3 (also needs TF_USE_LEGACY_KERAS=1)
+tf-keras # avoid keras 3 (also needs TF_USE_LEGACY_KERAS=1)

src/eynollah/mb_ro_on_layout.py

@@ -42,7 +42,6 @@ class Reorder(Eynollah):
         self.logger = logger or logging.getLogger('eynollah.mbreorder')
         self.model_zoo = model_zoo
         
-        self.model_zoo.load_model('reading_order')
         self.setup_models(device=device)
 
     def setup_models(self, device=''):

src/eynollah/model_zoo/model_zoo.py

@@ -334,6 +334,8 @@ class EynollahModelZoo:
             providers = [provider for provider in providers
                          if provider[:-17] in override_providers]
         # configure and prioritise
+        if 'AzureExecutionProvider' in providers:
+            providers.remove('AzureExecutionProvider')
         if 'CUDAExecutionProvider' in providers:
             providers.remove('CUDAExecutionProvider')
             if gpu >= 0:

src/eynollah/ocrd-tool.json

@@ -89,14 +89,14 @@
           "name": "models_inference_all_v0_9_0",
           "type": "archive",
 	  "size": 5177204769,
-          "description": "Models for layout detection, reading order detection, textline detection, page extraction, column classification, table detection, binarization and image enhancement",
+          "description": "Models for layout detection, reading order detection, textline detection, page extraction, column classification, table detection, binarization and image enhancement, as well as OCR",
           "version_range": ">= v0.9.0"
         },
         {
-          "url": "https://zenodo.org/records/21362927/files/models_inference_all_v0_9_0.zip",
+          "url": "https://zenodo.org/records/21362927/files/models_inference_layout_v0_9_0.zip",
-          "name": "models_inference_all_v0_9_0",
+          "name": "models_inference_layout_v0_9_0",
           "type": "archive",
-	  "size": 5177204769,
+	  "size": 1572255489,
           "description": "Models for layout detection, reading order detection, textline detection, page extraction, column classification, table detection, binarization and image enhancement",
           "version_range": ">= v0.9.0"
         },

src/eynollah/training/utils.py

@@ -1127,13 +1127,21 @@ def preprocess_img_ocr(
     if task == 'cnn-rnn-ocr':
         assert char_to_num, 'task is cnn-rnn-ocr, so preprocess_imgs_ocr should be passed "char_to_num"'
         lab = char_to_num(tf.strings.unicode_split(lab, input_encoding="UTF-8"))
-        yield_encoder = lambda x: x
+        def yield_encoder(x):
+            return x
     elif task == 'transformer-ocr':
+        import torch
         assert processor, 'task is transformer-ocr, so preprocess_imgs_ocr should be passed "processor"'
         # TODO make max_length configurable again, if deemed sensible
-        lab = [l if l != self.processor.tokenizer.pad_token_id else -100
+        lab = [tok if tok != processor.tokenizer.pad_token_id else -100
-                for l in processor.tokenizer(lab, padding="max_length", max_length=128).input_ids]
+                for tok in processor.tokenizer(lab,
-        yield_encoder = lambda img_, lab_: {"pixel_values": processor(Image.fromarray(img_), return_tensors="pt").pixel_values.squeeze(), "labels": torch.tensor(lab_)}
+                                               padding="max_length",
+                                               max_length=128
+                ).input_ids]
+        def yield_encoder(img_, lab_):
+            return {"pixel_values": processor(Image.fromarray(img_),
+                                              return_tensors="pt").pixel_values.squeeze(),
+                    "labels": torch.tensor(lab_)}
     yield yield_encoder(scale_image(img), lab)
     #to_yield = {"image": ret_x, "label": ret_y}