Merge branch 'fix-0.8-modelzoo-and-predictor' into release-v0.9

.dockerignore

@@ -2,5 +2,9 @@ tests
 dist
 build
 env*
+venv*
 *.egg-info
 models_eynollah*
+reloaded
+*.h5
+config_files*

.github/workflows/test-eynollah.yml

@@ -64,6 +64,9 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
+        # preempt CUDA dependencies (which need core's recipe)
+        pip install onnxruntime tensorflow tf-keras "torch<2.11"
+        sed -i '/onnxruntime-gpu/d;/tensorrt/d;/torch/d;/tensorflow/d' requirements*.txt
         make install-dev EXTRAS=OCR,plotting
         make deps-test EXTRAS=OCR,plotting
 

.gitignore

@@ -1,14 +1,18 @@
 *.egg-info
 __pycache__
 sbb_newspapers_org_image/pylint.log
-models_eynollah*
-models_ocr*
-models_layout*
 default-2021-03-09
 output.html
 /build
-/dist
 *.tif
 *.sw?
 TAGS
 uv.lock
+/dist
+/ignore
+*.log
+.env
+models/reloaded
+models/packages
+models/models_eynollah
+models/pretrained_model

Dockerfile

@@ -15,7 +15,7 @@ LABEL \
     org.opencontainers.image.documentation="https://github.com/qurator-spk/eynollah/blob/${VCS_REF}/README.md" \
     org.opencontainers.image.revision=$VCS_REF \
     org.opencontainers.image.created=$BUILD_DATE \
-    org.opencontainers.image.base.name=ocrd/core-cuda-tf2
+    org.opencontainers.image.base.name=ocrd/core-cuda-onnx
 
 ENV DEBIAN_FRONTEND=noninteractive
 # set proper locales
@@ -39,9 +39,10 @@ RUN ocrd ocrd-tool ocrd-tool.json dump-tools > $(dirname $(ocrd bashlib filename
 # prepackage ocrd-all-module-dir.json
 RUN ocrd ocrd-tool ocrd-tool.json dump-module-dirs > $(dirname $(ocrd bashlib filename))/ocrd-all-module-dir.json
 # install everything and reduce image size
-RUN make install EXTRAS=OCR && rm -rf /build/eynollah
+# FIXME: EXTRAS=OCR (should become extra Dockerfile based on ocrd/core-cuda-tf2 and ocrd/core-cuda-torch)
-# fixup for broken cuDNN installation (Torch pulls in 8.5.0, which is incompatible with Tensorflow)
+RUN make install && rm -rf /build/eynollah
-RUN pip install nvidia-cudnn-cu11==8.6.0.163
+# fixup for broken cuDNN installation (Torch may pull in version which is incompatible with Tensorflow)
+RUN pip install "nvidia-cudnn-cu12<9.10.2.21"
 # smoke test
 RUN eynollah --help
 

Makefile

@@ -2,7 +2,7 @@ PYTHON ?= python3
 PIP ?= pip3
 EXTRAS ?=
 
-DOCKER_BASE_IMAGE ?= docker.io/ocrd/core-cuda-tf2:v3.13.0
+DOCKER_BASE_IMAGE ?= docker.io/ocrd/core-cuda-onnx:v3.13.1
 DOCKER_TAG ?= ocrd/eynollah
 DOCKER ?= docker
 WGET = wget -O
@@ -13,7 +13,8 @@ WGET = wget -O
 #SEG_MODEL := https://github.com/qurator-spk/eynollah/releases/download/v0.3.0/models_eynollah.tar.gz
 #SEG_MODEL := https://github.com/qurator-spk/eynollah/releases/download/v0.3.1/models_eynollah.tar.gz
 #SEG_MODEL := https://zenodo.org/records/17194824/files/models_layout_v0_5_0.tar.gz?download=1
-EYNOLLAH_MODELS_URL := https://zenodo.org/records/17727267/files/models_all_v0_8_0.zip
+# EYNOLLAH_MODELS_URL := https://zenodo.org/records/21362927/files/models_inference_all_v0_9_0.zip
+EYNOLLAH_MODELS_URL := https://zenodo.org/records/21381102/files/models_inference_all_v0_9_1.zip
 EYNOLLAH_MODELS_ZIP = $(notdir $(EYNOLLAH_MODELS_URL))
 EYNOLLAH_MODELS_DIR = $(EYNOLLAH_MODELS_ZIP:%.zip=%)
 

README.md

@@ -25,9 +25,12 @@
 documents using a combination of multiple deep learning models and heuristics; therefore processing can be slow.
 
 ## Installation
-Python `3.8-3.11` with Tensorflow `<2.13` on Linux are currently supported.
+
-For (limited) GPU support the CUDA toolkit needs to be installed. 
+Python `3.8-3.11` with ONNX Runtime on Linux are currently supported.
-A working config is CUDA `11.8` with cuDNN `8.6`.
+
+For GPU support, NVidia drivers supporting CUDA 12 must be installed.
+The runtime dependencies will pull in ONNX, TensorRT and CUDA runtime
+libraries (including cuDNN) from PyPI.
 
 You can either install from PyPI
 
@@ -52,6 +55,13 @@ pip install "eynollah[OCR]"
 make install EXTRAS=OCR
 ```
 
+> **Note**: Requirements for OCR are more involved, 
+> as they may need Tensorflow (with tf-keras) and/or
+> Torch (with transformers). Those two frameworks may
+> also have conflicting CUDA dependencies. An ONNX
+> conversion for these models may be achieved soon.
+> :construction:
+
 ### Docker
 
 Use
@@ -64,7 +74,12 @@ When using Eynollah with Docker, see [`docker.md`](https://github.com/qurator-sp
 
 ## Models
 
-Pretrained models can be downloaded from [Zenodo](https://zenodo.org/records/17727267) or [Hugging Face](https://huggingface.co/SBB?search_models=eynollah). 
+Pretrained models can be downloaded from [Zenodo](https://doi.org/10.5281/zenodo.17194823) or [Hugging Face](https://huggingface.co/SBB?search_models=eynollah).
+
+For fast runtime inference, download the ONNX models distributed as `models_inference_...zip`. 
+
+For finetuning training, download the original (Tensorflow / Torch) models distributed as `models_training...zip`
+(and install the `[training]` extra).
 
 For model documentation and model cards, see [`models.md`](https://github.com/qurator-spk/eynollah/tree/main/docs/models.md).
 
@@ -83,18 +98,26 @@ Eynollah supports five use cases:
 
 Some example outputs can be found in [`examples.md`](https://github.com/qurator-spk/eynollah/tree/main/docs/examples.md).
 
+The **generic options** shared by all subcommands are:
+```sh
+  -m <directory containing model files>
+  -mv <model category> <model variant> <model path>
+  -D <device specifier>
+  -l <log level>
+```
+
 ### Layout Analysis
 
-The layout analysis module is responsible for detecting layout elements, identifying text lines, and determining reading 
+Detects layout elements, i.e. regions of various types and text lines,
-order using heuristic methods or a [pretrained model](https://github.com/qurator-spk/eynollah#machine-based-reading-order).
+and determines their reading order using either heuristic methods or a
+[pretrained model](https://github.com/qurator-spk/eynollah#machine-based-reading-order).
 
 The command-line interface for layout analysis can be called like this:
 
 ```sh
-eynollah layout \
+eynollah [GENERIC_OPTIONS] layout \
   -i <single image file> | -di <directory containing image files> \
   -o <output directory> \
-  -m <directory containing model files> \
      [OPTIONS]
 ```
 
@@ -106,7 +129,7 @@ The following options can be used to further configure the processing:
 | `-tab`            | apply table detection                                                                       |
 | `-ae`             | apply enhancement (the resulting image is saved to the output directory)                    |
 | `-as`             | apply scaling                                                                               |
-| `-cl`             | apply contour detection for curved text lines instead of bounding boxes                     |
+| `-cl`             | apply contour detection for curved text lines, deskewing all regions independently          |
 | `-ib`             | apply binarization (the resulting image is saved to the output directory)                   |
 | `-ep`             | enable plotting (MUST always be used with `-sl`, `-sd`, `-sa`, `-si` or `-ae`)              |
 | `-ho`             | ignore headers for reading order dectection                                                 |
@@ -115,79 +138,93 @@ The following options can be used to further configure the processing:
 | `-sl <directory>` | save layout prediction as plot to this directory                                            |
 | `-sp <directory>` | save cropped page image to this directory                                                   |
 | `-sa <directory>` | save all (plot, enhanced/binary image, layout) to this directory                            |
-| `-thart`          | threshold of artifical class in the case of textline detection. The default value is 0.1    |
+| `-thart`          | confidence threshold of artifical boundary class during textline detection                  |
-| `-tharl`          | threshold of artifical class in the case of layout detection. The default value is 0.1      |
+| `-tharl`          | confidence threshold of artifical boundary class during region detection                    |
 | `-ncu`            | upper limit of columns in document image                                                    |
 | `-ncl`            | lower limit of columns in document image                                                    |
 | `-slro`           | skip layout detection and reading order                                                     |
 | `-romb`           | apply machine based reading order detection                                                 |
 | `-ipe`            | ignore page extraction                                                                      |
+| `-j`              | number of CPU jobs to run parallel (useful with -di)                                        |
+| `-H`              | when to halt when some jobs fail                                                            |
 
+The default is to only perform layout detection of main regions
+(background, text, images, separators and marginals).
 
-If no further option is set, the tool performs layout detection of main regions (background, text, images, separators 
+The best output quality is achieved when RGB images are used as input
-and marginals).
+rather than greyscale or binarized images.
-The best output quality is achieved when RGB images are used as input rather than greyscale or binarized images.
 
-Additional documentation can be found in [`usage.md`](https://github.com/qurator-spk/eynollah/tree/main/docs/usage.md).
+Additional documentation can be found in
+[`usage.md`](https://github.com/qurator-spk/eynollah/tree/main/docs/usage.md).
 
 ### Binarization
 
-The binarization module performs document image binarization using pretrained pixelwise segmentation models. 
+Performs document image binarization (thresholding)
+using pretrained pixelwise segmentation models. 
 
 The command-line interface for binarization can be called like this:
 
 ```sh
-eynollah binarization \
+eynollah [GENERIC_OPTIONS] binarization \
   -i <single image file> | -di <directory containing image files> \
   -o <output directory> \
-  -m <directory containing model files> 
+  [OPTIONS]
 ```
 
 ### Image Enhancement
-TODO
+
+This enlarges and enhances images. Useful in case the scan quality is low.
+
+```sh
+eynollah [GENERIC_OPTIONS] enhancement \
+  -i <single image file> | -di <directory containing image files> \
+  -o <output directory> \
+  [OPTIONS]
+```
+
+| option            | description                                                                                 |
+|-------------------|:--------------------------------------------------------------------------------------------|
+| `-sos`            | save the enhanced image in original image size                                              |
+| `-ncu`            | upper limit of columns in document image                                                    |
+| `-ncl`            | lower limit of columns in document image                                                    |
 
 ### OCR
 
-The OCR module performs text recognition using either a CNN-RNN model or a Transformer model.
+Performs text recognition using either a CNN-RNN model or a Transformer model.
+Needs a PAGE-XML input file.
 
 The command-line interface for OCR can be called like this:
 
 ```sh
-eynollah ocr \
+eynollah [GENERIC_OPTIONS] ocr \
   -i <single image file> | -di <directory containing image files> \
   -dx <directory of xmls> \
   -o <output directory> \
-  -m <directory containing model files> | --model_name <path to specific model>
 ```
 
 The following options can be used to further configure the ocr processing:
 
 | option            | description                                                                                |
 |-------------------|:-------------------------------------------------------------------------------------------|
+| `-trocr`          | use transformer OCR model instead of CNN-RNN model                                         |
 | `-dib`            | directory of binarized images (file type must be '.png'), prediction with both RGB and bin |
 | `-doit`           | directory for output images rendered with the predicted text                               |
-| `--model_name`    | file path to use specific model for OCR                                                    |
-| `-trocr`          | use transformer ocr model (otherwise cnn_rnn model is used)                                |
-| `-etit`           | export textline images and text in xml to output dir (OCR training data)                   |
 | `-nmtc`           | cropped textline images will not be masked with textline contour                           |
 | `-bs`             | ocr inference batch size. Default batch size is 2 for trocr and 8 for cnn_rnn models       |
-| `-ds_pref`        | add an abbrevation of dataset name to generated training data                              |
 | `-min_conf`       | minimum OCR confidence value. OCR with textline conf lower than this will be ignored       |
 
 
 ### Reading Order Detection
-Reading order detection can be performed either as part of layout analysis based on image input, or, currently under 
-development, based on pre-existing layout analysis data in PAGE-XML format as input.
 
-The reading order detection module employs a pretrained model to identify the reading order from layouts represented in PAGE-XML files.
+Reading order can be detected either during layout analysis,
+or as a separate module, which requires a PAGE-XML input file.
 
 The command-line interface for machine based reading order can be called like this:
 
 ```sh
-eynollah machine-based-reading-order \
+eynollah [GENERIC_OPTIONS] machine-based-reading-order \
   -i <single image file> | -di <directory containing image files> \
   -xml <xml file name> | -dx <directory containing xml files> \
-  -m <path to directory containing model files> \
   -o <output directory> 
 ```
 

docs/ocrd.md

@@ -5,7 +5,7 @@ formally described in [`ocrd-tool.json`](https://github.com/qurator-spk/eynollah
 
 When using Eynollah in OCR-D, the source image file group with (preferably) RGB images should be used as input like this:
 
-    ocrd-eynollah-segment -I OCR-D-IMG -O OCR-D-SEG -P models eynollah_layout_v0_5_0
+    ocrd-eynollah-segment -I OCR-D-IMG -O OCR-D-SEG -P models eynollah_layout_v0_9_0
 
 If the input file group is PAGE-XML (from a previous OCR-D workflow step), Eynollah behaves as follows:
 - existing regions are kept and ignored (i.e. in effect they might overlap segments from Eynollah results)
@@ -17,7 +17,7 @@ If the input file group is PAGE-XML (from a previous OCR-D workflow step), Eynol
   (because some other preprocessing step was in effect like `denoised`), then
   the output PAGE-XML will be based on that as new top-level (`@imageFilename`)
 
-      ocrd-eynollah-segment -I OCR-D-XYZ -O OCR-D-SEG -P models eynollah_layout_v0_5_0
+      ocrd-eynollah-segment -I OCR-D-XYZ -O OCR-D-SEG -P models eynollah_layout_v0_9_0
 
 In general, it makes more sense to add other workflow steps **after** Eynollah.
 

docs/usage.md

@@ -3,49 +3,69 @@ The command-line interface can be called like this:
 
 ```sh
 eynollah \
+  [GENERIC_OPTIONS] \
+  layout \
   -i <single image file> | -di <directory containing image files> \
   -o <output directory> \
-  -m <directory containing model files> \
+  [LAYOUT_OPTIONS]
-     [OPTIONS]
 ```
 
-## Processing options
+## Generic options
-The following options can be used to further configure the processing:
+Pass any of the following options:
 
-| option            | description                                                                    |
+| **option**                                        | **description**                                                 |
+|---------------------------------------------------|:----------------------------------------------------------------|
+| -m <directory containing model files>             | override default directory `$PWD/models_eynollah`               |
+| -mv <model category> <model variant> <model path> | override specific models, e.g. `region_1_2 '' /path/to/my.onnx` |
+| -D <device specifier>                             | allocate models to GPUs, e.g. `col*:CPU,page:GPU1,*:GPU0`       |
+| -l <log level>                                    | override default `INFO` log level e.g. `DEBUG`                  |
+
+## Processing options
+The following options can be used to further control layout analysis:
+
+| **option**        | **description**                                                                |
 |-------------------|:-------------------------------------------------------------------------------|
-| `-fl`             | full layout analysis including all steps and segmentation classes              |
+| `-fl`             | full layout analysis including all steps and segmentation classes (recommended)|
-| `-light`          | lighter and faster but simpler method for main region detection and deskewing  |
 | `-tab`            | apply table detection                                                          |
 | `-ae`             | apply enhancement (the resulting image is saved to the output directory)       |
 | `-as`             | apply scaling                                                                  |
-| `-cl`             | apply contour detection for curved text lines instead of bounding boxes        |
+| `-cl`             | apply contour detection for curved text lines, deskewing regions independently |
 | `-ib`             | apply binarization (the resulting image is saved to the output directory)      |
 | `-ep`             | enable plotting (MUST always be used with `-sl`, `-sd`, `-sa`, `-si` or `-ae`) |
-| `-eoi`            | extract only images to output directory (other processing will not be done)    |
 | `-ho`             | ignore headers for reading order dectection                                    |
 | `-si <directory>` | save image regions detected to this directory                                  |
 | `-sd <directory>` | save deskewed image to this directory                                          |
 | `-sl <directory>` | save layout prediction as plot to this directory                               |
 | `-sp <directory>` | save cropped page image to this directory                                      |
 | `-sa <directory>` | save all (plot, enhanced/binary image, layout) to this directory               |
+| `-thart`          | confidence threshold of artifical boundary class during textline detection     |
+| `-tharl`          | confidence threshold of artifical boundary class during region detection       |
+| `-ncu`            | upper limit of columns in document image                                       |
+| `-ncl`            | lower limit of columns in document image                                       |
+| `-slro`           | skip layout detection and reading order                                        |
+| `-romb`           | apply machine based reading order detection                                    |
+| `-ipe`            | ignore page extraction                                                         |
+| `-j`              | number of CPU jobs to run parallel (useful with -di)                           |
+| `-H`              | when to halt if some jobs fail, e.g. `0.1` for 10% or `3` for 3 pages          |
 
 If no option is set, the tool performs detection of main regions (background, text, images, separators and marginals).
 
-### `--full-layout` vs `--no-full-layout`
+### `--full-layout` vs shallow
 
-Here are the difference in elements detected depending on the `--full-layout`/`--no-full-layout` command line flags:
+Here are the differences in segment types detected:
 
-|                          | `--full-layout` | `--no-full-layout` |
+|                          | `-fl` | without |
-|--------------------------|-----------------|--------------------|
+|--------------------------|-------|---------|
-| reading order            | x               | x                  |
+| reading order            | x     | x       |
-| header regions           | x               | -                  |
+| header regions           | x     | -       |
-| text regions             | x               | x                  |
+| text regions             | x     | x       |
-| text regions / text line | x               | x                  |
+| text regions / textlines | x     | x       |
-| drop-capitals            | x               | -                  |
+| drop-capitals            | x     | -       |
-| marginals                | x               | x                  |
+| marginals                | x     | x       |
-| marginals / text line    | x               | x                  |
+| marginals / textlines    | x     | x       |
-| image region             | x               | x                  |
+| image regions            | x     | x       |
+
+(Note: No marginals are detected for pages with 3 columns or more.)
 
 ## Use as OCR-D processor
 Eynollah ships with a CLI interface to be used as [OCR-D](https://ocr-d.de) processor that is described in 
@@ -54,13 +74,13 @@ Eynollah ships with a CLI interface to be used as [OCR-D](https://ocr-d.de) proc
 The source image file group with (preferably) RGB images should be used as input for Eynollah like this:
 
 ```
-ocrd-eynollah-segment -I OCR-D-IMG -O SEG-LINE -P models
+ocrd-eynollah-segment -I OCR-D-IMG -O SEG-LINE -P full_layout true
 ```
     
 Any image referenced by `@imageFilename` in PAGE-XML is passed on directly to Eynollah as a processor, so that e.g.
 
 ```
-ocrd-eynollah-segment -I OCR-D-IMG-BIN -O SEG-LINE -P models
+ocrd-eynollah-segment -I OCR-D-IMG-BIN -O SEG-LINE -P full_layout true
 ```
     
 uses the original (RGB) image despite any binarization that may have occured in previous OCR-D processing steps.

models/Makefile

@@ -0,0 +1,142 @@
+SHELL = bash -e
+
+VERSION = v0_9_1
+MODELS_SRC = models_eynollah
+MODELS_DST = reloaded/models_eynollah
+
+
+# eynollah-main-regions-aug-rotation_20210425
+# eynollah-main-regions-aug-scaling_20210425
+# eynollah-main-regions-ensembled_20210425
+# eynollah-main-regions_20220314
+# eynollah-main-regions_20231127_672_org_ens_11_13_16_17_18
+# eynollah-tables_20210319
+
+CURRENT_MODELS := 
+CURRENT_MODELS += eynollah-main-regions-aug-rotation_20210425
+CURRENT_MODELS += eynollah-main-regions-aug-scaling_20210425
+CURRENT_MODELS += eynollah-main-regions-ensembled_20210425
+CURRENT_MODELS += eynollah-main-regions_20220314
+CURRENT_MODELS += eynollah-main-regions_20231127_672_org_ens_11_13_16_17_18
+CURRENT_MODELS += eynollah-tables_20210319
+CURRENT_MODELS += eynollah-main-regions_20220314
+CURRENT_MODELS += model_eynollah_page_extraction_20250915
+CURRENT_MODELS += model_eynollah_reading_order_20250824
+CURRENT_MODELS += modelens_e_l_all_sp_0_1_2_3_4_171024
+CURRENT_MODELS += modelens_full_lay_1__4_3_091124
+CURRENT_MODELS += modelens_table_0t4_201124
+CURRENT_MODELS += modelens_textline_0_1__2_4_16092024
+CURRENT_MODELS += model_eynollah_ocr_cnnrnn_20250930
+CURRENT_MODELS += eynollah-binarization_20210425
+CURRENT_MODELS += eynollah-column-classifier_20210425
+CURRENT_MODELS += eynollah-enhancement_20210425
+
+help:
+	@echo "Targets:"
+	@echo ""
+	@echo "Model conversion:"
+	@echo "  all          Convert all current models to TensorFlow SavedModel (default)"
+	@echo "  tf[-serving] Convert all current models to TensorFlow SavedModel"
+	@echo "  keras        Convert all current models to Keras format (.keras)"
+	@echo "  hdf5         Convert all current models to HDF5 format (.h5)"
+	@echo "  onnx         Convert all eligible current models to ONNX (.onnx)"
+	@echo "  reload       Reload selected pre-v0.8 models without Lambda layers"
+	@echo ""
+	@echo "Model reloading:"
+	@echo "  compare      Compare original and v0.8 reloaded models"
+	@echo ""
+	@echo "Packaging:"
+	@echo "  packages     Build all Zenodo distribution archives"
+	@echo "  upload       Upload all distribution archives to Zenodo"
+	@echo ""
+	@echo "Cleanup:"
+	@echo "  clean           Remove all generated files"
+	@echo "  clean-reload    Remove reloaded model directories"
+	@echo "  clean-packages  Remove generated package archives"
+	@echo ""
+	@echo "Variables:"
+	@echo "  VERSION=$(VERSION)"
+
+
+# tf (SavedModel format) for training
+# onnx conversion for fast inference
+all: tf onnx
+
+tf-serving: $(CURRENT_MODELS:%=$(MODELS_DST)/%)
+tf: $(CURRENT_MODELS:%=$(MODELS_DST)/%)
+keras: $(CURRENT_MODELS:%=$(MODELS_DST)/%.keras)
+hdf5: $(CURRENT_MODELS:%=$(MODELS_DST)/%.h5)
+onnx: $(CURRENT_MODELS:%=$(MODELS_DST)/%.onnx)
+
+# distinguish tf from tf-serving: target pattern is the same,
+# so check if either is current goal, otherwise assumg tf
+$(MODELS_DST)/%: FORMAT = $(or $(filter tf-serving,$(MAKECMDGOALS)), tf)
+$(MODELS_DST)/%: $(MODELS_SRC)/%
+	$(if $(and $(filter tf-serving,$(FORMAT)),$(findstring _ocr,$@)),$(warning skipping $@: OCR CTC decoder fails in TF-Serving) : )\
+	eynollah-training convert \
+		$(and $(wildcard $</config.json),--rebuild) \
+		--in $< \
+		--format $(FORMAT) \
+		--out $@ \
+	> $(notdir $<).$(FORMAT).log 2>&1 || { cat $(notdir $<).$(FORMAT).log; false; }
+
+$(MODELS_DST)/%.keras: $(MODELS_SRC)/%
+	eynollah-training convert \
+		$(and $(wildcard $</config.json),--rebuild) \
+		--in $< \
+		--format keras \
+		--out $@ \
+	> $(notdir $<).keras.log 2>&1 || { cat $(notdir $<).keras.log; false; }
+
+$(MODELS_DST)/%.h5: $(MODELS_SRC)/%
+	eynollah-training convert \
+		$(and $(wildcard $</config.json),--rebuild) \
+		--in $< \
+		--format hdf5 \
+		--out $@ \
+	> $(notdir $<).hdf5.log 2>&1 || { cat $(notdir $<).hdf5.log; false; }
+
+$(MODELS_DST)/%.onnx: $(MODELS_SRC)/%
+	$(if $(findstring _ocr,$@),$(warning skipping $@: OCR CTC decoder is buggy in ONNX) : )\
+	eynollah-training convert \
+		$(and $(wildcard $</config.json),--rebuild) \
+		--in $< \
+		--format onnx \
+		--out $@ \
+	> $(notdir $<).onnx.log 2>&1 || { cat $(notdir $<).onnx.log; false; }
+
+compare: 
+	for i in `find $(MODELS_DST) -mindepth 2`;do \
+	 	n=$(MODELS_SRC)$${i#$(MODELS_DST)}; \
+		du -bs $$n $$i ; \
+	done
+
+clean: clean-reload clean-packages
+
+clean-reload:
+	rm -rf $(RELOADABLE_MODELS)
+
+clean-packages:
+	rm -rf $(PACKAGES_DIR)
+
+# NB: this works using symlinks
+PACKAGES_DIR = packages
+BUNDLES = inference training
+CATEGORIES = layout ocr extra all
+
+packages: $(foreach B,$(BUNDLES),$(foreach C,$(CATEGORIES),$(PACKAGES_DIR)/models_$(B)_$(C)_$(VERSION).zip))
+
+$(PACKAGES_DIR)/models_%.zip:
+	@mkdir -p $(PACKAGES_DIR) ;\
+	bundle=$(word 1,$(subst _, ,$*)); \
+	category=$(word 2,$(subst _, ,$*)); \
+	echo "Packaging $$bundle/$$category to $(notdir $@)" ;\
+	cd "dist/$$bundle/$$category" && \
+	zip -vqr "$$OLDPWD/$@" models_eynollah
+
+
+upload: $(foreach B,$(BUNDLES),$(foreach C,$(CATEGORIES),upload/$(B)_$(C)))
+
+upload/%:
+	@source .env ;\
+	bash scripts/zenodo_upload.sh $$ZENODO_ID $(PACKAGES_DIR)/models_$*_$(VERSION).zip --verbose

models/dist/inference/all/models_eynollah/characters_org.txt

@@ -0,0 +1 @@
+../../../../models_eynollah/characters_org.txt

models/dist/inference/all/models_eynollah/eynollah-binarization_20210309

@@ -0,0 +1 @@
+../../../../models_eynollah/eynollah-binarization_20210309

models/dist/inference/all/models_eynollah/eynollah-binarization_20210425.onnx

@@ -0,0 +1 @@
+../../../../reloaded/models_eynollah/eynollah-binarization_20210425.onnx

models/dist/inference/all/models_eynollah/eynollah-column-classifier_20210425.onnx

@@ -0,0 +1 @@
+../../../../reloaded/models_eynollah/eynollah-column-classifier_20210425.onnx

models/dist/inference/all/models_eynollah/eynollah-enhancement_20210425.onnx

@@ -0,0 +1 @@
+../../../../reloaded/models_eynollah/eynollah-enhancement_20210425.onnx

models/dist/inference/all/models_eynollah/eynollah-main-regions-aug-rotation_20210425.onnx

@@ -0,0 +1 @@
+../../../../reloaded/models_eynollah/eynollah-main-regions-aug-rotation_20210425.onnx

models/dist/inference/all/models_eynollah/eynollah-main-regions-aug-scaling_20210425.onnx

@@ -0,0 +1 @@
+../../../../reloaded/models_eynollah/eynollah-main-regions-aug-scaling_20210425.onnx

models/dist/inference/all/models_eynollah/eynollah-main-regions-ensembled_20210425.onnx

@@ -0,0 +1 @@
+../../../../reloaded/models_eynollah/eynollah-main-regions-ensembled_20210425.onnx

models/dist/inference/all/models_eynollah/eynollah-main-regions_20220314.onnx

@@ -0,0 +1 @@
+../../../../reloaded/models_eynollah/eynollah-main-regions_20220314.onnx

models/dist/inference/all/models_eynollah/eynollah-main-regions_20231127_672_org_ens_11_13_16_17_18.onnx

@@ -0,0 +1 @@
+../../../../reloaded/models_eynollah/eynollah-main-regions_20231127_672_org_ens_11_13_16_17_18.onnx

models/dist/inference/all/models_eynollah/eynollah-tables_20210319.onnx

@@ -0,0 +1 @@
+../../../../reloaded/models_eynollah/eynollah-tables_20210319.onnx

models/dist/inference/all/models_eynollah/model_eynollah_ocr_cnnrnn_20250930

@@ -0,0 +1 @@
+../../../../reloaded/models_eynollah/model_eynollah_ocr_cnnrnn_20250930

models/dist/inference/all/models_eynollah/model_eynollah_ocr_cnnrnn__degraded_20250805

@@ -0,0 +1 @@
+../../../../models_eynollah/model_eynollah_ocr_cnnrnn__degraded_20250805

models/dist/inference/all/models_eynollah/model_eynollah_ocr_trocr_20250919

@@ -0,0 +1 @@
+../../../../models_eynollah/model_eynollah_ocr_trocr_20250919

models/dist/inference/all/models_eynollah/model_eynollah_page_extraction_20250915.onnx

@@ -0,0 +1 @@
+../../../../reloaded/models_eynollah/model_eynollah_page_extraction_20250915.onnx

models/dist/inference/all/models_eynollah/model_eynollah_reading_order_20250824.onnx

@@ -0,0 +1 @@
+../../../../reloaded/models_eynollah/model_eynollah_reading_order_20250824.onnx

models/dist/inference/all/models_eynollah/modelens_e_l_all_sp_0_1_2_3_4_171024.onnx

@@ -0,0 +1 @@
+../../../../reloaded/models_eynollah/modelens_e_l_all_sp_0_1_2_3_4_171024.onnx

models/dist/inference/all/models_eynollah/modelens_full_lay_1__4_3_091124.onnx

@@ -0,0 +1 @@
+../../../../reloaded/models_eynollah/modelens_full_lay_1__4_3_091124.onnx

models/dist/inference/all/models_eynollah/modelens_table_0t4_201124.onnx

@@ -0,0 +1 @@
+../../../../reloaded/models_eynollah/modelens_table_0t4_201124.onnx

models/dist/inference/all/models_eynollah/modelens_textline_0_1__2_4_16092024.onnx

@@ -0,0 +1 @@
+../../../../reloaded/models_eynollah/modelens_textline_0_1__2_4_16092024.onnx

models/dist/inference/extra/models_eynollah/eynollah-binarization_20210309

@@ -0,0 +1 @@
+../../../../models_eynollah/eynollah-binarization_20210309

models/dist/inference/extra/models_eynollah/eynollah-binarization_20210425.onnx

@@ -0,0 +1 @@
+../../../../reloaded/models_eynollah/eynollah-binarization_20210425.onnx

models/dist/inference/extra/models_eynollah/eynollah-main-regions-aug-scaling_20210425.onnx

@@ -0,0 +1 @@
+../../../../reloaded/models_eynollah/eynollah-main-regions-aug-scaling_20210425.onnx

models/dist/inference/extra/models_eynollah/eynollah-tables_20210319.onnx

@@ -0,0 +1 @@
+../../../../reloaded/models_eynollah/eynollah-tables_20210319.onnx

models/dist/inference/extra/models_eynollah/modelens_textline_0_1__2_4_16092024.onnx

@@ -0,0 +1 @@
+../../../../reloaded/models_eynollah/modelens_textline_0_1__2_4_16092024.onnx

models/dist/inference/layout/models_eynollah/eynollah-binarization_20210425.onnx

@@ -0,0 +1 @@
+../../../../reloaded/models_eynollah/eynollah-binarization_20210425.onnx

models/dist/inference/layout/models_eynollah/eynollah-column-classifier_20210425.onnx

@@ -0,0 +1 @@
+../../../../reloaded/models_eynollah/eynollah-column-classifier_20210425.onnx

models/dist/inference/layout/models_eynollah/eynollah-enhancement_20210425.onnx

@@ -0,0 +1 @@
+../../../../reloaded/models_eynollah/eynollah-enhancement_20210425.onnx

models/dist/inference/layout/models_eynollah/eynollah-main-regions-aug-rotation_20210425.onnx

@@ -0,0 +1 @@
+../../../../reloaded/models_eynollah/eynollah-main-regions-aug-rotation_20210425.onnx

models/dist/inference/layout/models_eynollah/eynollah-main-regions-aug-scaling_20210425.onnx

@@ -0,0 +1 @@
+../../../../reloaded/models_eynollah/eynollah-main-regions-aug-scaling_20210425.onnx

models/dist/inference/layout/models_eynollah/eynollah-main-regions-ensembled_20210425.onnx

@@ -0,0 +1 @@
+../../../../reloaded/models_eynollah/eynollah-main-regions-ensembled_20210425.onnx

models/dist/inference/layout/models_eynollah/eynollah-main-regions_20220314.onnx

@@ -0,0 +1 @@
+../../../../reloaded/models_eynollah/eynollah-main-regions_20220314.onnx

models/dist/inference/layout/models_eynollah/eynollah-main-regions_20231127_672_org_ens_11_13_16_17_18.onnx

@@ -0,0 +1 @@
+../../../../reloaded/models_eynollah/eynollah-main-regions_20231127_672_org_ens_11_13_16_17_18.onnx

models/dist/inference/layout/models_eynollah/model_eynollah_page_extraction_20250915.onnx

@@ -0,0 +1 @@
+../../../../reloaded/models_eynollah/model_eynollah_page_extraction_20250915.onnx

models/dist/inference/layout/models_eynollah/model_eynollah_reading_order_20250824.onnx

@@ -0,0 +1 @@
+../../../../reloaded/models_eynollah/model_eynollah_reading_order_20250824.onnx

models/dist/inference/layout/models_eynollah/modelens_e_l_all_sp_0_1_2_3_4_171024.onnx

@@ -0,0 +1 @@
+../../../../reloaded/models_eynollah/modelens_e_l_all_sp_0_1_2_3_4_171024.onnx

models/dist/inference/layout/models_eynollah/modelens_full_lay_1__4_3_091124.onnx

@@ -0,0 +1 @@
+../../../../reloaded/models_eynollah/modelens_full_lay_1__4_3_091124.onnx

models/dist/inference/layout/models_eynollah/modelens_table_0t4_201124.onnx

@@ -0,0 +1 @@
+../../../../reloaded/models_eynollah/modelens_table_0t4_201124.onnx

models/dist/inference/layout/models_eynollah/modelens_textline_0_1__2_4_16092024.onnx

@@ -0,0 +1 @@
+../../../../reloaded/models_eynollah/modelens_textline_0_1__2_4_16092024.onnx

models/dist/inference/ocr/models_eynollah/characters_org.txt

@@ -0,0 +1 @@
+../../../../models_eynollah/characters_org.txt

models/dist/inference/ocr/models_eynollah/model_eynollah_ocr_cnnrnn_20250930

@@ -0,0 +1 @@
+../../../../reloaded/models_eynollah/model_eynollah_ocr_cnnrnn_20250930

models/dist/inference/ocr/models_eynollah/model_eynollah_ocr_cnnrnn__degraded_20250805

@@ -0,0 +1 @@
+../../../../models_eynollah/model_eynollah_ocr_cnnrnn__degraded_20250805

models/dist/inference/ocr/models_eynollah/model_eynollah_ocr_trocr_20250919

@@ -0,0 +1 @@
+../../../../models_eynollah/model_eynollah_ocr_trocr_20250919

models/dist/training/all/models_eynollah/characters_org.txt

@@ -0,0 +1 @@
+../../../../models_eynollah/characters_org.txt

models/dist/training/all/models_eynollah/eynollah-binarization-hybrid_20230504

@@ -0,0 +1 @@
+../../../../models_eynollah/eynollah-binarization-hybrid_20230504

models/dist/training/all/models_eynollah/eynollah-binarization_20210309

@@ -0,0 +1 @@
+../../../../models_eynollah/eynollah-binarization_20210309

models/dist/training/all/models_eynollah/eynollah-binarization_20210425

@@ -0,0 +1 @@
+../../../../models_eynollah/eynollah-binarization_20210425

models/dist/training/all/models_eynollah/eynollah-column-classifier_20210425

@@ -0,0 +1 @@
+../../../../models_eynollah/eynollah-column-classifier_20210425

models/dist/training/all/models_eynollah/eynollah-enhancement_20210425

@@ -0,0 +1 @@
+../../../../models_eynollah/eynollah-enhancement_20210425

models/dist/training/all/models_eynollah/eynollah-main-regions-aug-rotation_20210425

@@ -0,0 +1 @@
+../../../../models_eynollah/eynollah-main-regions-aug-rotation_20210425

models/dist/training/all/models_eynollah/eynollah-main-regions-aug-scaling_20210425

@@ -0,0 +1 @@
+../../../../models_eynollah/eynollah-main-regions-aug-scaling_20210425

models/dist/training/all/models_eynollah/eynollah-main-regions-ensembled_20210425

@@ -0,0 +1 @@
+../../../../models_eynollah/eynollah-main-regions-ensembled_20210425

models/dist/training/all/models_eynollah/eynollah-main-regions_20220314

@@ -0,0 +1 @@
+../../../../models_eynollah/eynollah-main-regions_20220314

models/dist/training/all/models_eynollah/eynollah-main-regions_20231127_672_org_ens_11_13_16_17_18

@@ -0,0 +1 @@
+../../../../models_eynollah/eynollah-main-regions_20231127_672_org_ens_11_13_16_17_18

models/dist/training/all/models_eynollah/eynollah-tables_20210319

@@ -0,0 +1 @@
+../../../../models_eynollah/eynollah-tables_20210319

models/dist/training/all/models_eynollah/model_eynollah_ocr_cnnrnn_20250930

@@ -0,0 +1 @@
+../../../../models_eynollah/model_eynollah_ocr_cnnrnn_20250930

models/dist/training/all/models_eynollah/model_eynollah_ocr_cnnrnn__degraded_20250805

@@ -0,0 +1 @@
+../../../../models_eynollah/model_eynollah_ocr_cnnrnn__degraded_20250805

models/dist/training/all/models_eynollah/model_eynollah_ocr_trocr_20250919

@@ -0,0 +1 @@
+../../../../models_eynollah/model_eynollah_ocr_trocr_20250919

models/dist/training/all/models_eynollah/model_eynollah_page_extraction_20250915

@@ -0,0 +1 @@
+../../../../models_eynollah/model_eynollah_page_extraction_20250915

models/dist/training/all/models_eynollah/model_eynollah_reading_order_20250824

@@ -0,0 +1 @@
+../../../../models_eynollah/model_eynollah_reading_order_20250824

models/dist/training/all/models_eynollah/modelens_e_l_all_sp_0_1_2_3_4_171024

@@ -0,0 +1 @@
+../../../../models_eynollah/modelens_e_l_all_sp_0_1_2_3_4_171024

models/dist/training/all/models_eynollah/modelens_full_lay_1__4_3_091124

@@ -0,0 +1 @@
+../../../../models_eynollah/modelens_full_lay_1__4_3_091124

models/dist/training/all/models_eynollah/modelens_table_0t4_201124

@@ -0,0 +1 @@
+../../../../models_eynollah/modelens_table_0t4_201124

models/dist/training/all/models_eynollah/modelens_textline_0_1__2_4_16092024

@@ -0,0 +1 @@
+../../../../models_eynollah/modelens_textline_0_1__2_4_16092024

models/dist/training/extra/models_eynollah/eynollah-binarization_20210309

@@ -0,0 +1 @@
+../../../../models_eynollah/eynollah-binarization_20210309

models/dist/training/extra/models_eynollah/eynollah-binarization_20210425

@@ -0,0 +1 @@
+../../../../models_eynollah/eynollah-binarization_20210425

models/dist/training/extra/models_eynollah/eynollah-main-regions-aug-scaling_20210425

@@ -0,0 +1 @@
+../../../../models_eynollah/eynollah-main-regions-aug-scaling_20210425

models/dist/training/extra/models_eynollah/eynollah-tables_20210319

@@ -0,0 +1 @@
+../../../../models_eynollah/eynollah-tables_20210319

models/dist/training/extra/models_eynollah/modelens_textline_0_1__2_4_16092024

@@ -0,0 +1 @@
+../../../../models_eynollah/modelens_textline_0_1__2_4_16092024

models/dist/training/layout/models_eynollah/eynollah-column-classifier_20210425

@@ -0,0 +1 @@
+../../../../models_eynollah/eynollah-column-classifier_20210425

models/dist/training/layout/models_eynollah/eynollah-enhancement_20210425

@@ -0,0 +1 @@
+../../../../models_eynollah/eynollah-enhancement_20210425

models/dist/training/layout/models_eynollah/eynollah-main-regions-aug-rotation_20210425

@@ -0,0 +1 @@
+../../../../models_eynollah/eynollah-main-regions-aug-rotation_20210425

models/dist/training/layout/models_eynollah/eynollah-main-regions-aug-scaling_20210425

@@ -0,0 +1 @@
+../../../../models_eynollah/eynollah-main-regions-aug-scaling_20210425

models/dist/training/layout/models_eynollah/eynollah-main-regions-ensembled_20210425

@@ -0,0 +1 @@
+../../../../models_eynollah/eynollah-main-regions-ensembled_20210425

models/dist/training/layout/models_eynollah/eynollah-main-regions_20220314

@@ -0,0 +1 @@
+../../../../models_eynollah/eynollah-main-regions_20220314

models/dist/training/layout/models_eynollah/eynollah-main-regions_20231127_672_org_ens_11_13_16_17_18

@@ -0,0 +1 @@
+../../../../models_eynollah/eynollah-main-regions_20231127_672_org_ens_11_13_16_17_18

models/dist/training/layout/models_eynollah/model_eynollah_page_extraction_20250915

@@ -0,0 +1 @@
+../../../../reloaded/models_eynollah/model_eynollah_page_extraction_20250915

models/dist/training/layout/models_eynollah/model_eynollah_reading_order_20250824

@@ -0,0 +1 @@
+../../../../models_eynollah/model_eynollah_reading_order_20250824

models/dist/training/layout/models_eynollah/modelens_e_l_all_sp_0_1_2_3_4_171024

@@ -0,0 +1 @@
+../../../../models_eynollah/modelens_e_l_all_sp_0_1_2_3_4_171024

models/dist/training/layout/models_eynollah/modelens_full_lay_1__4_3_091124

@@ -0,0 +1 @@
+../../../../models_eynollah/modelens_full_lay_1__4_3_091124

models/dist/training/layout/models_eynollah/modelens_table_0t4_201124

@@ -0,0 +1 @@
+../../../../models_eynollah/modelens_table_0t4_201124

models/dist/training/layout/models_eynollah/modelens_textline_0_1__2_4_16092024

@@ -0,0 +1 @@
+../../../../models_eynollah/modelens_textline_0_1__2_4_16092024

models/dist/training/ocr/models_eynollah/characters_org.txt

@@ -0,0 +1 @@
+../../../../models_eynollah/characters_org.txt

models/dist/training/ocr/models_eynollah/model_eynollah_ocr_cnnrnn_20250930

@@ -0,0 +1 @@
+../../../../models_eynollah/model_eynollah_ocr_cnnrnn_20250930

models/dist/training/ocr/models_eynollah/model_eynollah_ocr_cnnrnn__degraded_20250805

@@ -0,0 +1 @@
+../../../../models_eynollah/model_eynollah_ocr_cnnrnn__degraded_20250805

models/dist/training/ocr/models_eynollah/model_eynollah_ocr_trocr_20250919

@@ -0,0 +1 @@
+../../../../models_eynollah/model_eynollah_ocr_trocr_20250919

models/scripts/convert.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+for i in */models_eynollah/*;do
+	rslv=$(readlink $i|sed 's,models_eynollah,reloaded/models_eynollah,');
+	ln -srf $rslv $i;
+done

models/scripts/plot.py

@@ -0,0 +1,62 @@
+import os
+import sys
+import more_itertools
+import numpy as np
+import argparse
+
+parser = argparse.ArgumentParser(
+                    prog='plot',
+                    description='Print metrics',
+                    epilog='...')
+parser.add_argument('stem1', default='nohup.out.eynollah-206-noautosize-keepseps-refactoring9-crop-after2')
+parser.add_argument('stem2', default='eval-gt2-onlyfg-pixel-f1-v0.5-206-noautosize-keepseps-refactoring9-crop-after2')
+
+def main():
+    args = parser.parse_args()
+
+    prefix = os.path.dirname(sys.argv[0])
+    files = [path for path in os.listdir(prefix)
+             if path.startswith(args.stem1)
+             and path.endswith('.txt')]
+
+    for path in files:
+        name = path[len(args.stem1):]
+        metric = name.split('.')[-2]
+        series = name[:-(len(metric) + 5)]
+        path = os.path.join(prefix, path)
+
+        if metric == 'peak-vram':
+            cat, peak = np.loadtxt(path, unpack=True, dtype=np.dtype([("category", "U15"), ("peak", float)]))
+            peak //= 1024
+            peak //= 1024
+            print(metric, f"{series: <52}", dict(zip(cat.tolist(), peak.tolist())))
+        else:
+            val = np.loadtxt(path)
+            print(metric, f"{series: <52}", {'µ': float(np.round(np.mean(val), 2)),
+                                             'M': float(np.round(np.median(val), 2)),
+                                             'min': float(np.min(val)),
+                                             'max': float(np.max(val))})
+
+
+    files = [path for path in os.listdir(prefix)
+             if path.startswith(args.stem2)
+             and path.endswith('.log')]
+
+
+    for path in files:
+        name = path[len(args.stem2):]
+        series = name[:-4]
+        path = os.path.join(prefix, path)
+        with open(path, 'r') as fd:
+            lines = fd.readlines()
+        result_v0_5 = {}
+        result_v206 = {}
+        for metric, v0_5, v206 in more_itertools.chunked(lines, n=3, strict=True):
+            metric = metric[len(prefix) + 1:-1]
+            result_v0_5[metric] = float(np.round(float(v0_5.strip()), 3))
+            result_v206[metric] = float(np.round(float(v206.strip()), 3))
+        print("pixel-f1", f"{series: <52}", result_v206)
+
+    print("pixel-f1", "v0.5                                       ", result_v0_5)
+
+

models/scripts/run.sh

@@ -0,0 +1,34 @@
+for variant in reloaded.*; do
+
+for dir in Korrigierte_Layout_GT/*/; do test $dir = ${dir/.} || continue; out=${dir%/}.eynollah-206-noautosize; mkdir -p $out; nohup /usr/bin
+/time eynollah  -D GPU0 -m $variant layout --dir_in $dir -o
+ $out -fl -H 1 -O || break; done
+
+logfile=Korrigierte_Layout_GT/nohup.out.eynollah-206-noautosize-$variant
+
+mv nohup.out $logfile
+
+fgrep peaked $logfile | sort -u | cut -d\  -f2,5 > $logfile.peak-vram.txt
+fgrep initialization $logfile | sed "s/^.* (\([0-9.]*\)s)/\1/" > $logfile.secs-init.txt
+fgrep "Job done in" $logfile | sed "s/^.* //;s/s$//" > $logfile.secs-jobs.txt
+fgrep %CPU $logfile | sed "s/^.* \([0-9]*\)%CPU.*/\1/" > $logfile.prct-cpu.txt
+
+for dir in Korrigierte_Layout_GT/*.eynollah*/; do
+	test $dir = ${dir/-eval} || continue;
+	test $dir != ${dir/eynollah-206} || continue;
+	mets=${dir%.eynollah*}.mets.xml;
+	dir=$(basename $dir);
+	gt=${dir%.eynollah*}.gt2;
+	echo $mets: $dir;
+	ocrd-segment-evaluate -m $mets -I $gt,$dir -O $dir-eval,$dir-eval2 -P level-of-operation region -P ignore-subtype true -P only-fg true --overwrite;
+done
+
+for mets in Korrigierte_Layout_GT/*.mets.xml; do dir=${mets%.mets.xml}; echo $dir; jq '."by-category".TextRegion."pixel-f1".avg' $dir.eynollah-v0.5-eval/*.json $dir.eynollah-206-noautosize-eval/*.json; done > Korrigierte_Layout_GT/eval-gt2-onlyfg-pixel-f1-v0.5-206-noautosize-$variant.log
+
+
+done
+
+
+
+python Korrigierte_Layout_GT/plot.py | sort -V
+

models/scripts/zenodo_upload.sh

@@ -0,0 +1,71 @@
+#!/bin/bash
+#
+# Upload big files to Zenodo.
+#
+# usage: ./zenodo_upload.sh [deposition id] [filename] [--verbose|-v]
+#
+# Taken from https://github.com/jhpoelen/zenodo-upload
+# 
+# Copyright (c) 2019 Jorrit Poelen
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+#
+
+set -e
+
+VERBOSE=0
+if [ "$3" == "--verbose" ] || [ "$3" == "-v" ]; then
+    VERBOSE=1
+fi
+
+# strip deposition url prefix if provided; see https://github.com/jhpoelen/zenodo-upload/issues/2#issuecomment-797657717
+DEPOSITION=$( echo $1 | sed 's+^http[s]*://zenodo.org/deposit/++g' )
+FILEPATH="$2"
+FILENAME=$(echo $FILEPATH | sed 's+.*/++g')
+FILENAME=${FILENAME// /%20}
+ZENODO_ENDPOINT=${ZENODO_ENDPOINT:-https://zenodo.org}
+
+DEPOSIT_URL=${ZENODO_ENDPOINT}/api/deposit/depositions/"$DEPOSITION"?access_token="$ZENODO_TOKEN"
+BUCKET=$(curl $DEPOSIT_URL | jq --raw-output .links.bucket)
+
+if [ "$VERBOSE" -eq 1 ]; then
+    echo "Deposition ID: $DEPOSITION"
+    echo "File path: $FILEPATH"
+    echo "File name: $FILENAME"
+    echo "Bucket URL: $BUCKET"
+    echo "Upload URL: $BUCKET/$FILENAME"
+    echo "Deposit URL: $DEPOSIT_URL"
+    echo "Uploading file..."
+fi
+
+set -x
+curl --progress-bar \
+    --retry 5 \
+    --retry-delay 5 \
+    -o ${FILENAME}.upload.log \
+    -H "Authorization: Bearer $ZENODO_TOKEN" \
+    --upload-file "$FILEPATH" \
+    "$BUCKET/$FILENAME"
+
+# curl -i \
+#      -H "Authorization: Bearer $ZENODO_TOKEN" \
+#      -X POST \
+#      -F name=$FILENAME \
+#      -F file=@$FILEPATH \
+# 	https://zenodo.org/api/deposit/depositions/$DEPOSITION/files

pyproject.toml

@@ -67,6 +67,7 @@ source = ["eynollah"]
 
 [tool.ruff]
 line-length = 120
+include = ["pyproject.toml", "src/eynollah/**/*.py"]
 
 [tool.ruff.lint]
 ignore = [