model packaging/uploading

.gitignore

@@ -14,3 +14,8 @@ TAGS
 uv.lock
 /ignore
 *.log
+.env
+models/reloaded
+models/packages
+models/models_eynollah
+models/pretrained_model

models/Makefile

@@ -0,0 +1,142 @@
+SHELL = bash -e
+
+VERSION = v0_9_0
+MODELS_SRC = models_eynollah
+MODELS_DST = reloaded/models_eynollah
+
+
+# eynollah-main-regions-aug-rotation_20210425
+# eynollah-main-regions-aug-scaling_20210425
+# eynollah-main-regions-ensembled_20210425
+# eynollah-main-regions_20220314
+# eynollah-main-regions_20231127_672_org_ens_11_13_16_17_18
+# eynollah-tables_20210319
+
+CURRENT_MODELS := 
+CURRENT_MODELS += eynollah-main-regions-aug-rotation_20210425
+CURRENT_MODELS += eynollah-main-regions-aug-scaling_20210425
+CURRENT_MODELS += eynollah-main-regions-ensembled_20210425
+CURRENT_MODELS += eynollah-main-regions_20220314
+CURRENT_MODELS += eynollah-main-regions_20231127_672_org_ens_11_13_16_17_18
+CURRENT_MODELS += eynollah-tables_20210319
+CURRENT_MODELS += eynollah-main-regions_20220314
+CURRENT_MODELS += model_eynollah_page_extraction_20250915
+CURRENT_MODELS += model_eynollah_reading_order_20250824
+CURRENT_MODELS += modelens_e_l_all_sp_0_1_2_3_4_171024
+CURRENT_MODELS += modelens_full_lay_1__4_3_091124
+CURRENT_MODELS += modelens_table_0t4_201124
+CURRENT_MODELS += modelens_textline_0_1__2_4_16092024
+CURRENT_MODELS += model_eynollah_ocr_cnnrnn_20250930
+CURRENT_MODELS += eynollah-binarization_20210425
+CURRENT_MODELS += eynollah-column-classifier_20210425
+CURRENT_MODELS += eynollah-enhancement_20210425
+
+help:
+	@echo "Targets:"
+	@echo ""
+	@echo "Model conversion:"
+	@echo "  all          Convert all current models to TensorFlow SavedModel (default)"
+	@echo "  tf[-serving] Convert all current models to TensorFlow SavedModel"
+	@echo "  keras        Convert all current models to Keras format (.keras)"
+	@echo "  hdf5         Convert all current models to HDF5 format (.h5)"
+	@echo "  onnx         Convert all eligible current models to ONNX (.onnx)"
+	@echo "  reload       Reload selected pre-v0.8 models without Lambda layers"
+	@echo ""
+	@echo "Model reloading:"
+	@echo "  compare      Compare original and v0.8 reloaded models"
+	@echo ""
+	@echo "Packaging:"
+	@echo "  packages     Build all Zenodo distribution archives"
+	@echo "  upload       Upload all distribution archives to Zenodo"
+	@echo ""
+	@echo "Cleanup:"
+	@echo "  clean           Remove all generated files"
+	@echo "  clean-reload    Remove reloaded model directories"
+	@echo "  clean-packages  Remove generated package archives"
+	@echo ""
+	@echo "Variables:"
+	@echo "  VERSION=$(VERSION)"
+
+
+# tf (SavedModel format) for training
+# onnx conversion for fast inference
+all: tf onnx
+
+tf-serving: $(CURRENT_MODELS:%=$(MODELS_DST)/%)
+tf: $(CURRENT_MODELS:%=$(MODELS_DST)/%)
+keras: $(CURRENT_MODELS:%=$(MODELS_DST)/%.keras)
+hdf5: $(CURRENT_MODELS:%=$(MODELS_DST)/%.h5)
+onnx: $(CURRENT_MODELS:%=$(MODELS_DST)/%.onnx)
+
+# distinguish tf from tf-serving: target pattern is the same,
+# so check if either is current goal, otherwise assumg tf
+$(MODELS_DST)/%: FORMAT = $(or $(filter tf-serving,$(MAKECMDGOALS)), tf)
+$(MODELS_DST)/%: $(MODELS_SRC)/%
+	$(if $(and $(filter tf-serving,$(FORMAT)),$(findstring _ocr,$@)),$(warning skipping $@: OCR CTC decoder fails in TF-Serving) : )\
+	eynollah-training convert \
+		$(and $(wildcard $</config.json),--rebuild) \
+		--in $< \
+		--format $(FORMAT) \
+		--out $@ \
+	> $(notdir $<).$(FORMAT).log 2>&1 || { cat $(notdir $<).$(FORMAT).log; false; }
+
+$(MODELS_DST)/%.keras: $(MODELS_SRC)/%
+	eynollah-training convert \
+		$(and $(wildcard $</config.json),--rebuild) \
+		--in $< \
+		--format keras \
+		--out $@ \
+	> $(notdir $<).keras.log 2>&1 || { cat $(notdir $<).keras.log; false; }
+
+$(MODELS_DST)/%.h5: $(MODELS_SRC)/%
+	eynollah-training convert \
+		$(and $(wildcard $</config.json),--rebuild) \
+		--in $< \
+		--format hdf5 \
+		--out $@ \
+	> $(notdir $<).hdf5.log 2>&1 || { cat $(notdir $<).hdf5.log; false; }
+
+$(MODELS_DST)/%.onnx: $(MODELS_SRC)/%
+	$(if $(findstring _ocr,$@),$(warning skipping $@: OCR CTC decoder is buggy in ONNX) : )\
+	eynollah-training convert \
+		$(and $(wildcard $</config.json),--rebuild) \
+		--in $< \
+		--format onnx \
+		--out $@ \
+	> $(notdir $<).onnx.log 2>&1 || { cat $(notdir $<).onnx.log; false; }
+
+compare: 
+	for i in `find $(MODELS_DST) -mindepth 2`;do \
+	 	n=$(MODELS_SRC)$${i#$(MODELS_DST)}; \
+		du -bs $$n $$i ; \
+	done
+
+clean: clean-reload clean-packages
+
+clean-reload:
+	rm -rf $(RELOADABLE_MODELS)
+
+clean-packages:
+	rm -rf $(PACKAGES_DIR)
+
+# NB: this works using symlinks
+PACKAGES_DIR = packages
+BUNDLES = inference training
+CATEGORIES = layout ocr extra all
+
+packages: $(foreach B,$(BUNDLES),$(foreach C,$(CATEGORIES),$(PACKAGES_DIR)/models_$(B)_$(C)_$(VERSION).zip))
+
+$(PACKAGES_DIR)/models_%.zip:
+	@mkdir -p $(PACKAGES_DIR) ;\
+	bundle=$(word 1,$(subst _, ,$*)); \
+	category=$(word 2,$(subst _, ,$*)); \
+	echo "Packaging $$bundle/$$category to $(notdir $@)" ;\
+	cd "dist/$$bundle/$$category" && \
+	zip -vqr "$$OLDPWD/$@" models_eynollah
+
+
+upload: $(foreach B,$(BUNDLES),$(foreach C,$(CATEGORIES),upload/$(B)_$(C)))
+
+upload/%:
+	@source .env ;\
+	bash scripts/zenodo_upload.sh $$ZENODO_ID $(PACKAGES_DIR)/models_$*_$(VERSION).zip --verbose

models/scripts/convert.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+for i in */models_eynollah/*;do
+	rslv=$(readlink $i|sed 's,models_eynollah,reloaded/models_eynollah,');
+	ln -srf $rslv $i;
+done

models/scripts/plot.py

@@ -0,0 +1,62 @@
+import os
+import sys
+import more_itertools
+import numpy as np
+import argparse
+
+parser = argparse.ArgumentParser(
+                    prog='plot',
+                    description='Print metrics',
+                    epilog='...')
+parser.add_argument('stem1', default='nohup.out.eynollah-206-noautosize-keepseps-refactoring9-crop-after2')
+parser.add_argument('stem2', default='eval-gt2-onlyfg-pixel-f1-v0.5-206-noautosize-keepseps-refactoring9-crop-after2')
+
+def main():
+    args = parser.parse_args()
+
+    prefix = os.path.dirname(sys.argv[0])
+    files = [path for path in os.listdir(prefix)
+             if path.startswith(args.stem1)
+             and path.endswith('.txt')]
+
+    for path in files:
+        name = path[len(args.stem1):]
+        metric = name.split('.')[-2]
+        series = name[:-(len(metric) + 5)]
+        path = os.path.join(prefix, path)
+
+        if metric == 'peak-vram':
+            cat, peak = np.loadtxt(path, unpack=True, dtype=np.dtype([("category", "U15"), ("peak", float)]))
+            peak //= 1024
+            peak //= 1024
+            print(metric, f"{series: <52}", dict(zip(cat.tolist(), peak.tolist())))
+        else:
+            val = np.loadtxt(path)
+            print(metric, f"{series: <52}", {'µ': float(np.round(np.mean(val), 2)),
+                                             'M': float(np.round(np.median(val), 2)),
+                                             'min': float(np.min(val)),
+                                             'max': float(np.max(val))})
+
+
+    files = [path for path in os.listdir(prefix)
+             if path.startswith(args.stem2)
+             and path.endswith('.log')]
+
+
+    for path in files:
+        name = path[len(args.stem2):]
+        series = name[:-4]
+        path = os.path.join(prefix, path)
+        with open(path, 'r') as fd:
+            lines = fd.readlines()
+        result_v0_5 = {}
+        result_v206 = {}
+        for metric, v0_5, v206 in more_itertools.chunked(lines, n=3, strict=True):
+            metric = metric[len(prefix) + 1:-1]
+            result_v0_5[metric] = float(np.round(float(v0_5.strip()), 3))
+            result_v206[metric] = float(np.round(float(v206.strip()), 3))
+        print("pixel-f1", f"{series: <52}", result_v206)
+
+    print("pixel-f1", "v0.5                                       ", result_v0_5)
+
+

models/scripts/run.sh

@@ -0,0 +1,34 @@
+for variant in reloaded.*; do
+
+for dir in Korrigierte_Layout_GT/*/; do test $dir = ${dir/.} || continue; out=${dir%/}.eynollah-206-noautosize; mkdir -p $out; nohup /usr/bin
+/time eynollah  -D GPU0 -m $variant layout --dir_in $dir -o
+ $out -fl -H 1 -O || break; done
+
+logfile=Korrigierte_Layout_GT/nohup.out.eynollah-206-noautosize-$variant
+
+mv nohup.out $logfile
+
+fgrep peaked $logfile | sort -u | cut -d\  -f2,5 > $logfile.peak-vram.txt
+fgrep initialization $logfile | sed "s/^.* (\([0-9.]*\)s)/\1/" > $logfile.secs-init.txt
+fgrep "Job done in" $logfile | sed "s/^.* //;s/s$//" > $logfile.secs-jobs.txt
+fgrep %CPU $logfile | sed "s/^.* \([0-9]*\)%CPU.*/\1/" > $logfile.prct-cpu.txt
+
+for dir in Korrigierte_Layout_GT/*.eynollah*/; do
+	test $dir = ${dir/-eval} || continue;
+	test $dir != ${dir/eynollah-206} || continue;
+	mets=${dir%.eynollah*}.mets.xml;
+	dir=$(basename $dir);
+	gt=${dir%.eynollah*}.gt2;
+	echo $mets: $dir;
+	ocrd-segment-evaluate -m $mets -I $gt,$dir -O $dir-eval,$dir-eval2 -P level-of-operation region -P ignore-subtype true -P only-fg true --overwrite;
+done
+
+for mets in Korrigierte_Layout_GT/*.mets.xml; do dir=${mets%.mets.xml}; echo $dir; jq '."by-category".TextRegion."pixel-f1".avg' $dir.eynollah-v0.5-eval/*.json $dir.eynollah-206-noautosize-eval/*.json; done > Korrigierte_Layout_GT/eval-gt2-onlyfg-pixel-f1-v0.5-206-noautosize-$variant.log
+
+
+done
+
+
+
+python Korrigierte_Layout_GT/plot.py | sort -V
+

models/scripts/zenodo_upload.sh

@@ -0,0 +1,71 @@
+#!/bin/bash
+#
+# Upload big files to Zenodo.
+#
+# usage: ./zenodo_upload.sh [deposition id] [filename] [--verbose|-v]
+#
+# Taken from https://github.com/jhpoelen/zenodo-upload
+# 
+# Copyright (c) 2019 Jorrit Poelen
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+#
+
+set -e
+
+VERBOSE=0
+if [ "$3" == "--verbose" ] || [ "$3" == "-v" ]; then
+    VERBOSE=1
+fi
+
+# strip deposition url prefix if provided; see https://github.com/jhpoelen/zenodo-upload/issues/2#issuecomment-797657717
+DEPOSITION=$( echo $1 | sed 's+^http[s]*://zenodo.org/deposit/++g' )
+FILEPATH="$2"
+FILENAME=$(echo $FILEPATH | sed 's+.*/++g')
+FILENAME=${FILENAME// /%20}
+ZENODO_ENDPOINT=${ZENODO_ENDPOINT:-https://zenodo.org}
+
+DEPOSIT_URL=${ZENODO_ENDPOINT}/api/deposit/depositions/"$DEPOSITION"?access_token="$ZENODO_TOKEN"
+BUCKET=$(curl $DEPOSIT_URL | jq --raw-output .links.bucket)
+
+if [ "$VERBOSE" -eq 1 ]; then
+    echo "Deposition ID: $DEPOSITION"
+    echo "File path: $FILEPATH"
+    echo "File name: $FILENAME"
+    echo "Bucket URL: $BUCKET"
+    echo "Upload URL: $BUCKET/$FILENAME"
+    echo "Deposit URL: $DEPOSIT_URL"
+    echo "Uploading file..."
+fi
+
+set -x
+curl --progress-bar \
+    --retry 5 \
+    --retry-delay 5 \
+    -o ${FILENAME}.upload.log \
+    -H "Authorization: Bearer $ZENODO_TOKEN" \
+    --upload-file "$FILEPATH" \
+    "$BUCKET/$FILENAME"
+
+# curl -i \
+#      -H "Authorization: Bearer $ZENODO_TOKEN" \
+#      -X POST \
+#      -F name=$FILENAME \
+#      -F file=@$FILEPATH \
+# 	https://zenodo.org/api/deposit/depositions/$DEPOSITION/files