From c43e2198584031fc2065e2eaceae52c305548037 Mon Sep 17 00:00:00 2001 From: kba Date: Tue, 14 Jul 2026 19:02:52 +0200 Subject: [PATCH] model packaging/uploading --- .gitignore | 5 ++ models/Makefile | 142 ++++++++++++++++++++++++++++++++ models/scripts/convert.sh | 6 ++ models/scripts/plot.py | 62 ++++++++++++++ models/scripts/run.sh | 34 ++++++++ models/scripts/zenodo_upload.sh | 71 ++++++++++++++++ 6 files changed, 320 insertions(+) create mode 100644 models/Makefile create mode 100644 models/scripts/convert.sh create mode 100644 models/scripts/plot.py create mode 100644 models/scripts/run.sh create mode 100755 models/scripts/zenodo_upload.sh diff --git a/.gitignore b/.gitignore index e3356ea..5aa9023 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,8 @@ TAGS uv.lock /ignore *.log +.env +models/reloaded +models/packages +models/models_eynollah +models/pretrained_model diff --git a/models/Makefile b/models/Makefile new file mode 100644 index 0000000..c80571a --- /dev/null +++ b/models/Makefile @@ -0,0 +1,142 @@ +SHELL = bash -e + +VERSION = v0_9_0 +MODELS_SRC = models_eynollah +MODELS_DST = reloaded/models_eynollah + + +# eynollah-main-regions-aug-rotation_20210425 +# eynollah-main-regions-aug-scaling_20210425 +# eynollah-main-regions-ensembled_20210425 +# eynollah-main-regions_20220314 +# eynollah-main-regions_20231127_672_org_ens_11_13_16_17_18 +# eynollah-tables_20210319 + +CURRENT_MODELS := +CURRENT_MODELS += eynollah-main-regions-aug-rotation_20210425 +CURRENT_MODELS += eynollah-main-regions-aug-scaling_20210425 +CURRENT_MODELS += eynollah-main-regions-ensembled_20210425 +CURRENT_MODELS += eynollah-main-regions_20220314 +CURRENT_MODELS += eynollah-main-regions_20231127_672_org_ens_11_13_16_17_18 +CURRENT_MODELS += eynollah-tables_20210319 +CURRENT_MODELS += eynollah-main-regions_20220314 +CURRENT_MODELS += model_eynollah_page_extraction_20250915 +CURRENT_MODELS += model_eynollah_reading_order_20250824 +CURRENT_MODELS += modelens_e_l_all_sp_0_1_2_3_4_171024 +CURRENT_MODELS += modelens_full_lay_1__4_3_091124 +CURRENT_MODELS += modelens_table_0t4_201124 +CURRENT_MODELS += modelens_textline_0_1__2_4_16092024 +CURRENT_MODELS += model_eynollah_ocr_cnnrnn_20250930 +CURRENT_MODELS += eynollah-binarization_20210425 +CURRENT_MODELS += eynollah-column-classifier_20210425 +CURRENT_MODELS += eynollah-enhancement_20210425 + +help: + @echo "Targets:" + @echo "" + @echo "Model conversion:" + @echo " all Convert all current models to TensorFlow SavedModel (default)" + @echo " tf[-serving] Convert all current models to TensorFlow SavedModel" + @echo " keras Convert all current models to Keras format (.keras)" + @echo " hdf5 Convert all current models to HDF5 format (.h5)" + @echo " onnx Convert all eligible current models to ONNX (.onnx)" + @echo " reload Reload selected pre-v0.8 models without Lambda layers" + @echo "" + @echo "Model reloading:" + @echo " compare Compare original and v0.8 reloaded models" + @echo "" + @echo "Packaging:" + @echo " packages Build all Zenodo distribution archives" + @echo " upload Upload all distribution archives to Zenodo" + @echo "" + @echo "Cleanup:" + @echo " clean Remove all generated files" + @echo " clean-reload Remove reloaded model directories" + @echo " clean-packages Remove generated package archives" + @echo "" + @echo "Variables:" + @echo " VERSION=$(VERSION)" + + +# tf (SavedModel format) for training +# onnx conversion for fast inference +all: tf onnx + +tf-serving: $(CURRENT_MODELS:%=$(MODELS_DST)/%) +tf: $(CURRENT_MODELS:%=$(MODELS_DST)/%) +keras: $(CURRENT_MODELS:%=$(MODELS_DST)/%.keras) +hdf5: $(CURRENT_MODELS:%=$(MODELS_DST)/%.h5) +onnx: $(CURRENT_MODELS:%=$(MODELS_DST)/%.onnx) + +# distinguish tf from tf-serving: target pattern is the same, +# so check if either is current goal, otherwise assumg tf +$(MODELS_DST)/%: FORMAT = $(or $(filter tf-serving,$(MAKECMDGOALS)), tf) +$(MODELS_DST)/%: $(MODELS_SRC)/% + $(if $(and $(filter tf-serving,$(FORMAT)),$(findstring _ocr,$@)),$(warning skipping $@: OCR CTC decoder fails in TF-Serving) : )\ + eynollah-training convert \ + $(and $(wildcard $ $(notdir $<).$(FORMAT).log 2>&1 || { cat $(notdir $<).$(FORMAT).log; false; } + +$(MODELS_DST)/%.keras: $(MODELS_SRC)/% + eynollah-training convert \ + $(and $(wildcard $ $(notdir $<).keras.log 2>&1 || { cat $(notdir $<).keras.log; false; } + +$(MODELS_DST)/%.h5: $(MODELS_SRC)/% + eynollah-training convert \ + $(and $(wildcard $ $(notdir $<).hdf5.log 2>&1 || { cat $(notdir $<).hdf5.log; false; } + +$(MODELS_DST)/%.onnx: $(MODELS_SRC)/% + $(if $(findstring _ocr,$@),$(warning skipping $@: OCR CTC decoder is buggy in ONNX) : )\ + eynollah-training convert \ + $(and $(wildcard $ $(notdir $<).onnx.log 2>&1 || { cat $(notdir $<).onnx.log; false; } + +compare: + for i in `find $(MODELS_DST) -mindepth 2`;do \ + n=$(MODELS_SRC)$${i#$(MODELS_DST)}; \ + du -bs $$n $$i ; \ + done + +clean: clean-reload clean-packages + +clean-reload: + rm -rf $(RELOADABLE_MODELS) + +clean-packages: + rm -rf $(PACKAGES_DIR) + +# NB: this works using symlinks +PACKAGES_DIR = packages +BUNDLES = inference training +CATEGORIES = layout ocr extra all + +packages: $(foreach B,$(BUNDLES),$(foreach C,$(CATEGORIES),$(PACKAGES_DIR)/models_$(B)_$(C)_$(VERSION).zip)) + +$(PACKAGES_DIR)/models_%.zip: + @mkdir -p $(PACKAGES_DIR) ;\ + bundle=$(word 1,$(subst _, ,$*)); \ + category=$(word 2,$(subst _, ,$*)); \ + echo "Packaging $$bundle/$$category to $(notdir $@)" ;\ + cd "dist/$$bundle/$$category" && \ + zip -vqr "$$OLDPWD/$@" models_eynollah + + +upload: $(foreach B,$(BUNDLES),$(foreach C,$(CATEGORIES),upload/$(B)_$(C))) + +upload/%: + @source .env ;\ + bash scripts/zenodo_upload.sh $$ZENODO_ID $(PACKAGES_DIR)/models_$*_$(VERSION).zip --verbose diff --git a/models/scripts/convert.sh b/models/scripts/convert.sh new file mode 100644 index 0000000..225785f --- /dev/null +++ b/models/scripts/convert.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +for i in */models_eynollah/*;do + rslv=$(readlink $i|sed 's,models_eynollah,reloaded/models_eynollah,'); + ln -srf $rslv $i; +done diff --git a/models/scripts/plot.py b/models/scripts/plot.py new file mode 100644 index 0000000..2543860 --- /dev/null +++ b/models/scripts/plot.py @@ -0,0 +1,62 @@ +import os +import sys +import more_itertools +import numpy as np +import argparse + +parser = argparse.ArgumentParser( + prog='plot', + description='Print metrics', + epilog='...') +parser.add_argument('stem1', default='nohup.out.eynollah-206-noautosize-keepseps-refactoring9-crop-after2') +parser.add_argument('stem2', default='eval-gt2-onlyfg-pixel-f1-v0.5-206-noautosize-keepseps-refactoring9-crop-after2') + +def main(): + args = parser.parse_args() + + prefix = os.path.dirname(sys.argv[0]) + files = [path for path in os.listdir(prefix) + if path.startswith(args.stem1) + and path.endswith('.txt')] + + for path in files: + name = path[len(args.stem1):] + metric = name.split('.')[-2] + series = name[:-(len(metric) + 5)] + path = os.path.join(prefix, path) + + if metric == 'peak-vram': + cat, peak = np.loadtxt(path, unpack=True, dtype=np.dtype([("category", "U15"), ("peak", float)])) + peak //= 1024 + peak //= 1024 + print(metric, f"{series: <52}", dict(zip(cat.tolist(), peak.tolist()))) + else: + val = np.loadtxt(path) + print(metric, f"{series: <52}", {'ยต': float(np.round(np.mean(val), 2)), + 'M': float(np.round(np.median(val), 2)), + 'min': float(np.min(val)), + 'max': float(np.max(val))}) + + + files = [path for path in os.listdir(prefix) + if path.startswith(args.stem2) + and path.endswith('.log')] + + + for path in files: + name = path[len(args.stem2):] + series = name[:-4] + path = os.path.join(prefix, path) + with open(path, 'r') as fd: + lines = fd.readlines() + result_v0_5 = {} + result_v206 = {} + for metric, v0_5, v206 in more_itertools.chunked(lines, n=3, strict=True): + metric = metric[len(prefix) + 1:-1] + result_v0_5[metric] = float(np.round(float(v0_5.strip()), 3)) + result_v206[metric] = float(np.round(float(v206.strip()), 3)) + print("pixel-f1", f"{series: <52}", result_v206) + + print("pixel-f1", "v0.5 ", result_v0_5) + + diff --git a/models/scripts/run.sh b/models/scripts/run.sh new file mode 100644 index 0000000..79cd4e6 --- /dev/null +++ b/models/scripts/run.sh @@ -0,0 +1,34 @@ +for variant in reloaded.*; do + +for dir in Korrigierte_Layout_GT/*/; do test $dir = ${dir/.} || continue; out=${dir%/}.eynollah-206-noautosize; mkdir -p $out; nohup /usr/bin +/time eynollah -D GPU0 -m $variant layout --dir_in $dir -o + $out -fl -H 1 -O || break; done + +logfile=Korrigierte_Layout_GT/nohup.out.eynollah-206-noautosize-$variant + +mv nohup.out $logfile + +fgrep peaked $logfile | sort -u | cut -d\ -f2,5 > $logfile.peak-vram.txt +fgrep initialization $logfile | sed "s/^.* (\([0-9.]*\)s)/\1/" > $logfile.secs-init.txt +fgrep "Job done in" $logfile | sed "s/^.* //;s/s$//" > $logfile.secs-jobs.txt +fgrep %CPU $logfile | sed "s/^.* \([0-9]*\)%CPU.*/\1/" > $logfile.prct-cpu.txt + +for dir in Korrigierte_Layout_GT/*.eynollah*/; do + test $dir = ${dir/-eval} || continue; + test $dir != ${dir/eynollah-206} || continue; + mets=${dir%.eynollah*}.mets.xml; + dir=$(basename $dir); + gt=${dir%.eynollah*}.gt2; + echo $mets: $dir; + ocrd-segment-evaluate -m $mets -I $gt,$dir -O $dir-eval,$dir-eval2 -P level-of-operation region -P ignore-subtype true -P only-fg true --overwrite; +done + +for mets in Korrigierte_Layout_GT/*.mets.xml; do dir=${mets%.mets.xml}; echo $dir; jq '."by-category".TextRegion."pixel-f1".avg' $dir.eynollah-v0.5-eval/*.json $dir.eynollah-206-noautosize-eval/*.json; done > Korrigierte_Layout_GT/eval-gt2-onlyfg-pixel-f1-v0.5-206-noautosize-$variant.log + + +done + + + +python Korrigierte_Layout_GT/plot.py | sort -V + diff --git a/models/scripts/zenodo_upload.sh b/models/scripts/zenodo_upload.sh new file mode 100755 index 0000000..da67298 --- /dev/null +++ b/models/scripts/zenodo_upload.sh @@ -0,0 +1,71 @@ +#!/bin/bash +# +# Upload big files to Zenodo. +# +# usage: ./zenodo_upload.sh [deposition id] [filename] [--verbose|-v] +# +# Taken from https://github.com/jhpoelen/zenodo-upload +# +# Copyright (c) 2019 Jorrit Poelen +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + +set -e + +VERBOSE=0 +if [ "$3" == "--verbose" ] || [ "$3" == "-v" ]; then + VERBOSE=1 +fi + +# strip deposition url prefix if provided; see https://github.com/jhpoelen/zenodo-upload/issues/2#issuecomment-797657717 +DEPOSITION=$( echo $1 | sed 's+^http[s]*://zenodo.org/deposit/++g' ) +FILEPATH="$2" +FILENAME=$(echo $FILEPATH | sed 's+.*/++g') +FILENAME=${FILENAME// /%20} +ZENODO_ENDPOINT=${ZENODO_ENDPOINT:-https://zenodo.org} + +DEPOSIT_URL=${ZENODO_ENDPOINT}/api/deposit/depositions/"$DEPOSITION"?access_token="$ZENODO_TOKEN" +BUCKET=$(curl $DEPOSIT_URL | jq --raw-output .links.bucket) + +if [ "$VERBOSE" -eq 1 ]; then + echo "Deposition ID: $DEPOSITION" + echo "File path: $FILEPATH" + echo "File name: $FILENAME" + echo "Bucket URL: $BUCKET" + echo "Upload URL: $BUCKET/$FILENAME" + echo "Deposit URL: $DEPOSIT_URL" + echo "Uploading file..." +fi + +set -x +curl --progress-bar \ + --retry 5 \ + --retry-delay 5 \ + -o ${FILENAME}.upload.log \ + -H "Authorization: Bearer $ZENODO_TOKEN" \ + --upload-file "$FILEPATH" \ + "$BUCKET/$FILENAME" + +# curl -i \ +# -H "Authorization: Bearer $ZENODO_TOKEN" \ +# -X POST \ +# -F name=$FILENAME \ +# -F file=@$FILEPATH \ +# https://zenodo.org/api/deposit/depositions/$DEPOSITION/files