mirror of
https://github.com/qurator-spk/eynollah.git
synced 2026-07-25 21:19:18 +02:00
model packaging/uploading
This commit is contained in:
kba
parent
83fca95914
commit
1972e9975b
6 changed files with 310 additions and 0 deletions
5
.gitignore
vendored
5
.gitignore
vendored
|
|
@ -14,3 +14,8 @@ TAGS
|
|||
uv.lock
|
||||
/ignore
|
||||
*.log
|
||||
.env
|
||||
models/reloaded
|
||||
models/packages
|
||||
models/models_eynollah
|
||||
models/pretrained_model
|
||||
|
|
|
|||
144
models/Makefile
Normal file
144
models/Makefile
Normal file
|
|
@ -0,0 +1,144 @@
|
|||
SHELL = bash -e
|
||||
|
||||
VERSION = v0_9_0
|
||||
MODELS_SRC = models_eynollah
|
||||
MODELS_DST = reloaded/models_eynollah
|
||||
|
||||
|
||||
# eynollah-main-regions-aug-rotation_20210425
|
||||
# eynollah-main-regions-aug-scaling_20210425
|
||||
# eynollah-main-regions-ensembled_20210425
|
||||
# eynollah-main-regions_20220314
|
||||
# eynollah-main-regions_20231127_672_org_ens_11_13_16_17_18
|
||||
# eynollah-tables_20210319
|
||||
|
||||
CURRENT_MODELS :=
|
||||
CURRENT_MODELS += eynollah-main-regions-aug-rotation_20210425
|
||||
CURRENT_MODELS += eynollah-main-regions-aug-scaling_20210425
|
||||
CURRENT_MODELS += eynollah-main-regions-ensembled_20210425
|
||||
CURRENT_MODELS += eynollah-main-regions_20220314
|
||||
CURRENT_MODELS += eynollah-main-regions_20231127_672_org_ens_11_13_16_17_18
|
||||
CURRENT_MODELS += eynollah-tables_20210319
|
||||
CURRENT_MODELS += eynollah-main-regions_20220314
|
||||
CURRENT_MODELS += model_eynollah_page_extraction_20250915
|
||||
CURRENT_MODELS += model_eynollah_reading_order_20250824
|
||||
CURRENT_MODELS += modelens_e_l_all_sp_0_1_2_3_4_171024
|
||||
CURRENT_MODELS += modelens_full_lay_1__4_3_091124
|
||||
CURRENT_MODELS += modelens_table_0t4_201124
|
||||
CURRENT_MODELS += modelens_textline_0_1__2_4_16092024
|
||||
CURRENT_MODELS += model_eynollah_ocr_cnnrnn_20250930
|
||||
CURRENT_MODELS += eynollah-binarization_20210425
|
||||
CURRENT_MODELS += eynollah-column-classifier_20210425
|
||||
CURRENT_MODELS += eynollah-enhancement_20210425
|
||||
|
||||
help:
|
||||
@echo "Targets:"
|
||||
@echo ""
|
||||
@echo "Model conversion:"
|
||||
@echo " all Convert all current models to TensorFlow SavedModel (default)"
|
||||
@echo " tf[-serving] Convert all current models to TensorFlow SavedModel"
|
||||
@echo " keras Convert all current models to Keras format (.keras)"
|
||||
@echo " hdf5 Convert all current models to HDF5 format (.h5)"
|
||||
@echo " onnx Convert all eligible current models to ONNX (.onnx)"
|
||||
@echo " reload Reload selected pre-v0.8 models without Lambda layers
|
||||
@echo ""
|
||||
@echo "Model reloading:"
|
||||
@echo " compare Compare original and v0.8 reloaded models
|
||||
@echo ""
|
||||
@echo "Packaging:"
|
||||
@echo " packages Build all Zenodo distribution archives"
|
||||
@echo " upload Upload all distribution archives to Zenodo"
|
||||
@echo ""
|
||||
@echo "Cleanup:"
|
||||
@echo " clean Remove all generated files"
|
||||
@echo " clean-reload Remove reloaded model directories"
|
||||
@echo " clean-packages Remove generated package archives"
|
||||
@echo ""
|
||||
@echo "Variables:"
|
||||
@echo " VERSION=$(VERSION)"
|
||||
|
||||
|
||||
# tf (SavedModel format) for training
|
||||
# onnx conversion for fast inference
|
||||
all: tf onnx
|
||||
|
||||
tf-serving: $(CURRENT_MODELS:%=$(MODELS_DST)/%)
|
||||
tf: $(CURRENT_MODELS:%=$(MODELS_DST)/%)
|
||||
keras: $(CURRENT_MODELS:%=$(MODELS_DST)/%.keras)
|
||||
hdf5: $(CURRENT_MODELS:%=$(MODELS_DST)/%.h5)
|
||||
onnx: $(CURRENT_MODELS:%=$(MODELS_DST)/%.onnx)
|
||||
|
||||
# distinguish tf from tf-serving: target pattern is the same,
|
||||
# so check if either is current goal, otherwise assumg tf
|
||||
$(MODELS_DST)/%: FORMAT = $(or $(filter tf-serving,$(MAKECMDGOALS)), tf)
|
||||
$(MODELS_DST)/%: $(MODELS_SRC)/%
|
||||
$(if $(and $(filter tf-serving,$(FORMAT)),$(findstring _ocr,$@)),$(warning skipping $@: OCR CTC decoder fails in TF-Serving) : )\
|
||||
eynollah-training convert \
|
||||
$(and $(wildcard $</config.json),--rebuild) \
|
||||
--in $< \
|
||||
--format $(FORMAT) \
|
||||
--out $@ \
|
||||
> $(notdir $<).$(FORMAT).log 2>&1 || { cat $(notdir $<).$(FORMAT).log; false; }
|
||||
|
||||
$(MODELS_DST)/%.keras: $(MODELS_SRC)/%
|
||||
eynollah-training convert \
|
||||
$(and $(wildcard $</config.json),--rebuild) \
|
||||
--in $< \
|
||||
--format keras \
|
||||
--out $@ \
|
||||
> $(notdir $<).keras.log 2>&1 || { cat $(notdir $<).keras.log; false; }
|
||||
|
||||
$(MODELS_DST)/%.h5: $(MODELS_SRC)/%
|
||||
eynollah-training convert \
|
||||
$(and $(wildcard $</config.json),--rebuild) \
|
||||
--in $< \
|
||||
--format hdf5 \
|
||||
--out $@ \
|
||||
> $(notdir $<).hdf5.log 2>&1 || { cat $(notdir $<).hdf5.log; false; }
|
||||
|
||||
$(MODELS_DST)/%.onnx: $(MODELS_SRC)/%
|
||||
$(if $(findstring _ocr,$@),$(warning skipping $@: OCR CTC decoder is buggy in ONNX) : )\
|
||||
eynollah-training convert \
|
||||
$(and $(wildcard $</config.json),--rebuild) \
|
||||
--in $< \
|
||||
--format onnx \
|
||||
--out $@ \
|
||||
> $(notdir $<).onnx.log 2>&1 || { cat $(notdir $<).onnx.log; false; }
|
||||
|
||||
compare:
|
||||
for i in `find $(MODELS_DST) -mindepth 2`;do \
|
||||
n=$(MODELS_SRC)$${i#$(MODELS_DST)}; \
|
||||
du -bs $$n $$i ; \
|
||||
done
|
||||
|
||||
clean: clean-reload clean-packages
|
||||
|
||||
clean-reload:
|
||||
rm -rf $(RELOADABLE_MODELS)
|
||||
|
||||
clean-packages:
|
||||
rm -rf $(PACKAGES_DIR)
|
||||
|
||||
# NB: this works using symlinks
|
||||
PACKAGES_DIR = packages
|
||||
BUNDLES = inference training
|
||||
CATEGORIES = layout ocr extra all
|
||||
|
||||
packages: $(foreach B,$(BUNDLES),$(foreach C,$(CATEGORIES),$(PACKAGES_DIR)/models_$(B)_$(C)_$(VERSION).zip))
|
||||
|
||||
$(PACKAGES_DIR)/models_%.zip:
|
||||
@mkdir -p $(PACKAGES_DIR) ;\
|
||||
bundle=$(word 1,$(subst _, ,$*)); \
|
||||
category=$(word 2,$(subst _, ,$*)); \
|
||||
echo "Packaging $$bundle/$$category to $(notdir $@)" ;\
|
||||
cd "dist/$$bundle/$$category" && \
|
||||
zip -vqr "$$OLDPWD/$@" models_eynollah
|
||||
|
||||
|
||||
upload: $(foreach B,$(BUNDLES),$(foreach C,$(CATEGORIES),upload/$(B)_$(C)))
|
||||
|
||||
upload/%:
|
||||
@source .env ;\
|
||||
bundle=$(word 1,$(subst _, ,$*)); \
|
||||
category=$(word 2,$(subst _, ,$*)); \
|
||||
bash scripts/zenodo_upload.sh $$ZENODO_ID $(PACKAGES_DIR)/models_$(B)_$(C)_$(VERSION).zip --verbose
|
||||
6
models/scripts/convert.sh
Normal file
6
models/scripts/convert.sh
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
for i in */models_eynollah/*;do
|
||||
rslv=$(readlink $i|sed 's,models_eynollah,reloaded/models_eynollah,');
|
||||
ln -srf $rslv $i;
|
||||
done
|
||||
62
models/scripts/plot.py
Normal file
62
models/scripts/plot.py
Normal file
|
|
@ -0,0 +1,62 @@
|
|||
import os
|
||||
import sys
|
||||
import more_itertools
|
||||
import numpy as np
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
prog='plot',
|
||||
description='Print metrics',
|
||||
epilog='...')
|
||||
parser.add_argument('stem1', default='nohup.out.eynollah-206-noautosize-keepseps-refactoring9-crop-after2')
|
||||
parser.add_argument('stem2', default='eval-gt2-onlyfg-pixel-f1-v0.5-206-noautosize-keepseps-refactoring9-crop-after2')
|
||||
|
||||
def main():
|
||||
args = parser.parse_args()
|
||||
|
||||
prefix = os.path.dirname(sys.argv[0])
|
||||
files = [path for path in os.listdir(prefix)
|
||||
if path.startswith(args.stem1)
|
||||
and path.endswith('.txt')]
|
||||
|
||||
for path in files:
|
||||
name = path[len(args.stem1):]
|
||||
metric = name.split('.')[-2]
|
||||
series = name[:-(len(metric) + 5)]
|
||||
path = os.path.join(prefix, path)
|
||||
|
||||
if metric == 'peak-vram':
|
||||
cat, peak = np.loadtxt(path, unpack=True, dtype=np.dtype([("category", "U15"), ("peak", float)]))
|
||||
peak //= 1024
|
||||
peak //= 1024
|
||||
print(metric, f"{series: <52}", dict(zip(cat.tolist(), peak.tolist())))
|
||||
else:
|
||||
val = np.loadtxt(path)
|
||||
print(metric, f"{series: <52}", {'µ': float(np.round(np.mean(val), 2)),
|
||||
'M': float(np.round(np.median(val), 2)),
|
||||
'min': float(np.min(val)),
|
||||
'max': float(np.max(val))})
|
||||
|
||||
|
||||
files = [path for path in os.listdir(prefix)
|
||||
if path.startswith(args.stem2)
|
||||
and path.endswith('.log')]
|
||||
|
||||
|
||||
for path in files:
|
||||
name = path[len(args.stem2):]
|
||||
series = name[:-4]
|
||||
path = os.path.join(prefix, path)
|
||||
with open(path, 'r') as fd:
|
||||
lines = fd.readlines()
|
||||
result_v0_5 = {}
|
||||
result_v206 = {}
|
||||
for metric, v0_5, v206 in more_itertools.chunked(lines, n=3, strict=True):
|
||||
metric = metric[len(prefix) + 1:-1]
|
||||
result_v0_5[metric] = float(np.round(float(v0_5.strip()), 3))
|
||||
result_v206[metric] = float(np.round(float(v206.strip()), 3))
|
||||
print("pixel-f1", f"{series: <52}", result_v206)
|
||||
|
||||
print("pixel-f1", "v0.5 ", result_v0_5)
|
||||
|
||||
|
||||
34
models/scripts/run.sh
Normal file
34
models/scripts/run.sh
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
for variant in reloaded.*; do
|
||||
|
||||
for dir in Korrigierte_Layout_GT/*/; do test $dir = ${dir/.} || continue; out=${dir%/}.eynollah-206-noautosize; mkdir -p $out; nohup /usr/bin
|
||||
/time eynollah -D GPU0 -m $variant layout --dir_in $dir -o
|
||||
$out -fl -H 1 -O || break; done
|
||||
|
||||
logfile=Korrigierte_Layout_GT/nohup.out.eynollah-206-noautosize-$variant
|
||||
|
||||
mv nohup.out $logfile
|
||||
|
||||
fgrep peaked $logfile | sort -u | cut -d\ -f2,5 > $logfile.peak-vram.txt
|
||||
fgrep initialization $logfile | sed "s/^.* (\([0-9.]*\)s)/\1/" > $logfile.secs-init.txt
|
||||
fgrep "Job done in" $logfile | sed "s/^.* //;s/s$//" > $logfile.secs-jobs.txt
|
||||
fgrep %CPU $logfile | sed "s/^.* \([0-9]*\)%CPU.*/\1/" > $logfile.prct-cpu.txt
|
||||
|
||||
for dir in Korrigierte_Layout_GT/*.eynollah*/; do
|
||||
test $dir = ${dir/-eval} || continue;
|
||||
test $dir != ${dir/eynollah-206} || continue;
|
||||
mets=${dir%.eynollah*}.mets.xml;
|
||||
dir=$(basename $dir);
|
||||
gt=${dir%.eynollah*}.gt2;
|
||||
echo $mets: $dir;
|
||||
ocrd-segment-evaluate -m $mets -I $gt,$dir -O $dir-eval,$dir-eval2 -P level-of-operation region -P ignore-subtype true -P only-fg true --overwrite;
|
||||
done
|
||||
|
||||
for mets in Korrigierte_Layout_GT/*.mets.xml; do dir=${mets%.mets.xml}; echo $dir; jq '."by-category".TextRegion."pixel-f1".avg' $dir.eynollah-v0.5-eval/*.json $dir.eynollah-206-noautosize-eval/*.json; done > Korrigierte_Layout_GT/eval-gt2-onlyfg-pixel-f1-v0.5-206-noautosize-$variant.log
|
||||
|
||||
|
||||
done
|
||||
|
||||
|
||||
|
||||
python Korrigierte_Layout_GT/plot.py | sort -V
|
||||
|
||||
59
models/scripts/zenodo_upload.sh
Executable file
59
models/scripts/zenodo_upload.sh
Executable file
|
|
@ -0,0 +1,59 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Upload big files to Zenodo.
|
||||
#
|
||||
# usage: ./zenodo_upload.sh [deposition id] [filename] [--verbose|-v]
|
||||
#
|
||||
# Taken from https://github.com/jhpoelen/zenodo-upload
|
||||
#
|
||||
# Copyright (c) 2019 Jorrit Poelen
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in all
|
||||
# copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
#
|
||||
|
||||
set -xe
|
||||
|
||||
VERBOSE=0
|
||||
if [ "$3" == "--verbose" ] || [ "$3" == "-v" ]; then
|
||||
VERBOSE=1
|
||||
fi
|
||||
|
||||
# strip deposition url prefix if provided; see https://github.com/jhpoelen/zenodo-upload/issues/2#issuecomment-797657717
|
||||
DEPOSITION=$( echo $1 | sed 's+^http[s]*://zenodo.org/deposit/++g' )
|
||||
FILEPATH="$2"
|
||||
FILENAME=$(echo $FILEPATH | sed 's+.*/++g')
|
||||
FILENAME=${FILENAME// /%20}
|
||||
ZENODO_ENDPOINT=${ZENODO_ENDPOINT:-https://zenodo.org}
|
||||
|
||||
BUCKET=$(curl ${ZENODO_ENDPOINT}/api/deposit/depositions/"$DEPOSITION"?access_token="$ZENODO_TOKEN" | jq --raw-output .links.bucket)
|
||||
|
||||
if [ "$VERBOSE" -eq 1 ]; then
|
||||
echo "Deposition ID: $DEPOSITION"
|
||||
echo "File path: $FILEPATH"
|
||||
echo "File name: $FILENAME"
|
||||
echo "Bucket URL: $BUCKET"
|
||||
echo "Uploading file..."
|
||||
fi
|
||||
|
||||
curl --progress-bar \
|
||||
--retry 5 \
|
||||
--retry-delay 5 \
|
||||
-o /dev/null \
|
||||
--upload-file "$FILEPATH" \
|
||||
$BUCKET/"$FILENAME"?access_token="$ZENODO_TOKEN"
|
||||
Loading…
Add table
Add a link
Reference in a new issue