qurator-spk/eynollah
1
0
Fork 0
mirror of https://github.com/qurator-spk/eynollah.git synced 2026-05-13 01:13:54 +02:00
Code Issues Projects Releases Wiki Activity

Merge pull request #8 from bertsky/ro-fixes-training-reload

Browse source 
training: reload models
This commit is contained in:
Robert Sachunsky 2026-05-08 18:46:43 +02:00 • committed by GitHub
commit daf0c90d6e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 142 additions and 61 deletions
Download patch file Download diff file Expand all files Collapse all files

1
requirements.txt
View file

@ -6,5 +6,4 @@ tensorflow
tf-keras # avoid keras 3 (also needs TF_USE_LEGACY_KERAS=1)
numba <= 0.58.1
scikit-image
biopython
tabulate

1
src/eynollah/training/models.py
View file

@ -15,7 +15,6 @@ from tensorflow.keras.layers import (
Embedding,
Flatten,
Input,
Lambda,
Layer,
LayerNormalization,
LSTM,

48
src/eynollah/training/reload-models-v0.8.mk Normal file
View file

@ -0,0 +1,48 @@
SHELL = bash -e
MODELS_SRC = models_eynollah
MODELS_DST = reloaded/models_eynollah
# $(MODELS_DST)/eynollah-binarization_20210425 \
# $(MODELS_DST)/eynollah-column-classifier_20210425 \
# $(MODELS_DST)/eynollah-enhancement_20210425 \
# $(MODELS_DST)/eynollah-main-regions-aug-rotation_20210425 \
# $(MODELS_DST)/eynollah-main-regions-aug-scaling_20210425 \
# $(MODELS_DST)/eynollah-main-regions-ensembled_20210425 \
# $(MODELS_DST)/eynollah-main-regions_20220314 \
# $(MODELS_DST)/eynollah-main-regions_20231127_672_org_ens_11_13_16_17_18 \
# $(MODELS_DST)/eynollah-tables_20210319 \
# $(MODELS_DST)/model_eynollah_ocr_cnnrnn_20250930 \
RELOADABLE_MODELS = \
$(MODELS_DST)/model_eynollah_page_extraction_20250915 \
$(MODELS_DST)/model_eynollah_reading_order_20250824 \
$(MODELS_DST)/modelens_e_l_all_sp_0_1_2_3_4_171024 \
$(MODELS_DST)/modelens_full_lay_1__4_3_091124 \
$(MODELS_DST)/modelens_table_0t4_201124 \
$(MODELS_DST)/modelens_textline_0_1__2_4_16092024
all: $(RELOADABLE_MODELS)
$(MODELS_DST)/%: $(MODELS_SRC)/%
mkdir -p $@
test -e $</config.json || exit 1
eynollah-training train --force \
with $</config.json \
reload_weights=True \
continue_training=False \
dir_output=$(dir $@) \
dir_of_start_model=$< \
2>&1 | tee $(notdir $<).log
cp $</config.json $@/config.json
compare:
for i in `find $(MODELS_DST) -mindepth 2`;do \
n=$(MODELS_SRC)$${i#$(MODELS_DST)}; \
du -bs $$n $$i ; \
done
clear:
rm -rf $(MODELS_DST)

140
src/eynollah/training/train.py
View file

@ -355,6 +355,7 @@ def config_params():
dir_output = None # Directory where the augmented training data and the model checkpoints will be saved.
pretraining = False # Set to true to (down)load pretrained weights of ResNet50 encoder.
save_interval = None # frequency for writing model checkpoints (positive integer for number of batches saved under "model_step_{batch:04d}", otherwise epoch saved under "model_{epoch:02d}")
reload_weights = False # Set true to build new model from config, load weights from dir_of_start_model, save under dir_output and exit.
continue_training = False # Whether to continue training an existing model.
if continue_training:
dir_of_start_model = '' # Directory of model checkpoint to load to continue training. (E.g. if you already trained for 3 epochs, set "dir_of_start_model=dir_output/model_03".)
@ -378,6 +379,7 @@ def run(_config,
weight_decay,
learning_rate,
continue_training,
reload_weights,
save_interval,
augmentation,
# dependent config keys need a default,
@ -452,43 +454,6 @@ def run(_config,
dir_flow_eval_imgs = os.path.join(dir_eval_flowing, 'images')
dir_flow_eval_labels = os.path.join(dir_eval_flowing, 'labels')
if not data_is_provided:
# first create a directory in output for both training and evaluations
# in order to flow data from these directories.
if os.path.isdir(dir_train_flowing):
os.system('rm -rf ' + dir_train_flowing)
os.makedirs(dir_train_flowing)
if os.path.isdir(dir_eval_flowing):
os.system('rm -rf ' + dir_eval_flowing)
os.makedirs(dir_eval_flowing)
os.mkdir(dir_flow_train_imgs)
os.mkdir(dir_flow_train_labels)
os.mkdir(dir_flow_eval_imgs)
os.mkdir(dir_flow_eval_labels)
# writing patches into a sub-folder in order to be flowed from directory.
def gen(dir_img, dir_lab, dir_flow_imgs, dir_flow_labs, augmentation=True):
indexer = 0
for img, lab in tqdm(preprocess_imgs(_config,
dir_img,
dir_lab,
augmentation=augmentation),
desc="data_is_provided"):
fname = 'img_%d.png' % indexer
cv2.imwrite(os.path.join(dir_flow_imgs, fname), img)
cv2.imwrite(os.path.join(dir_flow_labs, fname), lab)
indexer += 1
gen(*get_dirs_or_files(dir_train),
dir_flow_train_imgs,
dir_flow_train_labels)
gen(*get_dirs_or_files(dir_eval),
dir_flow_eval_imgs,
dir_flow_eval_labels,
augmentation=False)
if weighted_loss:
weights = np.zeros(n_classes)
if data_is_provided:
@ -594,6 +559,52 @@ def run(_config,
optimizer=Adam(learning_rate=learning_rate),
metrics=metrics)
if reload_weights:
model.load_weights(dir_of_start_model).assert_existing_objects_matched().expect_partial()
dir_save = os.path.join(dir_output, os.path.basename(os.path.normpath(dir_of_start_model)))
model.save(dir_save, include_optimizer=False)
with open(os.path.join(dir_save, "config.json"), "w") as fp:
json.dump(_config, fp) # encode dict into JSON
_log.info("reloaded model from %s to %s", dir_of_start_model, dir_save)
return
if not data_is_provided:
# first create a directory in output for both training and evaluations
# in order to flow data from these directories.
if os.path.isdir(dir_train_flowing):
os.system('rm -rf ' + dir_train_flowing)
os.makedirs(dir_train_flowing)
if os.path.isdir(dir_eval_flowing):
os.system('rm -rf ' + dir_eval_flowing)
os.makedirs(dir_eval_flowing)
os.mkdir(dir_flow_train_imgs)
os.mkdir(dir_flow_train_labels)
os.mkdir(dir_flow_eval_imgs)
os.mkdir(dir_flow_eval_labels)
# writing patches into a sub-folder in order to be flowed from directory.
def gen(dir_img, dir_lab, dir_flow_imgs, dir_flow_labs, augmentation=True):
indexer = 0
for img, lab in tqdm(preprocess_imgs(_config,
dir_img,
dir_lab,
augmentation=augmentation),
desc="data_is_provided"):
fname = 'img_%d.png' % indexer
cv2.imwrite(os.path.join(dir_flow_imgs, fname), img)
cv2.imwrite(os.path.join(dir_flow_labs, fname), lab)
indexer += 1
gen(*get_dirs_or_files(dir_train),
dir_flow_train_imgs,
dir_flow_train_labels)
gen(*get_dirs_or_files(dir_eval),
dir_flow_eval_imgs,
dir_flow_eval_labels,
augmentation=False)
def _to_cv2float(img):
# rgb→bgr and uint8→float, as expected by Eynollah models
return tf.cast(tf.reverse(img, [-1]), tf.float32) / 255
@ -701,8 +712,25 @@ def run(_config,
image_width=input_width,
n_classes=n_classes,
max_seq=max_len)
#initial_learning_rate = 1e-4
#decay_steps = int (n_epochs * ( len_dataset / n_batch ))
#alpha = 0.01
#lr_schedule = 1e-4
#tf.keras.optimizers.schedules.CosineDecay(initial_learning_rate, decay_steps, alpha)
opt = Adam(learning_rate=learning_rate)
model.compile(optimizer=opt) # rs: loss seems to be (ctc_batch_cost) in last layer
#print(model.summary())
if reload_weights:
model.load_weights(dir_of_start_model).assert_existing_objects_matched().expect_partial()
dir_save = os.path.join(dir_output, os.path.basename(os.path.normpath(dir_of_start_model)))
model.save(dir_save, include_optimizer=False)
with open(os.path.join(dir_save, "config.json"), "w") as fp:
json.dump(_config, fp) # encode dict into JSON
_log.info("reloaded model from %s to %s", dir_of_start_model, dir_save)
return
# todo: use Dataset.map() on Dataset.list_files()
def get_dataset(dir_img, dir_lab):
def gen():
@ -726,14 +754,6 @@ def run(_config,
train_ds = get_dataset(*get_dirs_or_files(dir_train))
valdn_ds = get_dataset(*get_dirs_or_files(dir_eval))
#initial_learning_rate = 1e-4
#decay_steps = int (n_epochs * ( len_dataset / n_batch ))
#alpha = 0.01
#lr_schedule = 1e-4
#tf.keras.optimizers.schedules.CosineDecay(initial_learning_rate, decay_steps, alpha)
opt = Adam(learning_rate=learning_rate)
model.compile(optimizer=opt) # rs: loss seems to be (ctc_batch_cost) in last layer
callbacks = [TensorBoard(os.path.join(dir_output, 'logs'), write_graph=False),
EarlyStopping(verbose=1, patience=3, restore_best_weights=False, start_from_epoch=3),
SaveWeightsAfterSteps(0, dir_output, _config)]
@ -762,6 +782,15 @@ def run(_config,
optimizer=Adam(learning_rate=0.001), # rs: why not learning_rate?
metrics=['accuracy', F1Score(average='macro', name='f1')])
if reload_weights:
model.load_weights(dir_of_start_model).assert_existing_objects_matched().expect_partial()
dir_save = os.path.join(dir_output, os.path.basename(os.path.normpath(dir_of_start_model)))
model.save(dir_save, include_optimizer=False)
with open(os.path.join(dir_save, "config.json"), "w") as fp:
json.dump(_config, fp) # encode dict into JSON
_log.info("reloaded model from %s to %s", dir_of_start_model, dir_save)
return
list_classes = list(classification_classes_name.values())
data_args = dict(label_mode="categorical",
class_names=list_classes,
@ -805,6 +834,21 @@ def run(_config,
weight_decay,
pretraining)
#f1score_tot = [0]
model.compile(loss="binary_crossentropy",
#optimizer=SGD(learning_rate=0.01, momentum=0.9),
optimizer=Adam(learning_rate=0.0001), # rs: why not learning_rate?
metrics=['accuracy'])
if reload_weights:
model.load_weights(dir_of_start_model).assert_existing_objects_matched().expect_partial()
dir_save = os.path.join(dir_output, os.path.basename(os.path.normpath(dir_of_start_model)))
model.save(dir_save, include_optimizer=False)
with open(os.path.join(dir_save, "config.json"), "w") as fp:
json.dump(_config, fp) # encode dict into JSON
_log.info("reloaded model from %s to %s", dir_of_start_model, dir_save)
return
dir_flow_train_imgs = os.path.join(dir_train, 'images')
dir_flow_train_labels = os.path.join(dir_train, 'labels')
@ -815,12 +859,6 @@ def run(_config,
num_rows = len(classes)
#ls_test = os.listdir(dir_flow_train_labels)
#f1score_tot = [0]
model.compile(loss="binary_crossentropy",
#optimizer=SGD(learning_rate=0.01, momentum=0.9),
optimizer=Adam(learning_rate=0.0001), # rs: why not learning_rate?
metrics=['accuracy'])
callbacks = [TensorBoard(os.path.join(dir_output, 'logs'), write_graph=False),
SaveWeightsAfterSteps(0, dir_output, _config)]
if save_interval:

6
src/eynollah/utils/utils_ocr.py
View file

@ -7,7 +7,6 @@ import tensorflow as tf
from scipy.signal import find_peaks
from scipy.ndimage import gaussian_filter1d
from PIL import Image, ImageDraw, ImageFont
from Bio import pairwise2
from .resize import resize_image
@ -503,8 +502,3 @@ def return_rnn_cnn_ocr_of_given_textlines(image,
ocr_textline_in_textregion.append(text_textline)
ocr_all_textlines.append(ocr_textline_in_textregion)
return ocr_all_textlines
def biopython_align(str1, str2):
alignments = pairwise2.align.globalms(str1, str2, 2, -1, -2, -2)
best_alignment = alignments[0] # Get the best alignment
return best_alignment.seqA, best_alignment.seqB

7
train/requirements.txt
View file

@ -1,7 +1,10 @@
sacred
ocrd-fork-sacred >= 0.8.7.post1
seaborn
numpy
tqdm
imutils
scipy
tensorflow-addons # for connected_components
tensorflow-addons # for connected_components, depublished and only compatible with tensorflow < 2.16
tensorflow < 2.16 # for tensorflow-addons, so only needed in training
tf_data < 2.16 # for tensorflow-addons, so only needed in training
protobuf < 5 # for tensorflow-addons, so only needed in training