From e2754da4f5f81ce34d5a21bf726741c27ac2aecf Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Tue, 20 Jan 2026 04:04:07 +0100 Subject: [PATCH 01/21] =?UTF-8?q?adapt=20to=20Numpy=201.25=20changes?= =?UTF-8?q?=E2=80=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (esp. `np.array(...)` now not allowed on ragged arrays unless `dtype=object`, but then coercing sub-arrays to `object` as well) --- src/eynollah/eynollah.py | 22 +++++++++++++--------- src/eynollah/utils/__init__.py | 10 +++++++++- src/eynollah/utils/contour.py | 13 ++++++++----- 3 files changed, 30 insertions(+), 15 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index cceab31..c33b9f8 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -117,6 +117,7 @@ from .utils.marginals import get_marginals from .utils.resize import resize_image from .utils.shm import share_ndarray from .utils import ( + ensure_array, is_image_filename, boosting_headers_by_longshot_region_segmentation, crop_image_inside_box, @@ -2475,8 +2476,8 @@ class Eynollah: self, contours_only_text_parent, contours_only_text_parent_h, boxes, textline_mask_tot): self.logger.debug("enter do_order_of_regions") - contours_only_text_parent = np.array(contours_only_text_parent) - contours_only_text_parent_h = np.array(contours_only_text_parent_h) + contours_only_text_parent = ensure_array(contours_only_text_parent) + contours_only_text_parent_h = ensure_array(contours_only_text_parent_h) boxes = np.array(boxes, dtype=int) # to be on the safe side c_boxes = np.stack((0.5 * boxes[:, 2:4].sum(axis=1), 0.5 * boxes[:, 0:2].sum(axis=1))) @@ -3987,7 +3988,7 @@ class Eynollah: def filterfun(lis): if len(lis) == 0: return [] - return list(np.array(lis)[indices]) + return list(ensure_array(lis)[indices]) return (filterfun(contours_par), filterfun(contours_textline), @@ -4378,7 +4379,8 @@ class Eynollah: areas_cnt_text = np.array([cv2.contourArea(c) for c in contours_only_text_parent]) areas_cnt_text = areas_cnt_text / float(areas_tot_text) #self.logger.info('areas_cnt_text %s', areas_cnt_text) - contours_only_text_parent = np.array(contours_only_text_parent)[areas_cnt_text > MIN_AREA_REGION] + contours_only_text_parent = ensure_array(contours_only_text_parent) + contours_only_text_parent = contours_only_text_parent[areas_cnt_text > MIN_AREA_REGION] areas_cnt_text_parent = areas_cnt_text[areas_cnt_text > MIN_AREA_REGION] index_con_parents = np.argsort(areas_cnt_text_parent) @@ -4397,12 +4399,13 @@ class Eynollah: areas_cnt_text_d = np.array([cv2.contourArea(c) for c in contours_only_text_parent_d]) areas_cnt_text_d = areas_cnt_text_d / float(areas_tot_text_d) - contours_only_text_parent_d = np.array(contours_only_text_parent_d)[areas_cnt_text_d > MIN_AREA_REGION] + contours_only_text_parent_d = ensure_array(contours_only_text_parent_d) + contours_only_text_parent_d = contours_only_text_parent_d[areas_cnt_text_d > MIN_AREA_REGION] areas_cnt_text_d = areas_cnt_text_d[areas_cnt_text_d > MIN_AREA_REGION] if len(contours_only_text_parent_d): index_con_parents_d = np.argsort(areas_cnt_text_d) - contours_only_text_parent_d = np.array(contours_only_text_parent_d)[index_con_parents_d] + contours_only_text_parent_d = contours_only_text_parent_d[index_con_parents_d] areas_cnt_text_d = areas_cnt_text_d[index_con_parents_d] centers_d = np.stack(find_center_of_contours(contours_only_text_parent_d)) # [2, N] @@ -4546,9 +4549,10 @@ class Eynollah: #print("text region early 3 in %.1fs", time.time() - t0) if self.light_version: contours_only_text_parent = dilate_textregion_contours(contours_only_text_parent) - contours_only_text_parent , contours_only_text_parent_d_ordered = self.filter_contours_inside_a_bigger_one( - contours_only_text_parent, contours_only_text_parent_d_ordered, text_only, - marginal_cnts=polygons_of_marginals) + contours_only_text_parent, contours_only_text_parent_d_ordered = \ + self.filter_contours_inside_a_bigger_one( + contours_only_text_parent, contours_only_text_parent_d_ordered, text_only, + marginal_cnts=polygons_of_marginals) #print("text region early 3.5 in %.1fs", time.time() - t0) conf_contours_textregions = get_textregion_contours_in_org_image_light( contours_only_text_parent, self.image, confidence_matrix) diff --git a/src/eynollah/utils/__init__.py b/src/eynollah/utils/__init__.py index 43d5d75..4e55aef 100644 --- a/src/eynollah/utils/__init__.py +++ b/src/eynollah/utils/__init__.py @@ -1,4 +1,4 @@ -from typing import List, Tuple +from typing import Iterable, List, Tuple from logging import getLogger import time import math @@ -1929,3 +1929,11 @@ def is_image_filename(fname: str) -> bool: def is_xml_filename(fname: str) -> bool: return fname.lower().endswith('.xml') + +def ensure_array(obj: Iterable) -> np.ndarray: + """convert sequence to array of type `object` so items can be of heterogeneous shape + (but ensure not to convert inner arrays to `object` if len=1) + """ + if not isinstance(obj, np.ndarray): + return np.fromiter(obj, object) + return obj diff --git a/src/eynollah/utils/contour.py b/src/eynollah/utils/contour.py index 393acdd..7d01e74 100644 --- a/src/eynollah/utils/contour.py +++ b/src/eynollah/utils/contour.py @@ -12,6 +12,7 @@ from shapely import set_precision from shapely.ops import unary_union, nearest_points from .rotate import rotate_image, rotation_image_new +from . import ensure_array def contours_in_same_horizon(cy_main_hor): """ @@ -248,13 +249,15 @@ def return_contours_of_image(image): return contours, hierarchy def dilate_textline_contours(all_found_textline_polygons): - return [[polygon2contour(contour2polygon(contour, dilate=6)) - for contour in region] + return [ensure_array( + [polygon2contour(contour2polygon(contour, dilate=6)) + for contour in region]) for region in all_found_textline_polygons] -def dilate_textregion_contours(all_found_textline_polygons): - return [polygon2contour(contour2polygon(contour, dilate=6)) - for contour in all_found_textline_polygons] +def dilate_textregion_contours(all_found_textregion_polygons): + return ensure_array( + [polygon2contour(contour2polygon(contour, dilate=6)) + for contour in all_found_textregion_polygons]) def contour2polygon(contour: Union[np.ndarray, Sequence[Sequence[Sequence[Number]]]], dilate=0): polygon = Polygon([point[0] for point in contour]) From 3c3effcfda9b8d4dfd9dc8f685bb520fab1840b3 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Tue, 20 Jan 2026 04:18:55 +0100 Subject: [PATCH 02/21] =?UTF-8?q?drop=20TF1=20vernacular,=20relax=20TF/Ker?= =?UTF-8?q?as=20and=20Torch=20requirements=E2=80=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - do not restrict TF version, but depend on tf-keras and set `TF_USE_LEGACY_KERAS=1` to avoid Keras 3 behaviour - relax Numpy version requirement up to v2 - relax Torch version requirement - drop TF1 session management code - drop TF1 config in favour of TF2 config code for memory growth - training.*: also simplify and limit line length - training.train: always train with TensorBoard callback --- requirements-ocr.txt | 2 +- requirements.txt | 5 +- src/eynollah/eynollah.py | 12 +- src/eynollah/sbb_binarize.py | 28 +- ..._model_load_pretrained_weights_and_save.py | 8 +- src/eynollah/training/inference.py | 192 ++++------ src/eynollah/training/train.py | 333 +++++++++++------- src/eynollah/utils/contour.py | 3 +- 8 files changed, 289 insertions(+), 294 deletions(-) diff --git a/requirements-ocr.txt b/requirements-ocr.txt index 9f31ebb..8f3b062 100644 --- a/requirements-ocr.txt +++ b/requirements-ocr.txt @@ -1,2 +1,2 @@ -torch <= 2.0.1 +torch transformers <= 4.30.2 diff --git a/requirements.txt b/requirements.txt index db1d7df..5699566 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,9 @@ # ocrd includes opencv, numpy, shapely, click ocrd >= 3.3.0 -numpy <1.24.0 +numpy < 2.0 scikit-learn >= 0.23.2 -tensorflow < 2.13 +tensorflow +tf-keras # avoid keras 3 (also needs TF_USE_LEGACY_KERAS=1) numba <= 0.58.1 scikit-image biopython diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index c33b9f8..4a83c0a 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -56,14 +56,12 @@ except ImportError: TrOCRProcessor = VisionEncoderDecoderModel = None #os.environ['CUDA_VISIBLE_DEVICES'] = '-1' +os.environ['TF_USE_LEGACY_KERAS'] = '1' # avoid Keras 3 after TF 2.15 tf_disable_interactive_logs() import tensorflow as tf -from tensorflow.python.keras import backend as K from tensorflow.keras.models import load_model tf.get_logger().setLevel("ERROR") warnings.filterwarnings("ignore") -# use tf1 compatibility for keras backend -from tensorflow.compat.v1.keras.backend import set_session from tensorflow.keras import layers from tensorflow.keras.layers import StringLookup @@ -277,14 +275,6 @@ class Eynollah: t_start = time.time() - # #gpu_options = tf.compat.v1.GPUOptions(allow_growth=True) - # #gpu_options = tf.compat.v1.GPUOptions(per_process_gpu_memory_fraction=7.7, allow_growth=True) - # #session = tf.compat.v1.Session(config=tf.compat.v1.ConfigProto(gpu_options=gpu_options)) - # config = tf.compat.v1.ConfigProto() - # config.gpu_options.allow_growth = True - # #session = tf.InteractiveSession() - # session = tf.compat.v1.Session(config=config) - # set_session(session) try: for device in tf.config.list_physical_devices('GPU'): tf.config.experimental.set_memory_growth(device, True) diff --git a/src/eynollah/sbb_binarize.py b/src/eynollah/sbb_binarize.py index b81f45e..2ca4a40 100644 --- a/src/eynollah/sbb_binarize.py +++ b/src/eynollah/sbb_binarize.py @@ -2,19 +2,19 @@ Tool to load model and binarize a given image. """ -import sys from glob import glob import os import logging +from PIL import Image import numpy as np -from PIL import Image import cv2 from ocrd_utils import tf_disable_interactive_logs + +os.environ['TF_USE_LEGACY_KERAS'] = '1' # avoid Keras 3 after TF 2.15 tf_disable_interactive_logs() import tensorflow as tf from tensorflow.keras.models import load_model -from tensorflow.python.keras import backend as tensorflow_backend from .utils import is_image_filename @@ -27,26 +27,17 @@ class SbbBinarizer: self.model_dir = model_dir self.logger = logger if logger else logging.getLogger('SbbBinarizer') - self.start_new_session() - - self.model_files = glob(self.model_dir+"/*/", recursive = True) + try: + for device in tf.config.list_physical_devices('GPU'): + tf.config.experimental.set_memory_growth(device, True) + except: + self.logger.warning("no GPU device available") + self.model_files = glob(self.model_dir + "/*/", recursive=True) self.models = [] for model_file in self.model_files: self.models.append(self.load_model(model_file)) - def start_new_session(self): - config = tf.compat.v1.ConfigProto() - config.gpu_options.allow_growth = True - - self.session = tf.compat.v1.Session(config=config) # tf.InteractiveSession() - tensorflow_backend.set_session(self.session) - - def end_session(self): - tensorflow_backend.clear_session() - self.session.close() - del self.session - def load_model(self, model_name): model = load_model(os.path.join(self.model_dir, model_name), compile=False) model_height = model.layers[len(model.layers)-1].output_shape[1] @@ -55,7 +46,6 @@ class SbbBinarizer: return model, model_height, model_width, n_classes def predict(self, model_in, img, use_patches, n_batch_inference=5): - tensorflow_backend.set_session(self.session) model, model_height, model_width, n_classes = model_in img_org_h = img.shape[0] diff --git a/src/eynollah/training/build_model_load_pretrained_weights_and_save.py b/src/eynollah/training/build_model_load_pretrained_weights_and_save.py index 40fc1fe..9fba66b 100644 --- a/src/eynollah/training/build_model_load_pretrained_weights_and_save.py +++ b/src/eynollah/training/build_model_load_pretrained_weights_and_save.py @@ -1,3 +1,4 @@ +import sys import click import tensorflow as tf @@ -5,8 +6,11 @@ from .models import resnet50_unet def configuration(): - gpu_options = tf.compat.v1.GPUOptions(allow_growth=True) - session = tf.compat.v1.Session(config=tf.compat.v1.ConfigProto(gpu_options=gpu_options)) + try: + for device in tf.config.list_physical_devices('GPU'): + tf.config.experimental.set_memory_growth(device, True) + except: + print("no GPU device available", file=sys.stderr) @click.command() def build_model_load_pretrained_weights_and_save(): diff --git a/src/eynollah/training/inference.py b/src/eynollah/training/inference.py index 3fa8fd6..15d1e6a 100644 --- a/src/eynollah/training/inference.py +++ b/src/eynollah/training/inference.py @@ -1,16 +1,19 @@ +""" +Tool to load model and predict for given image. +""" + import sys import os import warnings import json +import click import numpy as np import cv2 -from tensorflow.keras.models import load_model + +os.environ['TF_USE_LEGACY_KERAS'] = '1' # avoid Keras 3 after TF 2.15 import tensorflow as tf -from tensorflow.keras import backend as K -from tensorflow.keras.layers import * -import click -from tensorflow.python.keras import backend as tensorflow_backend +from tensorflow.keras.models import load_model import xml.etree.ElementTree as ET from .gt_gen_utils import ( @@ -24,17 +27,29 @@ from .models import ( PatchEncoder, Patches ) +from .metrics import ( + soft_dice_loss, + weighted_categorical_crossentropy, +) with warnings.catch_warnings(): warnings.simplefilter("ignore") -__doc__=\ -""" -Tool to load model and predict for given image. -""" +class SBBPredict: + def __init__(self, + image, + dir_in, + model, + task, + config_params_model, + patches, + save, + save_layout, + ground_truth, + xml_file, + out, + min_area): -class sbb_predict: - def __init__(self,image, dir_in, model, task, config_params_model, patches, save, save_layout, ground_truth, xml_file, out, min_area): self.image=image self.dir_in=dir_in self.patches=patches @@ -52,8 +67,9 @@ class sbb_predict: self.min_area = 0 def resize_image(self,img_in,input_height,input_width): - return cv2.resize( img_in, ( input_width,input_height) ,interpolation=cv2.INTER_NEAREST) - + return cv2.resize(img_in, (input_width, + input_height), + interpolation=cv2.INTER_NEAREST) def color_images(self,seg): ann_u=range(self.n_classes) @@ -69,68 +85,6 @@ class sbb_predict: seg_img[:,:,2][seg==c]=c return seg_img - def otsu_copy_binary(self,img): - img_r=np.zeros((img.shape[0],img.shape[1],3)) - img1=img[:,:,0] - - #print(img.min()) - #print(img[:,:,0].min()) - #blur = cv2.GaussianBlur(img,(5,5)) - #ret3,th3 = cv2.threshold(blur,0,255,cv2.THRESH_BINARY+cv2.THRESH_OTSU) - retval1, threshold1 = cv2.threshold(img1, 0, 255, cv2.THRESH_BINARY+cv2.THRESH_OTSU) - - - - img_r[:,:,0]=threshold1 - img_r[:,:,1]=threshold1 - img_r[:,:,2]=threshold1 - #img_r=img_r/float(np.max(img_r))*255 - return img_r - - def otsu_copy(self,img): - img_r=np.zeros((img.shape[0],img.shape[1],3)) - #img1=img[:,:,0] - - #print(img.min()) - #print(img[:,:,0].min()) - #blur = cv2.GaussianBlur(img,(5,5)) - #ret3,th3 = cv2.threshold(blur,0,255,cv2.THRESH_BINARY+cv2.THRESH_OTSU) - _, threshold1 = cv2.threshold(img[:,:,0], 0, 255, cv2.THRESH_BINARY+cv2.THRESH_OTSU) - _, threshold2 = cv2.threshold(img[:,:,1], 0, 255, cv2.THRESH_BINARY+cv2.THRESH_OTSU) - _, threshold3 = cv2.threshold(img[:,:,2], 0, 255, cv2.THRESH_BINARY+cv2.THRESH_OTSU) - - - - img_r[:,:,0]=threshold1 - img_r[:,:,1]=threshold2 - img_r[:,:,2]=threshold3 - ###img_r=img_r/float(np.max(img_r))*255 - return img_r - - def soft_dice_loss(self,y_true, y_pred, epsilon=1e-6): - - axes = tuple(range(1, len(y_pred.shape)-1)) - - numerator = 2. * K.sum(y_pred * y_true, axes) - - denominator = K.sum(K.square(y_pred) + K.square(y_true), axes) - return 1.00 - K.mean(numerator / (denominator + epsilon)) # average over classes and batch - - def weighted_categorical_crossentropy(self,weights=None): - - def loss(y_true, y_pred): - labels_floats = tf.cast(y_true, tf.float32) - per_pixel_loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=labels_floats,logits=y_pred) - - if weights is not None: - weight_mask = tf.maximum(tf.reduce_max(tf.constant( - np.array(weights, dtype=np.float32)[None, None, None]) - * labels_floats, axis=-1), 1.0) - per_pixel_loss = per_pixel_loss * weight_mask[:, :, :, None] - return tf.reduce_mean(per_pixel_loss) - return self.loss - - def IoU(self,Yi,y_predi): ## mean Intersection over Union ## Mean IoU = TP/(FN + TP + FP) @@ -157,30 +111,28 @@ class sbb_predict: return mIoU def start_new_session_and_model(self): - - config = tf.compat.v1.ConfigProto() - config.gpu_options.allow_growth = True + try: + for device in tf.config.list_physical_devices('GPU'): + tf.config.experimental.set_memory_growth(device, True) + except: + print("no GPU device available", file=sys.stderr) - session = tf.compat.v1.Session(config=config) # tf.InteractiveSession() - tensorflow_backend.set_session(session) #tensorflow.keras.layers.custom_layer = PatchEncoder #tensorflow.keras.layers.custom_layer = Patches - self.model = load_model(self.model_dir , compile=False,custom_objects = {"PatchEncoder": PatchEncoder, "Patches": Patches}) - #config = tf.ConfigProto() - #config.gpu_options.allow_growth=True - - #self.session = tf.InteractiveSession() - #keras.losses.custom_loss = self.weighted_categorical_crossentropy - #self.model = load_model(self.model_dir , compile=False) + self.model = load_model(self.model_dir, compile=False, + custom_objects={"PatchEncoder": PatchEncoder, + "Patches": Patches}) + #keras.losses.custom_loss = weighted_categorical_crossentropy + #self.model = load_model(self.model_dir, compile=False) - ##if self.weights_dir!=None: ##self.model.load_weights(self.weights_dir) if self.task != 'classification' and self.task != 'reading_order': - self.img_height=self.model.layers[len(self.model.layers)-1].output_shape[1] - self.img_width=self.model.layers[len(self.model.layers)-1].output_shape[2] - self.n_classes=self.model.layers[len(self.model.layers)-1].output_shape[3] + last = self.model.layers[-1] + self.img_height = last.output_shape[1] + self.img_width = last.output_shape[2] + self.n_classes = last.output_shape[3] def visualize_model_output(self, prediction, img, task): if task == "binarization": @@ -208,21 +160,16 @@ class sbb_predict: '15' : [255, 0, 255]} layout_only = np.zeros(prediction.shape) - for unq_class in unique_classes: + where = prediction[:,:,0]==unq_class rgb_class_unique = rgb_colors[str(int(unq_class))] - layout_only[:,:,0][prediction[:,:,0]==unq_class] = rgb_class_unique[0] - layout_only[:,:,1][prediction[:,:,0]==unq_class] = rgb_class_unique[1] - layout_only[:,:,2][prediction[:,:,0]==unq_class] = rgb_class_unique[2] - - + layout_only[:,:,0][where] = rgb_class_unique[0] + layout_only[:,:,1][where] = rgb_class_unique[1] + layout_only[:,:,2][where] = rgb_class_unique[2] + layout_only = layout_only.astype(np.int32) img = self.resize_image(img, layout_only.shape[0], layout_only.shape[1]) - - layout_only = layout_only.astype(np.int32) img = img.astype(np.int32) - - added_image = cv2.addWeighted(img,0.5,layout_only,0.1,0) @@ -231,10 +178,10 @@ class sbb_predict: def predict(self, image_dir): if self.task == 'classification': classes_names = self.config_params_model['classification_classes_name'] - img_1ch = img=cv2.imread(image_dir, 0) - - img_1ch = img_1ch / 255.0 - img_1ch = cv2.resize(img_1ch, (self.config_params_model['input_height'], self.config_params_model['input_width']), interpolation=cv2.INTER_NEAREST) + img_1ch = cv2.imread(image_dir, 0) / 255.0 + img_1ch = cv2.resize(img_1ch, (self.config_params_model['input_height'], + self.config_params_model['input_width']), + interpolation=cv2.INTER_NEAREST) img_in = np.zeros((1, img_1ch.shape[0], img_1ch.shape[1], 3)) img_in[0, :, :, 0] = img_1ch[:, :] img_in[0, :, :, 1] = img_1ch[:, :] @@ -244,23 +191,27 @@ class sbb_predict: index_class = np.argmax(label_p_pred[0]) print("Predicted Class: {}".format(classes_names[str(int(index_class))])) + elif self.task == 'reading_order': img_height = self.config_params_model['input_height'] img_width = self.config_params_model['input_width'] - tree_xml, root_xml, bb_coord_printspace, file_name, id_paragraph, id_header, co_text_paragraph, co_text_header, tot_region_ref, x_len, y_len, index_tot_regions, img_poly = read_xml(self.xml_file) - _, cy_main, x_min_main, x_max_main, y_min_main, y_max_main, _ = find_new_features_of_contours(co_text_header) + tree_xml, root_xml, bb_coord_printspace, file_name, \ + id_paragraph, id_header, \ + co_text_paragraph, co_text_header, \ + tot_region_ref, x_len, y_len, index_tot_regions, \ + img_poly = read_xml(self.xml_file) + _, cy_main, x_min_main, x_max_main, y_min_main, y_max_main, _ = \ + find_new_features_of_contours(co_text_header) img_header_and_sep = np.zeros((y_len,x_len), dtype='uint8') - - for j in range(len(cy_main)): - img_header_and_sep[int(y_max_main[j]):int(y_max_main[j])+12,int(x_min_main[j]):int(x_max_main[j]) ] = 1 - + img_header_and_sep[int(y_max_main[j]): int(y_max_main[j]) + 12, + int(x_min_main[j]): int(x_max_main[j])] = 1 + co_text_all = co_text_paragraph + co_text_header id_all_text = id_paragraph + id_header - ##texts_corr_order_index = [index_tot_regions[tot_region_ref.index(i)] for i in id_all_text ] ##texts_corr_order_index_int = [int(x) for x in texts_corr_order_index] texts_corr_order_index_int = list(np.array(range(len(co_text_all)))) @@ -271,7 +222,8 @@ class sbb_predict: #print(np.shape(co_text_all[0]), len( np.shape(co_text_all[0]) ),'co_text_all') #co_text_all = filter_contours_area_of_image_tables(img_poly, co_text_all, _, max_area, min_area) #print(co_text_all,'co_text_all') - co_text_all, texts_corr_order_index_int, _ = filter_contours_area_of_image(img_poly, co_text_all, texts_corr_order_index_int, max_area, self.min_area) + co_text_all, texts_corr_order_index_int, _ = filter_contours_area_of_image( + img_poly, co_text_all, texts_corr_order_index_int, max_area, self.min_area) #print(texts_corr_order_index_int) @@ -664,17 +616,15 @@ class sbb_predict: help="min area size of regions considered for reading order detection. The default value is zero and means that all text regions are considered for reading order.", ) def main(image, dir_in, model, patches, save, save_layout, ground_truth, xml_file, out, min_area): - assert image or dir_in, "Either a single image -i or a dir_in -di is required" + assert image or dir_in, "Either a single image -i or a dir_in -di input is required" with open(os.path.join(model,'config.json')) as f: config_params_model = json.load(f) task = config_params_model['task'] if task != 'classification' and task != 'reading_order': - if image and not save: - print("Error: You used one of segmentation or binarization task with image input but not set -s, you need a filename to save visualized output with -s") - sys.exit(1) - if dir_in and not out: - print("Error: You used one of segmentation or binarization task with dir_in but not set -out") - sys.exit(1) - x=sbb_predict(image, dir_in, model, task, config_params_model, patches, save, save_layout, ground_truth, xml_file, out, min_area) + assert not image or save, "For segmentation or binarization, an input single image -i also requires an output filename -s" + assert not dir_in or out, "For segmentation or binarization, an input directory -di also requires an output directory -o" + x = SBBPredict(image, dir_in, model, task, config_params_model, + patches, save, save_layout, ground_truth, xml_file, out, + min_area) x.run() diff --git a/src/eynollah/training/train.py b/src/eynollah/training/train.py index 97736e0..da901b0 100644 --- a/src/eynollah/training/train.py +++ b/src/eynollah/training/train.py @@ -28,14 +28,14 @@ from eynollah.training.utils import ( ) os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' +os.environ['TF_USE_LEGACY_KERAS'] = '1' # avoid Keras 3 after TF 2.15 import tensorflow as tf -from tensorflow.compat.v1.keras.backend import set_session from tensorflow.keras.optimizers import SGD, Adam -from sacred import Experiment from tensorflow.keras.models import load_model +from tensorflow.keras.callbacks import Callback, TensorBoard +from sacred import Experiment from tqdm import tqdm from sklearn.metrics import f1_score -from tensorflow.keras.callbacks import Callback import numpy as np import cv2 @@ -63,10 +63,11 @@ class SaveWeightsAfterSteps(Callback): def configuration(): - config = tf.compat.v1.ConfigProto() - config.gpu_options.allow_growth = True - session = tf.compat.v1.Session(config=config) - set_session(session) + try: + for device in tf.config.list_physical_devices('GPU'): + tf.config.experimental.set_memory_growth(device, True) + except: + print("no GPU device available", file=sys.stderr) def get_dirs_or_files(input_data): @@ -171,12 +172,11 @@ def run(_config, n_classes, n_epochs, input_height, else: list_all_possible_foreground_rgbs = None - if task == "segmentation" or task == "enhancement" or task == "binarization": + if task in ["segmentation", "enhancement", "binarization"]: if data_is_provided: dir_train_flowing = os.path.join(dir_output, 'train') dir_eval_flowing = os.path.join(dir_output, 'eval') - dir_flow_train_imgs = os.path.join(dir_train_flowing, 'images') dir_flow_train_labels = os.path.join(dir_train_flowing, 'labels') @@ -227,176 +227,228 @@ def run(_config, n_classes, n_epochs, input_height, segs_list_test=np.array(os.listdir(dir_seg_val)) # writing patches into a sub-folder in order to be flowed from directory. - provide_patches(imgs_list, segs_list, dir_img, dir_seg, dir_flow_train_imgs, - dir_flow_train_labels, input_height, input_width, blur_k, - blur_aug, padding_white, padding_black, flip_aug, binarization, adding_rgb_background,adding_rgb_foreground, add_red_textlines, channels_shuffling, - scaling, shifting, degrading, brightening, scales, degrade_scales, brightness, - flip_index,shuffle_indexes, scaling_bluring, scaling_brightness, scaling_binarization, - rotation, rotation_not_90, thetha, scaling_flip, task, augmentation=augmentation, - patches=patches, dir_img_bin=dir_img_bin,number_of_backgrounds_per_image=number_of_backgrounds_per_image,list_all_possible_background_images=list_all_possible_background_images, dir_rgb_backgrounds=dir_rgb_backgrounds, dir_rgb_foregrounds=dir_rgb_foregrounds,list_all_possible_foreground_rgbs=list_all_possible_foreground_rgbs) - - provide_patches(imgs_list_test, segs_list_test, dir_img_val, dir_seg_val, - dir_flow_eval_imgs, dir_flow_eval_labels, input_height, input_width, - blur_k, blur_aug, padding_white, padding_black, flip_aug, binarization, adding_rgb_background, adding_rgb_foreground, add_red_textlines, channels_shuffling, - scaling, shifting, degrading, brightening, scales, degrade_scales, brightness, - flip_index, shuffle_indexes, scaling_bluring, scaling_brightness, scaling_binarization, - rotation, rotation_not_90, thetha, scaling_flip, task, augmentation=False, patches=patches,dir_img_bin=dir_img_bin,number_of_backgrounds_per_image=number_of_backgrounds_per_image,list_all_possible_background_images=list_all_possible_background_images, dir_rgb_backgrounds=dir_rgb_backgrounds,dir_rgb_foregrounds=dir_rgb_foregrounds,list_all_possible_foreground_rgbs=list_all_possible_foreground_rgbs ) + common_args = [input_height, input_width, + blur_k, blur_aug, + padding_white, padding_black, + flip_aug, binarization, + adding_rgb_background, + adding_rgb_foreground, + add_red_textlines, + channels_shuffling, + scaling, shifting, degrading, brightening, + scales, degrade_scales, brightness, + flip_index, shuffle_indexes, + scaling_bluring, scaling_brightness, scaling_binarization, + rotation, rotation_not_90, thetha, + scaling_flip, task, + ] + common_kwargs = dict(patches= + patches, + dir_img_bin= + dir_img_bin, + number_of_backgrounds_per_image= + number_of_backgrounds_per_image, + list_all_possible_background_images= + list_all_possible_background_images, + dir_rgb_backgrounds= + dir_rgb_backgrounds, + dir_rgb_foregrounds= + dir_rgb_foregrounds, + list_all_possible_foreground_rgbs= + list_all_possible_foreground_rgbs, + ) + provide_patches(imgs_list, segs_list, + dir_img, dir_seg, + dir_flow_train_imgs, + dir_flow_train_labels, + *common_args, + augmentation=augmentation, + **common_kwargs) + provide_patches(imgs_list_test, segs_list_test, + dir_img_val, dir_seg_val, + dir_flow_eval_imgs, + dir_flow_eval_labels, + *common_args, + augmentation=False, + **common_kwargs) if weighted_loss: weights = np.zeros(n_classes) if data_is_provided: - for obj in os.listdir(dir_flow_train_labels): - try: - label_obj = cv2.imread(dir_flow_train_labels + '/' + obj) - label_obj_one_hot = get_one_hot(label_obj, label_obj.shape[0], label_obj.shape[1], n_classes) - weights += (label_obj_one_hot.sum(axis=0)).sum(axis=0) - except: - pass + dirs = dir_flow_train_labels else: - - for obj in os.listdir(dir_seg): - try: - label_obj = cv2.imread(dir_seg + '/' + obj) - label_obj_one_hot = get_one_hot(label_obj, label_obj.shape[0], label_obj.shape[1], n_classes) - weights += (label_obj_one_hot.sum(axis=0)).sum(axis=0) - except: - pass + dirs = dir_seg + for obj in os.listdir(dirs): + label_file = os.path.join(dirs, + obj) + try: + label_obj = cv2.imread(label_file) + label_obj_one_hot = get_one_hot(label_obj, label_obj.shape[0], label_obj.shape[1], n_classes) + weights += (label_obj_one_hot.sum(axis=0)).sum(axis=0) + except Exception as e: + print("error reading data file '%s': %s" % (label_file, e), file=sys.stderr) weights = 1.00 / weights - weights = weights / float(np.sum(weights)) weights = weights / float(np.min(weights)) weights = weights / float(np.sum(weights)) if continue_training: - if backbone_type=='nontransformer': - if is_loss_soft_dice and (task == "segmentation" or task == "binarization"): - model = load_model(dir_of_start_model, compile=True, custom_objects={'soft_dice_loss': soft_dice_loss}) - if weighted_loss and (task == "segmentation" or task == "binarization"): - model = load_model(dir_of_start_model, compile=True, custom_objects={'loss': weighted_categorical_crossentropy(weights)}) - if not is_loss_soft_dice and not weighted_loss: + if backbone_type == 'nontransformer': + if is_loss_soft_dice and task in ["segmentation", "binarization"]: + model = load_model(dir_of_start_model, compile=True, + custom_objects={'soft_dice_loss': soft_dice_loss}) + elif weighted_loss and task in ["segmentation", "binarization"]: + model = load_model(dir_of_start_model, compile=True, + custom_objects={'loss': weighted_categorical_crossentropy(weights)}) + else: model = load_model(dir_of_start_model , compile=True) - elif backbone_type=='transformer': - if is_loss_soft_dice and (task == "segmentation" or task == "binarization"): - model = load_model(dir_of_start_model, compile=True, custom_objects={"PatchEncoder": PatchEncoder, "Patches": Patches,'soft_dice_loss': soft_dice_loss}) - if weighted_loss and (task == "segmentation" or task == "binarization"): - model = load_model(dir_of_start_model, compile=True, custom_objects={'loss': weighted_categorical_crossentropy(weights)}) - if not is_loss_soft_dice and not weighted_loss: - model = load_model(dir_of_start_model , compile=True,custom_objects = {"PatchEncoder": PatchEncoder, "Patches": Patches}) + + elif backbone_type == 'transformer': + if is_loss_soft_dice and task in ["segmentation", "binarization"]: + model = load_model(dir_of_start_model, compile=True, + custom_objects={"PatchEncoder": PatchEncoder, + "Patches": Patches, + 'soft_dice_loss': soft_dice_loss}) + elif weighted_loss and task in ["segmentation", "binarization"]: + model = load_model(dir_of_start_model, compile=True, + custom_objects={'loss': weighted_categorical_crossentropy(weights)}) + else: + model = load_model(dir_of_start_model, compile=True, + custom_objects = {"PatchEncoder": PatchEncoder, + "Patches": Patches}) else: index_start = 0 - if backbone_type=='nontransformer': - model = resnet50_unet(n_classes, input_height, input_width, task, weight_decay, pretraining) - elif backbone_type=='transformer': + if backbone_type == 'nontransformer': + model = resnet50_unet(n_classes, + input_height, + input_width, + task, + weight_decay, + pretraining) + elif backbone_type == 'transformer': num_patches_x = transformer_num_patches_xy[0] num_patches_y = transformer_num_patches_xy[1] num_patches = num_patches_x * num_patches_y if transformer_cnn_first: - if input_height != (num_patches_y * transformer_patchsize_y * 32): - print("Error: transformer_patchsize_y or transformer_num_patches_xy height value error . input_height should be equal to ( transformer_num_patches_xy height value * transformer_patchsize_y * 32)") - sys.exit(1) - if input_width != (num_patches_x * transformer_patchsize_x * 32): - print("Error: transformer_patchsize_x or transformer_num_patches_xy width value error . input_width should be equal to ( transformer_num_patches_xy width value * transformer_patchsize_x * 32)") - sys.exit(1) - if (transformer_projection_dim % (transformer_patchsize_y * transformer_patchsize_x)) != 0: - print("Error: transformer_projection_dim error. The remainder when parameter transformer_projection_dim is divided by (transformer_patchsize_y*transformer_patchsize_x) should be zero") - sys.exit(1) - - - model = vit_resnet50_unet(n_classes, transformer_patchsize_x, transformer_patchsize_y, num_patches, transformer_mlp_head_units, transformer_layers, transformer_num_heads, transformer_projection_dim, input_height, input_width, task, weight_decay, pretraining) + model_builder = vit_resnet50_unet + multiple_of_32 = True else: - if input_height != (num_patches_y * transformer_patchsize_y): - print("Error: transformer_patchsize_y or transformer_num_patches_xy height value error . input_height should be equal to ( transformer_num_patches_xy height value * transformer_patchsize_y)") - sys.exit(1) - if input_width != (num_patches_x * transformer_patchsize_x): - print("Error: transformer_patchsize_x or transformer_num_patches_xy width value error . input_width should be equal to ( transformer_num_patches_xy width value * transformer_patchsize_x)") - sys.exit(1) - if (transformer_projection_dim % (transformer_patchsize_y * transformer_patchsize_x)) != 0: - print("Error: transformer_projection_dim error. The remainder when parameter transformer_projection_dim is divided by (transformer_patchsize_y*transformer_patchsize_x) should be zero") - sys.exit(1) - model = vit_resnet50_unet_transformer_before_cnn(n_classes, transformer_patchsize_x, transformer_patchsize_y, num_patches, transformer_mlp_head_units, transformer_layers, transformer_num_heads, transformer_projection_dim, input_height, input_width, task, weight_decay, pretraining) + model_builder = vit_resnet50_unet_transformer_before_cnn + multiple_of_32 = False + + assert input_height == num_patches_y * transformer_patchsize_y * (32 if multiple_of_32 else 1), \ + "transformer_patchsize_y or transformer_num_patches_xy height value error: " \ + "input_height should be equal to " \ + "(transformer_num_patches_xy height value * transformer_patchsize_y%s)" % \ + " * 32" if multiple_of_32 else "" + assert input_width == num_patches_x * transformer_patchsize_x * (32 if multiple_of_32 else 1), \ + "transformer_patchsize_x or transformer_num_patches_xy width value error: " \ + "input_width should be equal to " \ + "(transformer_num_patches_xy width value * transformer_patchsize_x%s)" % \ + " * 32" if multiple_of_32 else "" + assert 0 == transformer_projection_dim % (transformer_patchsize_y * transformer_patchsize_x), \ + "transformer_projection_dim error: " \ + "The remainder when parameter transformer_projection_dim is divided by " \ + "(transformer_patchsize_y*transformer_patchsize_x) should be zero" + + model = model_builder( + n_classes, + transformer_patchsize_x, + transformer_patchsize_y, + num_patches, + transformer_mlp_head_units, + transformer_layers, + transformer_num_heads, + transformer_projection_dim, + input_height, + input_width, + task, + weight_decay, + pretraining) #if you want to see the model structure just uncomment model summary. model.summary() - - if task == "segmentation" or task == "binarization": - if not is_loss_soft_dice and not weighted_loss: - model.compile(loss='categorical_crossentropy', - optimizer=Adam(learning_rate=learning_rate), metrics=['accuracy']) + if task in ["segmentation", "binarization"]: if is_loss_soft_dice: - model.compile(loss=soft_dice_loss, - optimizer=Adam(learning_rate=learning_rate), metrics=['accuracy']) - if weighted_loss: - model.compile(loss=weighted_categorical_crossentropy(weights), - optimizer=Adam(learning_rate=learning_rate), metrics=['accuracy']) - elif task == "enhancement": - model.compile(loss='mean_squared_error', - optimizer=Adam(learning_rate=learning_rate), metrics=['accuracy']) - + loss = soft_dice_loss + elif weighted_loss: + loss = weighted_categorical_crossentropy(weights) + else: + loss = 'categorical_crossentropy' + else: # task == "enhancement" + loss = 'mean_squared_error' + model.compile(loss=loss, + optimizer=Adam(learning_rate=learning_rate), + metrics=['accuracy']) # generating train and evaluation data - train_gen = data_gen(dir_flow_train_imgs, dir_flow_train_labels, batch_size=n_batch, - input_height=input_height, input_width=input_width, n_classes=n_classes, task=task) - val_gen = data_gen(dir_flow_eval_imgs, dir_flow_eval_labels, batch_size=n_batch, - input_height=input_height, input_width=input_width, n_classes=n_classes, task=task) - + gen_kwargs = dict(batch_size=n_batch, + input_height=input_height, + input_width=input_width, + n_classes=n_classes, + task=task) + train_gen = data_gen(dir_flow_train_imgs, dir_flow_train_labels, **gen_kwargs) + val_gen = data_gen(dir_flow_eval_imgs, dir_flow_eval_labels, **gen_kwargs) + ##img_validation_patches = os.listdir(dir_flow_eval_imgs) ##score_best=[] ##score_best.append(0) + callbacks = [TensorBoard(os.path.join(dir_output, 'logs'), write_graph=False)] if save_interval: - save_weights_callback = SaveWeightsAfterSteps(save_interval, dir_output, _config) - + callbacks.append(SaveWeightsAfterSteps(save_interval, dir_output, _config)) for i in tqdm(range(index_start, n_epochs + index_start)): - if save_interval: - model.fit( - train_gen, - steps_per_epoch=int(len(os.listdir(dir_flow_train_imgs)) / n_batch) - 1, - validation_data=val_gen, - validation_steps=1, - epochs=1, callbacks=[save_weights_callback]) - else: - model.fit( - train_gen, - steps_per_epoch=int(len(os.listdir(dir_flow_train_imgs)) / n_batch) - 1, - validation_data=val_gen, - validation_steps=1, - epochs=1) - - model.save(os.path.join(dir_output,'model_'+str(i))) - - with open(os.path.join(os.path.join(dir_output,'model_'+str(i)),"config.json"), "w") as fp: + model.fit( + train_gen, + steps_per_epoch=int(len(os.listdir(dir_flow_train_imgs)) / n_batch) - 1, + validation_data=val_gen, + validation_steps=1, + epochs=1, + callbacks=callbacks) + + dir_model = os.path.join(dir_output, 'model_' + str(i)) + model.save(dir_model) + with open(os.path.join(dir_model, "config.json"), "w") as fp: json.dump(_config, fp) # encode dict into JSON #os.system('rm -rf '+dir_train_flowing) #os.system('rm -rf '+dir_eval_flowing) #model.save(dir_output+'/'+'model'+'.h5') + elif task=='classification': configuration() - model = resnet50_classifier(n_classes, input_height, input_width, weight_decay, pretraining) + model = resnet50_classifier(n_classes, + input_height, + input_width, + weight_decay, + pretraining) - opt_adam = Adam(learning_rate=0.001) model.compile(loss='categorical_crossentropy', - optimizer = opt_adam,metrics=['accuracy']) - + optimizer=Adam(learning_rate=0.001), # rs: why not learning_rate? + metrics=['accuracy']) list_classes = list(classification_classes_name.values()) - testX, testY = generate_data_from_folder_evaluation(dir_eval, input_height, input_width, n_classes, list_classes) - - y_tot=np.zeros((testX.shape[0],n_classes)) + trainXY = generate_data_from_folder_training( + dir_train, n_batch, input_height, input_width, n_classes, list_classes) + testX, testY = generate_data_from_folder_evaluation( + dir_eval, input_height, input_width, n_classes, list_classes) + y_tot = np.zeros((testX.shape[0], n_classes)) score_best= [0] - num_rows = return_number_of_total_training_data(dir_train) weights=[] + callbacks = [TensorBoard(os.path.join(dir_output, 'logs'), write_graph=False)] for i in range(n_epochs): - history = model.fit( generate_data_from_folder_training(dir_train, n_batch , input_height, input_width, n_classes, list_classes), steps_per_epoch=num_rows / n_batch, verbose=1)#,class_weight=weights) - + history = model.fit(trainXY, + steps_per_epoch=num_rows / n_batch, + #class_weight=weights) + verbose=1, + callbacks=callbacks) y_pr_class = [] for jj in range(testY.shape[0]): y_pr=model.predict(testX[jj,:,:,:].reshape(1,input_height,input_width,3), verbose=0) @@ -433,7 +485,8 @@ def run(_config, n_classes, n_epochs, input_height, elif task=='reading_order': configuration() - model = machine_based_reading_order_model(n_classes,input_height,input_width,weight_decay,pretraining) + model = machine_based_reading_order_model( + n_classes, input_height, input_width, weight_decay, pretraining) dir_flow_train_imgs = os.path.join(dir_train, 'images') dir_flow_train_labels = os.path.join(dir_train, 'labels') @@ -447,20 +500,26 @@ def run(_config, n_classes, n_epochs, input_height, #f1score_tot = [0] indexer_start = 0 - # opt = SGD(learning_rate=0.01, momentum=0.9) - opt_adam = tf.keras.optimizers.Adam(learning_rate=0.0001) model.compile(loss="binary_crossentropy", - optimizer = opt_adam,metrics=['accuracy']) + #optimizer=SGD(learning_rate=0.01, momentum=0.9), + optimizer=Adam(learning_rate=0.0001), # rs: why not learning_rate? + metrics=['accuracy']) + callbacks = [TensorBoard(os.path.join(dir_output, 'logs'), write_graph=False)] if save_interval: - save_weights_callback = SaveWeightsAfterSteps(save_interval, dir_output, _config) - + callbacks.append(SaveWeightsAfterSteps(save_interval, dir_output, _config)) + + trainXY = generate_arrays_from_folder_reading_order( + dir_flow_train_labels, dir_flow_train_imgs, + n_batch, input_height, input_width, n_classes, + thetha, augmentation) + for i in range(n_epochs): - if save_interval: - history = model.fit(generate_arrays_from_folder_reading_order(dir_flow_train_labels, dir_flow_train_imgs, n_batch, input_height, input_width, n_classes, thetha, augmentation), steps_per_epoch=num_rows / n_batch, verbose=1, callbacks=[save_weights_callback]) - else: - history = model.fit(generate_arrays_from_folder_reading_order(dir_flow_train_labels, dir_flow_train_imgs, n_batch, input_height, input_width, n_classes, thetha, augmentation), steps_per_epoch=num_rows / n_batch, verbose=1) - model.save( os.path.join(dir_output,'model_'+str(i+indexer_start) )) + history = model.fit(trainXY, + steps_per_epoch=num_rows / n_batch, + verbose=1, + callbacks=callbacks) + model.save(os.path.join(dir_output, 'model_'+str(i+indexer_start) )) with open(os.path.join(os.path.join(dir_output,'model_'+str(i)),"config.json"), "w") as fp: json.dump(_config, fp) # encode dict into JSON diff --git a/src/eynollah/utils/contour.py b/src/eynollah/utils/contour.py index 7d01e74..c8caca9 100644 --- a/src/eynollah/utils/contour.py +++ b/src/eynollah/utils/contour.py @@ -12,7 +12,6 @@ from shapely import set_precision from shapely.ops import unary_union, nearest_points from .rotate import rotate_image, rotation_image_new -from . import ensure_array def contours_in_same_horizon(cy_main_hor): """ @@ -249,12 +248,14 @@ def return_contours_of_image(image): return contours, hierarchy def dilate_textline_contours(all_found_textline_polygons): + from . import ensure_array return [ensure_array( [polygon2contour(contour2polygon(contour, dilate=6)) for contour in region]) for region in all_found_textline_polygons] def dilate_textregion_contours(all_found_textregion_polygons): + from . import ensure_array return ensure_array( [polygon2contour(contour2polygon(contour, dilate=6)) for contour in all_found_textregion_polygons]) From 87d7ffbdd84283f0e2e6dca23d4d05431cf8bb3f Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Thu, 22 Jan 2026 11:25:00 +0100 Subject: [PATCH 03/21] training: use proper Keras callbacks and top-level loop --- ..._model_load_pretrained_weights_and_save.py | 10 -- src/eynollah/training/gt_gen_utils.py | 1 + src/eynollah/training/models.py | 3 + src/eynollah/training/train.py | 168 ++++++++---------- train/requirements.txt | 2 +- 5 files changed, 84 insertions(+), 100 deletions(-) diff --git a/src/eynollah/training/build_model_load_pretrained_weights_and_save.py b/src/eynollah/training/build_model_load_pretrained_weights_and_save.py index 9fba66b..15eaf64 100644 --- a/src/eynollah/training/build_model_load_pretrained_weights_and_save.py +++ b/src/eynollah/training/build_model_load_pretrained_weights_and_save.py @@ -1,17 +1,9 @@ import sys import click -import tensorflow as tf from .models import resnet50_unet -def configuration(): - try: - for device in tf.config.list_physical_devices('GPU'): - tf.config.experimental.set_memory_growth(device, True) - except: - print("no GPU device available", file=sys.stderr) - @click.command() def build_model_load_pretrained_weights_and_save(): n_classes = 2 @@ -21,8 +13,6 @@ def build_model_load_pretrained_weights_and_save(): pretraining = False dir_of_weights = 'model_bin_sbb_ens.h5' - # configuration() - model = resnet50_unet(n_classes, input_height, input_width, weight_decay, pretraining) model.load_weights(dir_of_weights) model.save('./name_in_another_python_version.h5') diff --git a/src/eynollah/training/gt_gen_utils.py b/src/eynollah/training/gt_gen_utils.py index 2e3428b..b7c35ee 100644 --- a/src/eynollah/training/gt_gen_utils.py +++ b/src/eynollah/training/gt_gen_utils.py @@ -653,6 +653,7 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ num_col = int(text_comments.split('num_col')[1]) comment_is_sub_element = True if not comment_is_sub_element: + # FIXME: look in /Page/@custom as well num_col = None if num_col: diff --git a/src/eynollah/training/models.py b/src/eynollah/training/models.py index fdc5437..3b38fe8 100644 --- a/src/eynollah/training/models.py +++ b/src/eynollah/training/models.py @@ -1,3 +1,6 @@ +import os + +os.environ['TF_USE_LEGACY_KERAS'] = '1' # avoid Keras 3 after TF 2.15 import tensorflow as tf from tensorflow import keras from tensorflow.keras.models import * diff --git a/src/eynollah/training/train.py b/src/eynollah/training/train.py index da901b0..7ee63f9 100644 --- a/src/eynollah/training/train.py +++ b/src/eynollah/training/train.py @@ -32,7 +32,7 @@ os.environ['TF_USE_LEGACY_KERAS'] = '1' # avoid Keras 3 after TF 2.15 import tensorflow as tf from tensorflow.keras.optimizers import SGD, Adam from tensorflow.keras.models import load_model -from tensorflow.keras.callbacks import Callback, TensorBoard +from tensorflow.keras.callbacks import ModelCheckpoint, TensorBoard from sacred import Experiment from tqdm import tqdm from sklearn.metrics import f1_score @@ -40,26 +40,28 @@ from sklearn.metrics import f1_score import numpy as np import cv2 -class SaveWeightsAfterSteps(Callback): - def __init__(self, save_interval, save_path, _config): - super(SaveWeightsAfterSteps, self).__init__() - self.save_interval = save_interval - self.save_path = save_path - self.step_count = 0 +class SaveWeightsAfterSteps(ModelCheckpoint): + def __init__(self, save_interval, save_path, _config, **kwargs): + if save_interval: + # batches + super().__init__( + os.path.join(save_path, "model_step_{batch:04d}"), + save_freq=save_interval, + verbose=1, + **kwargs) + else: + super().__init__( + os.path.join(save_path, "model_{epoch:02d}"), + save_freq="epoch", + verbose=1, + **kwargs) self._config = _config - def on_train_batch_end(self, batch, logs=None): - self.step_count += 1 - - if self.step_count % self.save_interval ==0: - save_file = f"{self.save_path}/model_step_{self.step_count}" - #os.system('mkdir '+save_file) - - self.model.save(save_file) - - with open(os.path.join(os.path.join(self.save_path, f"model_step_{self.step_count}"),"config.json"), "w") as fp: - json.dump(self._config, fp) # encode dict into JSON - print(f"saved model as steps {self.step_count} to {save_file}") + # overwrite tf-keras (Keras 2) implementation to get our _config JSON in + def _save_handler(self, filepath): + super()._save_handler(filepath) + with open(os.path.join(filepath, "config.json"), "w") as fp: + json.dump(self._config, fp) # encode dict into JSON def configuration(): @@ -396,23 +398,19 @@ def run(_config, n_classes, n_epochs, input_height, ##score_best=[] ##score_best.append(0) - callbacks = [TensorBoard(os.path.join(dir_output, 'logs'), write_graph=False)] + callbacks = [TensorBoard(os.path.join(dir_output, 'logs'), write_graph=False), + SaveWeightsAfterSteps(0, dir_output, _config)] if save_interval: callbacks.append(SaveWeightsAfterSteps(save_interval, dir_output, _config)) - for i in tqdm(range(index_start, n_epochs + index_start)): - model.fit( - train_gen, - steps_per_epoch=int(len(os.listdir(dir_flow_train_imgs)) / n_batch) - 1, - validation_data=val_gen, - validation_steps=1, - epochs=1, - callbacks=callbacks) - - dir_model = os.path.join(dir_output, 'model_' + str(i)) - model.save(dir_model) - with open(os.path.join(dir_model, "config.json"), "w") as fp: - json.dump(_config, fp) # encode dict into JSON + model.fit( + train_gen, + steps_per_epoch=len(os.listdir(dir_flow_train_imgs)) // n_batch - 1, + validation_data=val_gen, + #validation_steps=1, # rs: only one batch?? + validation_steps=len(os.listdir(dir_flow_eval_imgs)) // n_batch - 1, + epochs=n_epochs, + callbacks=callbacks) #os.system('rm -rf '+dir_train_flowing) #os.system('rm -rf '+dir_eval_flowing) @@ -434,54 +432,49 @@ def run(_config, n_classes, n_epochs, input_height, list_classes = list(classification_classes_name.values()) trainXY = generate_data_from_folder_training( dir_train, n_batch, input_height, input_width, n_classes, list_classes) - testX, testY = generate_data_from_folder_evaluation( + testXY = generate_data_from_folder_evaluation( dir_eval, input_height, input_width, n_classes, list_classes) y_tot = np.zeros((testX.shape[0], n_classes)) - score_best= [0] num_rows = return_number_of_total_training_data(dir_train) - weights=[] - callbacks = [TensorBoard(os.path.join(dir_output, 'logs'), write_graph=False)] + callbacks = [TensorBoard(os.path.join(dir_output, 'logs'), write_graph=False), + SaveWeightsAfterSteps(0, dir_output, _config, + monitor='val_f1', + save_best_only=True, mode='max')] - for i in range(n_epochs): - history = model.fit(trainXY, - steps_per_epoch=num_rows / n_batch, - #class_weight=weights) - verbose=1, - callbacks=callbacks) - y_pr_class = [] - for jj in range(testY.shape[0]): - y_pr=model.predict(testX[jj,:,:,:].reshape(1,input_height,input_width,3), verbose=0) - y_pr_ind= np.argmax(y_pr,axis=1) - y_pr_class.append(y_pr_ind) - - y_pr_class = np.array(y_pr_class) - f1score=f1_score(np.argmax(testY,axis=1), y_pr_class, average='macro') - print(i,f1score) - - if f1score>score_best[0]: - score_best[0]=f1score - model.save(os.path.join(dir_output,'model_best')) - - if f1score > f1_threshold_classification: - weights.append(model.get_weights() ) - + history = model.fit(trainXY, + steps_per_epoch=num_rows / n_batch, + #class_weight=weights) + validation_data=testXY, + verbose=1, + epochs=n_epochs, + metrics=[F1Score(average='macro', name='f1')], + callbacks=callbacks) - if len(weights) >= 1: - new_weights=list() - for weights_list_tuple in zip(*weights): - new_weights.append( [np.array(weights_).mean(axis=0) for weights_ in zip(*weights_list_tuple)] ) + usable_checkpoints = np.flatnonzero(np.array(history['val_f1']) > f1_threshold_classification) + if len(usable_checkpoints) >= 1: + print("averaging over usable checkpoints", usable_checkpoints) + all_weights = [] + for epoch in usable_checkpoints: + cp_path = os.path.join(dir_output, 'model_{epoch:02d}'.format(epoch=epoch)) + assert os.path.isdir(cp_path) + model = load_model(cp_path, compile=False) + all_weights.append(model.get_weights()) + + new_weights = [] + for layer_weights in zip(*all_weights): + layer_weights = np.array([np.array(weights).mean(axis=0) + for weights in zip(*layer_weights)]) + new_weights.append(layer_weights) - new_weights = [np.array(x) for x in new_weights] - model_weight_averaged=tf.keras.models.clone_model(model) - model_weight_averaged.set_weights(new_weights) - - model_weight_averaged.save(os.path.join(dir_output,'model_ens_avg')) - with open(os.path.join( os.path.join(dir_output,'model_ens_avg'), "config.json"), "w") as fp: + #model = tf.keras.models.clone_model(model) + model.set_weights(new_weights) + + cp_path = os.path.join(dir_output, 'model_ens_avg') + model.save(cp_path) + with open(os.path.join(cp_path, "config.json"), "w") as fp: json.dump(_config, fp) # encode dict into JSON - - with open(os.path.join( os.path.join(dir_output,'model_best'), "config.json"), "w") as fp: - json.dump(_config, fp) # encode dict into JSON + print("ensemble model saved under", cp_path) elif task=='reading_order': configuration() @@ -505,7 +498,8 @@ def run(_config, n_classes, n_epochs, input_height, optimizer=Adam(learning_rate=0.0001), # rs: why not learning_rate? metrics=['accuracy']) - callbacks = [TensorBoard(os.path.join(dir_output, 'logs'), write_graph=False)] + callbacks = [TensorBoard(os.path.join(dir_output, 'logs'), write_graph=False), + SaveWeightsAfterSteps(0, dir_output, _config)] if save_interval: callbacks.append(SaveWeightsAfterSteps(save_interval, dir_output, _config)) @@ -514,20 +508,16 @@ def run(_config, n_classes, n_epochs, input_height, n_batch, input_height, input_width, n_classes, thetha, augmentation) - for i in range(n_epochs): - history = model.fit(trainXY, - steps_per_epoch=num_rows / n_batch, - verbose=1, - callbacks=callbacks) - model.save(os.path.join(dir_output, 'model_'+str(i+indexer_start) )) - - with open(os.path.join(os.path.join(dir_output,'model_'+str(i)),"config.json"), "w") as fp: - json.dump(_config, fp) # encode dict into JSON - ''' - if f1score>f1score_tot[0]: - f1score_tot[0] = f1score - model_dir = os.path.join(dir_out,'model_best') - model.save(model_dir) - ''' + history = model.fit(trainXY, + steps_per_epoch=num_rows / n_batch, + verbose=1, + epochs=n_epochs, + callbacks=callbacks) + ''' + if f1score>f1score_tot[0]: + f1score_tot[0] = f1score + model_dir = os.path.join(dir_out,'model_best') + model.save(model_dir) + ''' diff --git a/train/requirements.txt b/train/requirements.txt index 63f3813..8ad884d 100644 --- a/train/requirements.txt +++ b/train/requirements.txt @@ -1,6 +1,6 @@ sacred seaborn -numpy <1.24.0 +numpy tqdm imutils scipy From 6a81db934e16971bc7edcf4b0b41a918dc444d5c Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Thu, 22 Jan 2026 11:25:50 +0100 Subject: [PATCH 04/21] improve docs/train.md --- docs/train.md | 168 ++++++++++++++++++++++++++++---------------------- 1 file changed, 96 insertions(+), 72 deletions(-) diff --git a/docs/train.md b/docs/train.md index 252bead..4e76740 100644 --- a/docs/train.md +++ b/docs/train.md @@ -9,9 +9,9 @@ on how to generate the corresponding training dataset. The following three tasks can all be accomplished using the code in the [`train`](https://github.com/qurator-spk/eynollah/tree/main/train) directory: -* generate training dataset -* train a model -* inference with the trained model +* [Generate training dataset](#generate-training-dataset) +* [Train a model](#train-a-model) +* [Inference with the trained model](#inference-with-the-trained-model) ## Training, evaluation and output @@ -63,7 +63,7 @@ serve as labels. The enhancement model can be trained with this generated datase For machine-based reading order, we aim to determine the reading priority between two sets of text regions. The model's input is a three-channel image: the first and last channels contain information about each of the two text regions, while the middle channel encodes prominent layout elements necessary for reading order, such as separators and headers. -To generate the training dataset, our script requires a page XML file that specifies the image layout with the correct +To generate the training dataset, our script requires a PAGE XML file that specifies the image layout with the correct reading order. For output images, it is necessary to specify the width and height. Additionally, a minimum text region size can be set @@ -82,8 +82,14 @@ eynollah-training generate-gt machine-based-reading-order \ ### pagexml2label -pagexml2label is designed to generate labels from GT page XML files for various pixel-wise segmentation use cases, -including 'layout,' 'textline,' 'printspace,' 'glyph,' and 'word' segmentation. +`pagexml2label` is designed to generate labels from PAGE XML GT files for various pixel-wise segmentation use cases, +including: +- `printspace` (i.e. page frame), +- `layout` (i.e. regions), +- `textline`, +- `word`, and +- `glyph`. + To train a pixel-wise segmentation model, we require images along with their corresponding labels. Our training script expects a PNG image where each pixel corresponds to a label, represented by an integer. The background is always labeled as zero, while other elements are assigned different integers. For instance, if we have ground truth data with four @@ -93,7 +99,7 @@ In binary segmentation scenarios such as textline or page extraction, the backgr element is automatically encoded as 1 in the PNG label. To specify the desired use case and the elements to be extracted in the PNG labels, a custom JSON file can be passed. -For example, in the case of 'textline' detection, the JSON file would resemble this: +For example, in the case of textline detection, the JSON contents could be this: ```yaml { @@ -101,61 +107,77 @@ For example, in the case of 'textline' detection, the JSON file would resemble t } ``` -In the case of layout segmentation a custom config json file can look like this: +In the case of layout segmentation, the config JSON file might look like this: ```yaml { "use_case": "layout", -"textregions":{"rest_as_paragraph":1 , "drop-capital": 1, "header":2, "heading":2, "marginalia":3}, -"imageregion":4, -"separatorregion":5, -"graphicregions" :{"rest_as_decoration":6 ,"stamp":7} +"textregions": {"rest_as_paragraph": 1, "drop-capital": 1, "header": 2, "heading": 2, "marginalia": 3}, +"imageregion": 4, +"separatorregion": 5, +"graphicregions": {"rest_as_decoration": 6, "stamp": 7} } ``` -A possible custom config json file for layout segmentation where the "printspace" is a class: +The same example if `PrintSpace` (or `Border`) should be represented as a unique class: ```yaml { "use_case": "layout", -"textregions":{"rest_as_paragraph":1 , "drop-capital": 1, "header":2, "heading":2, "marginalia":3}, -"imageregion":4, -"separatorregion":5, -"graphicregions" :{"rest_as_decoration":6 ,"stamp":7} -"printspace_as_class_in_layout" : 8 +"textregions": {"rest_as_paragraph": 1, "drop-capital": 1, "header": 2, "heading": 2, "marginalia": 3}, +"imageregion": 4, +"separatorregion": 5, +"graphicregions": {"rest_as_decoration": 6, "stamp": 7} +"printspace_as_class_in_layout": 8 } ``` -For the layout use case, it is beneficial to first understand the structure of the page XML file and its elements. -In a given image, the annotations of elements are recorded in a page XML file, including their contours and classes. -For an image document, the known regions are 'textregion', 'separatorregion', 'imageregion', 'graphicregion', -'noiseregion', and 'tableregion'. +In the `layout` use-case, it is beneficial to first understand the structure of the PAGE XML file and its elements. +For a given page image, the visible segments are annotated in XML with their polygon coordinates and types. +On the region level, available segment types include `TextRegion`, `SeparatorRegion`, `ImageRegion`, `GraphicRegion`, +`NoiseRegion` and `TableRegion`. -Text regions and graphic regions also have their own specific types. The known types for text regions are 'paragraph', -'header', 'heading', 'marginalia', 'drop-capital', 'footnote', 'footnote-continued', 'signature-mark', 'page-number', -and 'catch-word'. The known types for graphic regions are 'handwritten-annotation', 'decoration', 'stamp', and -'signature'. -Since we don't know all types of text and graphic regions, unknown cases can arise. To handle these, we have defined -two additional types, "rest_as_paragraph" and "rest_as_decoration", to ensure that no unknown types are missed. -This way, users can extract all known types from the labels and be confident that no unknown types are overlooked. +Moreover, text regions and graphic regions in particular are subdivided via `@type`: +- The allowed subtypes for text regions are `paragraph`, `heading`, `marginalia`, `drop-capital`, `header`, `footnote`, +`footnote-continued`, `signature-mark`, `page-number` and `catch-word`. +- The known subtypes for graphic regions are `handwritten-annotation`, `decoration`, `stamp` and `signature`. -In the custom JSON file shown above, "header" and "heading" are extracted as the same class, while "marginalia" is shown -as a different class. All other text region types, including "drop-capital," are grouped into the same class. For the -graphic region, "stamp" has its own class, while all other types are classified together. "Image region" and "separator -region" are also present in the label. However, other regions like "noise region" and "table region" will not be -included in the label PNG file, even if they have information in the page XML files, as we chose not to include them. +These types and subtypes must be mapped to classes for the segmentation model. However, sometimes these fine-grained +distinctions are not useful or the existing annotations are not very usable (too scarce or too unreliable). +In that case, instead of these subtypes with a specific mapping, they can be pooled together by using the two special +types: +- `rest_as_paragraph` (mapping missing TextRegion subtypes and `paragraph`) +- `rest_as_decoration` (mapping missing GraphicRegion subtypes and `decoration`) + +(That way, users can extract all known types from the labels and be confident that no subtypes are overlooked.) + +In the custom JSON example shown above, `header` and `heading` are extracted as the same class, +while `marginalia` is modelled as a different class. All other text region types, including `drop-capital`, +are grouped into the same class. For graphic regions, `stamp` has its own class, while all other types +are classified together. `ImageRegion` and `SeparatorRegion` will also represented with a class label in the +training data. However, other regions like `NoiseRegion` or `TableRegion` will not be included in the PNG files, +even if they were present in the PAGE XML. + +The tool expects various command-line options: ```sh eynollah-training generate-gt pagexml2label \ - -dx "dir of GT xml files" \ - -do "dir where output label png files will be written" \ - -cfg "custom config json file" \ - -to "output type which has 2d and 3d. 2d is used for training and 3d is just to visualise the labels" + -dx "dir of input PAGE XML files" \ + -do "dir of output label PNG files" \ + -cfg "custom config JSON file" \ + -to "output type (2d or 3d)" ``` -We have also defined an artificial class that can be added to the boundary of text region types or text lines. This key -is called "artificial_class_on_boundary." If users want to apply this to certain text regions in the layout use case, -the example JSON config file should look like this: +As output type, use +- `2d` for training, +- `3d` to just visualise the labels. + +We have also defined an artificial class that can be added to (rendered around) the boundary +of text region types or text lines in order to make separation of neighbouring segments more +reliable. The key is called `artificial_class_on_boundary`, and it takes a list of text region +types to be applied to. + +Our example JSON config file could then look like this: ```yaml { @@ -177,14 +199,15 @@ the example JSON config file should look like this: } ``` -This implies that the artificial class label, denoted by 7, will be present on PNG files and will only be added to the -elements labeled as "paragraph," "header," "heading," and "marginalia." +This implies that the artificial class label (denoted by 7) will be present in the generated PNG files +and will only be added around segments labeled `paragraph`, `header`, `heading` or `marginalia`. (This +class will be handled specially during decoding at inference, and not show up in final results.) -For "textline", "word", and "glyph", the artificial class on the boundaries will be activated only if the -"artificial_class_label" key is specified in the config file. Its value should be set as 2 since these elements -represent binary cases. For example, if the background and textline are denoted as 0 and 1 respectively, then the -artificial class should be assigned the value 2. The example JSON config file should look like this for "textline" use -case: +For `printspace`, `textline`, `word`, and `glyph` segmentation use-cases, there is no `artificial_class_on_boundary` key, +but `artificial_class_label` is available. If specified in the config file, then its value should be set at 2, because +these elements represent binary classification problems (with background represented as 0, and segments as 1, respectively). + +For example, the JSON config for textline detection could look as follows: ```yaml { @@ -193,33 +216,33 @@ case: } ``` -If the coordinates of "PrintSpace" or "Border" are present in the page XML ground truth files, and the user wishes to -crop only the print space area, this can be achieved by activating the "-ps" argument. However, it should be noted that -in this scenario, since cropping will be applied to the label files, the directory of the original images must be -provided to ensure that they are cropped in sync with the labels. This ensures that the correct images and labels -required for training are obtained. The command should resemble the following: +If the coordinates of `PrintSpace` (or `Border`) are present in the PAGE XML ground truth files, +and one wishes to crop images to only cover the print space bounding box, this can be achieved +by passing the `-ps` option. Note that in this scenario, the directory of the original images +must also be provided, to ensure that the images are cropped in sync with the labels. The command +line would then resemble this: ```sh eynollah-training generate-gt pagexml2label \ - -dx "dir of GT xml files" \ - -do "dir where output label png files will be written" \ - -cfg "custom config json file" \ - -to "output type which has 2d and 3d. 2d is used for training and 3d is just to visualise the labels" \ + -dx "dir of input PAGE XML files" \ + -do "dir of output label PNG files" \ + -cfg "custom config JSON file" \ + -to "output type (2d or 3d)" \ -ps \ - -di "dir where the org images are located" \ - -doi "dir where the cropped output images will be written" + -di "dir of input original images" \ + -doi "dir of output cropped images" ``` ## Train a model ### classification -For the classification use case, we haven't provided a ground truth generator, as it's unnecessary. For classification, -all we require is a training directory with subdirectories, each containing images of its respective classes. We need +For the image classification use-case, we have not provided a ground truth generator, as it is unnecessary. +All we require is a training directory with subdirectories, each containing images of its respective classes. We need separate directories for training and evaluation, and the class names (subdirectories) must be consistent across both directories. Additionally, the class names should be specified in the config JSON file, as shown in the following example. If, for instance, we aim to classify "apple" and "orange," with a total of 2 classes, the -"classification_classes_name" key in the config file should appear as follows: +`classification_classes_name` key in the config file should appear as follows: ```yaml { @@ -241,7 +264,7 @@ example. If, for instance, we aim to classify "apple" and "orange," with a total } ``` -The "dir_train" should be like this: +Then `dir_train` should be like this: ``` . @@ -250,7 +273,7 @@ The "dir_train" should be like this: └── orange # directory of images for orange class ``` -And the "dir_eval" the same structure as train directory: +And `dir_eval` analogously: ``` . @@ -310,7 +333,7 @@ And the "dir_eval" the same structure as train directory: └── labels # directory of labels ``` -The classification model can be trained like the classification case command line. +The reading-order model can be trained like the classification case command line. ### Segmentation (Textline, Binarization, Page extraction and layout) and enhancement @@ -374,9 +397,9 @@ classification and machine-based reading order, as you can see in their example * `transformer_num_heads`: Transformer number of heads. Default value is 4. * `transformer_cnn_first`: We have two types of vision transformers. In one type, a CNN is applied first, followed by a transformer. In the other type, this order is reversed. If transformer_cnn_first is true, it means the CNN will be applied before the transformer. Default value is true. -In the case of segmentation and enhancement the train and evaluation directory should be as following. +In case of segmentation and enhancement the train and evaluation data should be organised as follows. -The "dir_train" should be like this: +The "dir_train" directory should be like this: ``` . @@ -394,11 +417,12 @@ And the "dir_eval" the same structure as train directory: └── labels # directory of labels ``` -After configuring the JSON file for segmentation or enhancement, training can be initiated by running the following -command, similar to the process for classification and reading order: +After configuring the JSON file for segmentation or enhancement, +training can be initiated by running the following command line, +similar to classification and reading-order model training: -``` -eynollah-training train with config_classification.json` +```sh +eynollah-training train with config_classification.json ``` #### Binarization @@ -690,7 +714,7 @@ This will straightforwardly return the class of the image. ### machine based reading order -To infer the reading order using a reading order model, we need a page XML file containing layout information but +To infer the reading order using a reading order model, we need a PAGE XML file containing layout information but without the reading order. We simply need to provide the model directory, the XML file, and the output directory. The new XML file with the added reading order will be written to the output directory with the same name. We need to run: From eb92760f73f9d8eefa9028ea697c4152d07e39ec Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Thu, 22 Jan 2026 19:49:39 +0100 Subject: [PATCH 05/21] training: download pretrained RESNET weights if missing --- src/eynollah/training/models.py | 17 ++++++++++------- src/eynollah/training/train.py | 15 ++++++++++++++- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/src/eynollah/training/models.py b/src/eynollah/training/models.py index 3b38fe8..011c614 100644 --- a/src/eynollah/training/models.py +++ b/src/eynollah/training/models.py @@ -12,7 +12,10 @@ from tensorflow.keras.regularizers import l2 ###projection_dim = 64 ##transformer_layers = 2#8 ##num_heads = 1#4 -resnet50_Weights_path = './pretrained_model/resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5' +RESNET50_WEIGHTS_PATH = './pretrained_model/resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5' +RESNET50_WEIGHTS_URL = ('https://github.com/fchollet/deep-learning-models/releases/download/v0.2/' + 'resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5') + IMAGE_ORDERING = 'channels_last' MERGE_AXIS = -1 @@ -242,7 +245,7 @@ def resnet50_unet_light(n_classes, input_height=224, input_width=224, taks="segm f5 = x if pretraining: - model = Model(img_input, x).load_weights(resnet50_Weights_path) + model = Model(img_input, x).load_weights(RESNET50_WEIGHTS_PATH) v512_2048 = Conv2D(512, (1, 1), padding='same', data_format=IMAGE_ORDERING, kernel_regularizer=l2(weight_decay))(f5) v512_2048 = (BatchNormalization(axis=bn_axis))(v512_2048) @@ -343,7 +346,7 @@ def resnet50_unet(n_classes, input_height=224, input_width=224, task="segmentati f5 = x if pretraining: - Model(img_input, x).load_weights(resnet50_Weights_path) + Model(img_input, x).load_weights(RESNET50_WEIGHTS_PATH) v1024_2048 = Conv2D(1024, (1, 1), padding='same', data_format=IMAGE_ORDERING, kernel_regularizer=l2(weight_decay))( f5) @@ -442,7 +445,7 @@ def vit_resnet50_unet(n_classes, patch_size_x, patch_size_y, num_patches, mlp_he f5 = x if pretraining: - model = Model(inputs, x).load_weights(resnet50_Weights_path) + model = Model(inputs, x).load_weights(RESNET50_WEIGHTS_PATH) #num_patches = x.shape[1]*x.shape[2] @@ -590,7 +593,7 @@ def vit_resnet50_unet_transformer_before_cnn(n_classes, patch_size_x, patch_size f5 = x if pretraining: - model = Model(encoded_patches, x).load_weights(resnet50_Weights_path) + model = Model(encoded_patches, x).load_weights(RESNET50_WEIGHTS_PATH) v1024_2048 = Conv2D( 1024 , (1, 1), padding='same', data_format=IMAGE_ORDERING,kernel_regularizer=l2(weight_decay))(x) v1024_2048 = (BatchNormalization(axis=bn_axis))(v1024_2048) @@ -690,7 +693,7 @@ def resnet50_classifier(n_classes,input_height=224,input_width=224,weight_decay= f5 = x if pretraining: - Model(img_input, x).load_weights(resnet50_Weights_path) + Model(img_input, x).load_weights(RESNET50_WEIGHTS_PATH) x = AveragePooling2D((7, 7), name='avg_pool')(x) x = Flatten()(x) @@ -746,7 +749,7 @@ def machine_based_reading_order_model(n_classes,input_height=224,input_width=224 x1 = identity_block(x1, 3, [512, 512, 2048], stage=5, block='c') if pretraining: - Model(img_input , x1).load_weights(resnet50_Weights_path) + Model(img_input , x1).load_weights(RESNET50_WEIGHTS_PATH) x1 = AveragePooling2D((7, 7), name='avg_pool1')(x1) flattened = Flatten()(x1) diff --git a/src/eynollah/training/train.py b/src/eynollah/training/train.py index 7ee63f9..6353474 100644 --- a/src/eynollah/training/train.py +++ b/src/eynollah/training/train.py @@ -2,6 +2,7 @@ import os import sys import json +import requests import click from eynollah.training.metrics import ( @@ -15,7 +16,9 @@ from eynollah.training.models import ( resnet50_classifier, resnet50_unet, vit_resnet50_unet, - vit_resnet50_unet_transformer_before_cnn + vit_resnet50_unet_transformer_before_cnn, + RESNET50_WEIGHTS_PATH, + RESNET50_WEIGHTS_URL ) from eynollah.training.utils import ( data_gen, @@ -80,6 +83,12 @@ def get_dirs_or_files(input_data): assert os.path.isdir(labels_input), "{} is not a directory".format(labels_input) return image_input, labels_input +def download_file(url, path): + with open(path, 'wb') as f: + with requests.get(url, stream=True) as r: + r.raise_for_status() + for data in r.iter_content(chunk_size=4096): + f.write(data) ex = Experiment(save_git_info=False) @@ -163,6 +172,10 @@ def run(_config, n_classes, n_epochs, input_height, transformer_patchsize_x, transformer_patchsize_y, transformer_num_patches_xy, backbone_type, save_interval, flip_index, dir_eval, dir_output, pretraining, learning_rate, task, f1_threshold_classification, classification_classes_name, dir_img_bin, number_of_backgrounds_per_image,dir_rgb_backgrounds, dir_rgb_foregrounds): + + if pretraining and not os.path.isfile(RESNET50_WEIGHTS_PATH): + print("downloading RESNET50 pretrained weights to", RESNET50_WEIGHTS_PATH) + download_file(RESNET50_WEIGHTS_URL, RESNET50_WEIGHTS_PATH) if dir_rgb_backgrounds: list_all_possible_background_images = os.listdir(dir_rgb_backgrounds) From acda9c84eecca75e5260b2172923f59e86838a73 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Wed, 28 Jan 2026 13:28:03 +0100 Subject: [PATCH 06/21] =?UTF-8?q?training.gt=5Fgen=5Futils:=20improve=20XM?= =?UTF-8?q?L=E2=86=92img=20path=20mapping=E2=80=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit when matching files in `dir_images` by XML path name stem, * use `dict` instead of `list` to assign reliably * filter out `.xml` files (so input directories can be mixed) * show informative warnings for files which cannot be matched --- src/eynollah/training/gt_gen_utils.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/src/eynollah/training/gt_gen_utils.py b/src/eynollah/training/gt_gen_utils.py index b7c35ee..f4defdd 100644 --- a/src/eynollah/training/gt_gen_utils.py +++ b/src/eynollah/training/gt_gen_utils.py @@ -627,7 +627,10 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ if dir_images: ls_org_imgs = os.listdir(dir_images) - ls_org_imgs_stem = [os.path.splitext(item)[0] for item in ls_org_imgs] + ls_org_imgs = {os.path.splitext(item)[0]: item + for item in ls_org_imgs + if not item.endswith('.xml')} + for index in tqdm(range(len(gt_list))): #try: print(gt_list[index]) @@ -802,7 +805,13 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ cv2.imwrite(os.path.join(output_dir, xml_file_stem + '.png'), img_poly) if dir_images: - org_image_name = ls_org_imgs[ls_org_imgs_stem.index(xml_file_stem)] + org_image_name = ls_org_imgs[xml_file_stem] + if not org_image_name: + print("image file for XML stem", xml_file_stem, "is missing") + continue + if not os.path.isfile(os.path.join(dir_images, org_image_name)): + print("image file for XML stem", xml_file_stem, "is not readable") + continue img_org = cv2.imread(os.path.join(dir_images, org_image_name)) if printspace and config_params['use_case']!='printspace': @@ -1266,7 +1275,13 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ if dir_images: - org_image_name = ls_org_imgs[ls_org_imgs_stem.index(xml_file_stem)] + org_image_name = ls_org_imgs[xml_file_stem] + if not org_image_name: + print("image file for XML stem", xml_file_stem, "is missing") + continue + if not os.path.isfile(os.path.join(dir_images, org_image_name)): + print("image file for XML stem", xml_file_stem, "is not readable") + continue img_org = cv2.imread(os.path.join(dir_images, org_image_name)) if printspace: From 0372fd7a1ec2e4d654c0f24171c9b30c77a3e09b Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Wed, 28 Jan 2026 13:42:59 +0100 Subject: [PATCH 07/21] =?UTF-8?q?training.gt=5Fgen=5Futils:=20fix+simplify?= =?UTF-8?q?=20cropping=E2=80=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit when parsing `PrintSpace` or `Border` from PAGE-XML, - use `lxml` XPath instead of nested loops - convert points to polygons directly (instead of painting on canvas and retrieving contours) - pass result bbox in slice notation (instead of xywh) --- src/eynollah/training/gt_gen_utils.py | 151 ++++++++------------------ src/eynollah/training/inference.py | 18 ++- 2 files changed, 51 insertions(+), 118 deletions(-) diff --git a/src/eynollah/training/gt_gen_utils.py b/src/eynollah/training/gt_gen_utils.py index f4defdd..f068afd 100644 --- a/src/eynollah/training/gt_gen_utils.py +++ b/src/eynollah/training/gt_gen_utils.py @@ -1,15 +1,18 @@ import os import numpy as np import warnings -import xml.etree.ElementTree as ET +from lxml import etree as ET from tqdm import tqdm import cv2 from shapely import geometry from pathlib import Path from PIL import ImageFont +from ocrd_utils import bbox_from_points KERNEL = np.ones((5, 5), np.uint8) +NS = { 'pc': 'http://schema.primaresearch.org/PAGE/gts/pagecontent/2019-07-15' +} with warnings.catch_warnings(): warnings.simplefilter("ignore") @@ -664,52 +667,13 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ y_new = int ( x_new * (y_len / float(x_len)) ) if printspace or "printspace_as_class_in_layout" in list(config_params.keys()): - region_tags = np.unique([x for x in alltags if x.endswith('PrintSpace') or x.endswith('Border')]) - co_use_case = [] - - for tag in region_tags: - tag_endings = ['}PrintSpace','}Border'] - - if tag.endswith(tag_endings[0]) or tag.endswith(tag_endings[1]): - for nn in root1.iter(tag): - c_t_in = [] - sumi = 0 - for vv in nn.iter(): - # check the format of coords - if vv.tag == link + 'Coords': - coords = bool(vv.attrib) - if coords: - p_h = vv.attrib['points'].split(' ') - c_t_in.append( - np.array([[int(x.split(',')[0]), int(x.split(',')[1])] for x in p_h])) - break - else: - pass - - if vv.tag == link + 'Point': - c_t_in.append([int(float(vv.attrib['x'])), int(float(vv.attrib['y']))]) - sumi += 1 - elif vv.tag != link + 'Point' and sumi >= 1: - break - co_use_case.append(np.array(c_t_in)) - - img = np.zeros((y_len, x_len, 3)) - - img_poly = cv2.fillPoly(img, pts=co_use_case, color=(1, 1, 1)) - - img_poly = img_poly.astype(np.uint8) - - imgray = cv2.cvtColor(img_poly, cv2.COLOR_BGR2GRAY) - _, thresh = cv2.threshold(imgray, 0, 255, 0) - - contours, _ = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) - - cnt_size = np.array([cv2.contourArea(contours[j]) for j in range(len(contours))]) - - cnt = contours[np.argmax(cnt_size)] - - x, y, w, h = cv2.boundingRect(cnt) - bb_xywh = [x, y, w, h] + ps = (root1.xpath('/pc:PcGts/pc:Page/pc:Border', namespaces=NS) + + root1.xpath('/pc:PcGts/pc:Page/pc:PrintSpace', namespaces=NS)) + if len(ps): + points = ps[0].find('pc:Coords', NS).get('points') + ps_bbox = bbox_from_points(points) + else: + ps_bbox = [0, 0, None, None] if config_file and (config_params['use_case']=='textline' or config_params['use_case']=='word' or config_params['use_case']=='glyph' or config_params['use_case']=='printspace'): @@ -791,7 +755,8 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ if printspace and config_params['use_case']!='printspace': - img_poly = img_poly[bb_xywh[1]:bb_xywh[1]+bb_xywh[3], bb_xywh[0]:bb_xywh[0]+bb_xywh[2], :] + img_poly = img_poly[ps_bbox[1]:ps_bbox[3], + ps_bbox[0]:ps_bbox[2], :] if 'columns_width' in list(config_params.keys()) and num_col and config_params['use_case']!='printspace': @@ -815,7 +780,8 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ img_org = cv2.imread(os.path.join(dir_images, org_image_name)) if printspace and config_params['use_case']!='printspace': - img_org = img_org[bb_xywh[1]:bb_xywh[1]+bb_xywh[3], bb_xywh[0]:bb_xywh[0]+bb_xywh[2], :] + img_org = img_org[ps_bbox[1]:ps_bbox[3], + ps_bbox[0]:ps_bbox[2], :] if 'columns_width' in list(config_params.keys()) and num_col and config_params['use_case']!='printspace': img_org = resize_image(img_org, y_new, x_new) @@ -1194,7 +1160,8 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ if "printspace_as_class_in_layout" in list(config_params.keys()): printspace_mask = np.zeros((img_poly.shape[0], img_poly.shape[1])) - printspace_mask[bb_xywh[1]:bb_xywh[1]+bb_xywh[3], bb_xywh[0]:bb_xywh[0]+bb_xywh[2]] = 1 + printspace_mask[ps_bbox[1]:ps_bbox[3], + ps_bbox[0]:ps_bbox[2]] = 1 img_poly[:,:,0][printspace_mask[:,:] == 0] = printspace_class_rgb_color[0] img_poly[:,:,1][printspace_mask[:,:] == 0] = printspace_class_rgb_color[1] @@ -1252,7 +1219,8 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ if "printspace_as_class_in_layout" in list(config_params.keys()): printspace_mask = np.zeros((img_poly.shape[0], img_poly.shape[1])) - printspace_mask[bb_xywh[1]:bb_xywh[1]+bb_xywh[3], bb_xywh[0]:bb_xywh[0]+bb_xywh[2]] = 1 + printspace_mask[ps_bbox[1]:ps_bbox[3], + ps_bbox[0]:ps_bbox[2]] = 1 img_poly[:,:,0][printspace_mask[:,:] == 0] = printspace_class_label img_poly[:,:,1][printspace_mask[:,:] == 0] = printspace_class_label @@ -1261,7 +1229,8 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ if printspace: - img_poly = img_poly[bb_xywh[1]:bb_xywh[1]+bb_xywh[3], bb_xywh[0]:bb_xywh[0]+bb_xywh[2], :] + img_poly = img_poly[ps_bbox[1]:ps_bbox[3], + ps_bbox[0]:ps_bbox[2], :] if 'columns_width' in list(config_params.keys()) and num_col: img_poly = resize_image(img_poly, y_new, x_new) @@ -1285,7 +1254,8 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ img_org = cv2.imread(os.path.join(dir_images, org_image_name)) if printspace: - img_org = img_org[bb_xywh[1]:bb_xywh[1]+bb_xywh[3], bb_xywh[0]:bb_xywh[0]+bb_xywh[2], :] + img_org = img_org[ps_bbox[1]:ps_bbox[3], + ps_bbox[0]:ps_bbox[2], :] if 'columns_width' in list(config_params.keys()) and num_col: img_org = resize_image(img_org, y_new, x_new) @@ -1326,6 +1296,7 @@ def find_new_features_of_contours(contours_main): y_max_main = np.array([np.max(contours_main[j][:, 1]) for j in range(len(contours_main))]) return cx_main, cy_main, x_min_main, x_max_main, y_min_main, y_max_main, y_corr_x_min_from_argmin + def read_xml(xml_file): file_name = Path(xml_file).stem tree1 = ET.parse(xml_file, parser = ET.XMLParser(encoding='utf-8')) @@ -1344,57 +1315,13 @@ def read_xml(xml_file): index_tot_regions.append(jj.attrib['index']) tot_region_ref.append(jj.attrib['regionRef']) - if (link+'PrintSpace' in alltags) or (link+'Border' in alltags): - co_printspace = [] - if link+'PrintSpace' in alltags: - region_tags_printspace = np.unique([x for x in alltags if x.endswith('PrintSpace')]) - elif link+'Border' in alltags: - region_tags_printspace = np.unique([x for x in alltags if x.endswith('Border')]) - - for tag in region_tags_printspace: - if link+'PrintSpace' in alltags: - tag_endings_printspace = ['}PrintSpace','}printspace'] - elif link+'Border' in alltags: - tag_endings_printspace = ['}Border','}border'] - - if tag.endswith(tag_endings_printspace[0]) or tag.endswith(tag_endings_printspace[1]): - for nn in root1.iter(tag): - c_t_in = [] - sumi = 0 - for vv in nn.iter(): - # check the format of coords - if vv.tag == link + 'Coords': - coords = bool(vv.attrib) - if coords: - p_h = vv.attrib['points'].split(' ') - c_t_in.append( - np.array([[int(x.split(',')[0]), int(x.split(',')[1])] for x in p_h])) - break - else: - pass - - if vv.tag == link + 'Point': - c_t_in.append([int(float(vv.attrib['x'])), int(float(vv.attrib['y']))]) - sumi += 1 - elif vv.tag != link + 'Point' and sumi >= 1: - break - co_printspace.append(np.array(c_t_in)) - img_printspace = np.zeros( (y_len,x_len,3) ) - img_printspace=cv2.fillPoly(img_printspace, pts =co_printspace, color=(1,1,1)) - img_printspace = img_printspace.astype(np.uint8) - - imgray = cv2.cvtColor(img_printspace, cv2.COLOR_BGR2GRAY) - _, thresh = cv2.threshold(imgray, 0, 255, 0) - contours, _ = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) - cnt_size = np.array([cv2.contourArea(contours[j]) for j in range(len(contours))]) - cnt = contours[np.argmax(cnt_size)] - x, y, w, h = cv2.boundingRect(cnt) - - bb_coord_printspace = [x, y, w, h] - + ps = (root1.xpath('/pc:PcGts/pc:Page/pc:Border', namespaces=NS) + + root1.xpath('/pc:PcGts/pc:Page/pc:PrintSpace', namespaces=NS)) + if len(ps): + points = ps[0].find('pc:Coords', NS).get('points') + ps_bbox = bbox_from_points(points) else: - bb_coord_printspace = None - + ps_bbox = [0, 0, None, None] region_tags=np.unique([x for x in alltags if x.endswith('Region')]) co_text_paragraph=[] @@ -1749,11 +1676,19 @@ def read_xml(xml_file): img_poly=cv2.fillPoly(img, pts =co_img, color=(4,4,4)) img_poly=cv2.fillPoly(img, pts =co_sep, color=(5,5,5)) - return tree1, root1, bb_coord_printspace, file_name, id_paragraph, id_header+id_heading, co_text_paragraph, co_text_header+co_text_heading,\ -tot_region_ref,x_len, y_len,index_tot_regions, img_poly - - - + return (tree1, + root1, + ps_bbox, + file_name, + id_paragraph, + id_header + id_heading, + co_text_paragraph, + co_text_header + co_text_heading, + tot_region_ref, + x_len, + y_len, + index_tot_regions, + img_poly) def bounding_box(cnt,color, corr_order_index ): x, y, w, h = cv2.boundingRect(cnt) diff --git a/src/eynollah/training/inference.py b/src/eynollah/training/inference.py index 15d1e6a..2ef1a91 100644 --- a/src/eynollah/training/inference.py +++ b/src/eynollah/training/inference.py @@ -196,7 +196,7 @@ class SBBPredict: img_height = self.config_params_model['input_height'] img_width = self.config_params_model['input_width'] - tree_xml, root_xml, bb_coord_printspace, file_name, \ + tree_xml, root_xml, ps_bbox, file_name, \ id_paragraph, id_header, \ co_text_paragraph, co_text_header, \ tot_region_ref, x_len, y_len, index_tot_regions, \ @@ -236,15 +236,13 @@ class SBBPredict: img_label=cv2.fillPoly(img_label, pts =[co_text_all[i]], color=(1,1,1)) labels_con[:,:,i] = img_label[:,:,0] - if bb_coord_printspace: - #bb_coord_printspace[x,y,w,h,_,_] - x = bb_coord_printspace[0] - y = bb_coord_printspace[1] - w = bb_coord_printspace[2] - h = bb_coord_printspace[3] - labels_con = labels_con[y:y+h, x:x+w, :] - img_poly = img_poly[y:y+h, x:x+w, :] - img_header_and_sep = img_header_and_sep[y:y+h, x:x+w] + if ps_bbox: + labels_con = labels_con[ps_bbox[1]:ps_bbox[3], + ps_bbox[0]:ps_bbox[2], :] + img_poly = img_poly[ps_bbox[1]:ps_bbox[3], + ps_bbox[0]:ps_bbox[2], :] + img_header_and_sep = img_header_and_sep[ps_bbox[1]:ps_bbox[3], + ps_bbox[0]:ps_bbox[2]] From e69b35b49c4e7816b0e88d0d5d48f79aaf3f46db Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Wed, 28 Jan 2026 13:49:23 +0100 Subject: [PATCH 08/21] training.train.config_params: re-organise to reflect dependencies - re-order keys belonging together logically - make keys dependent on each other --- src/eynollah/training/train.py | 222 +++++++++++++++++---------------- 1 file changed, 115 insertions(+), 107 deletions(-) diff --git a/src/eynollah/training/train.py b/src/eynollah/training/train.py index 6353474..e93281a 100644 --- a/src/eynollah/training/train.py +++ b/src/eynollah/training/train.py @@ -95,136 +95,144 @@ ex = Experiment(save_git_info=False) @ex.config def config_params(): + task = "segmentation" # This parameter defines task of model which can be segmentation, enhancement or classification. + backbone_type = None # Type of image feature map network backbone. Either a vision transformer alongside a CNN we call "transformer", or only a CNN which we call "nontransformer" n_classes = None # Number of classes. In the case of binary classification this should be 2. - n_epochs = 1 # Number of epochs. + n_epochs = 1 # Number of epochs to train. + n_batch = 1 # Number of images per batch at each iteration. (Try as large as fits on VRAM.) input_height = 224 * 1 # Height of model's input in pixels. input_width = 224 * 1 # Width of model's input in pixels. weight_decay = 1e-6 # Weight decay of l2 regularization of model layers. - n_batch = 1 # Number of batches at each iteration. learning_rate = 1e-4 # Set the learning rate. - patches = False # Divides input image into smaller patches (input size of the model) when set to true. For the model to see the full image, like page extraction, set this to false. - augmentation = False # To apply any kind of augmentation, this parameter must be set to true. - flip_aug = False # If true, different types of flipping will be applied to the image. Types of flips are defined with "flip_index" in config_params.json. - blur_aug = False # If true, different types of blurring will be applied to the image. Types of blur are defined with "blur_k" in config_params.json. - padding_white = False # If true, white padding will be applied to the image. - padding_black = False # If true, black padding will be applied to the image. - scaling = False # If true, scaling will be applied to the image. The amount of scaling is defined with "scales" in config_params.json. - shifting = False - degrading = False # If true, degrading will be applied to the image. The amount of degrading is defined with "degrade_scales" in config_params.json. - brightening = False # If true, brightening will be applied to the image. The amount of brightening is defined with "brightness" in config_params.json. - binarization = False # If true, Otsu thresholding will be applied to augment the input with binarized images. - adding_rgb_background = False - adding_rgb_foreground = False - add_red_textlines = False - channels_shuffling = False - dir_train = None # Directory of training dataset with subdirectories having the names "images" and "labels". - dir_eval = None # Directory of validation dataset with subdirectories having the names "images" and "labels". - dir_output = None # Directory where the output model will be saved. - pretraining = False # Set to true to load pretrained weights of ResNet50 encoder. - scaling_bluring = False # If true, a combination of scaling and blurring will be applied to the image. - scaling_binarization = False # If true, a combination of scaling and binarization will be applied to the image. - rotation = False # If true, a 90 degree rotation will be implemeneted. - rotation_not_90 = False # If true rotation based on provided angles with thetha will be implemeneted. - scaling_brightness = False # If true, a combination of scaling and brightening will be applied to the image. - scaling_flip = False # If true, a combination of scaling and flipping will be applied to the image. - thetha = None # Rotate image by these angles for augmentation. - shuffle_indexes = None - blur_k = None # Blur image for augmentation. - scales = None # Scale patches for augmentation. - degrade_scales = None # Degrade image for augmentation. - brightness = None # Brighten image for augmentation. - flip_index = None # Flip image for augmentation. - continue_training = False # Set to true if you would like to continue training an already trained a model. - transformer_patchsize_x = None # Patch size of vision transformer patches in x direction. - transformer_patchsize_y = None # Patch size of vision transformer patches in y direction. - transformer_num_patches_xy = None # Number of patches for vision transformer in x and y direction respectively. - transformer_projection_dim = 64 # Transformer projection dimension. Default value is 64. - transformer_mlp_head_units = [128, 64] # Transformer Multilayer Perceptron (MLP) head units. Default value is [128, 64] - transformer_layers = 8 # transformer layers. Default value is 8. - transformer_num_heads = 4 # Transformer number of heads. Default value is 4. - transformer_cnn_first = True # We have two types of vision transformers. In one type, a CNN is applied first, followed by a transformer. In the other type, this order is reversed. If transformer_cnn_first is true, it means the CNN will be applied before the transformer. Default value is true. - index_start = 0 # Index of model to continue training from. E.g. if you trained for 3 epochs and last index is 2, to continue from model_1.h5, set "index_start" to 3 to start naming model with index 3. - dir_of_start_model = '' # Directory containing pretrained encoder to continue training the model. is_loss_soft_dice = False # Use soft dice as loss function. When set to true, "weighted_loss" must be false. weighted_loss = False # Use weighted categorical cross entropy as loss fucntion. When set to true, "is_loss_soft_dice" must be false. - data_is_provided = False # Only set this to true when you have already provided the input data and the train and eval data are in "dir_output". - task = "segmentation" # This parameter defines task of model which can be segmentation, enhancement or classification. f1_threshold_classification = None # This threshold is used to consider models with an evaluation f1 scores bigger than it. The selected model weights undergo a weights ensembling. And avreage ensembled model will be written to output. classification_classes_name = None # Dictionary of classification classes names. - backbone_type = None # As backbone we have 2 types of backbones. A vision transformer alongside a CNN and we call it "transformer" and only CNN called "nontransformer" - save_interval = None - dir_img_bin = None - number_of_backgrounds_per_image = 1 - dir_rgb_backgrounds = None - dir_rgb_foregrounds = None + patches = False # Divides input image into smaller patches (input size of the model) when set to true. For the model to see the full image, like page extraction, set this to false. + augmentation = False # To apply any kind of augmentation, this parameter must be set to true. + if augmentation: + flip_aug = False # If true, different types of flipping will be applied to the image. Types of flips are defined with "flip_index" in config_params.json. + if flip_aug: + flip_index = None # Flip image for augmentation. + blur_aug = False # If true, different types of blurring will be applied to the image. Types of blur are defined with "blur_k" in config_params.json. + if blur_aug: + blur_k = None # Blur image for augmentation. + padding_white = False # If true, white padding will be applied to the image. + padding_black = False # If true, black padding will be applied to the image. + scaling = False # If true, scaling will be applied to the image. The amount of scaling is defined with "scales" in config_params.json. + scaling_bluring = False # If true, a combination of scaling and blurring will be applied to the image. + scaling_binarization = False # If true, a combination of scaling and binarization will be applied to the image. + scaling_brightness = False # If true, a combination of scaling and brightening will be applied to the image. + scaling_flip = False # If true, a combination of scaling and flipping will be applied to the image. + if scaling or scaling_brightness or scaling_bluring or scaling_binarization or scaling_flip: + scales = None # Scale patches for augmentation. + shifting = False + degrading = False # If true, degrading will be applied to the image. The amount of degrading is defined with "degrade_scales" in config_params.json. + if degrading: + degrade_scales = None # Degrade image for augmentation. + brightening = False # If true, brightening will be applied to the image. The amount of brightening is defined with "brightness" in config_params.json. + if brightening: + brightness = None # Brighten image for augmentation. + binarization = False # If true, Otsu thresholding will be applied to augment the input with binarized images. + if binarization: + dir_img_bin = None # Directory of training dataset subdirectory of binarized images + add_red_textlines = False + adding_rgb_background = False + if adding_rgb_background: + dir_rgb_backgrounds = None # Directory of texture images for synthetic background + adding_rgb_foreground = False + if adding_rgb_foreground: + dir_rgb_foregrounds = None # Directory of texture images for synthetic foreground + if adding_rgb_background or adding_rgb_foreground: + number_of_backgrounds_per_image = 1 + channels_shuffling = False # Re-arrange color channels. + if channels_shuffling: + shuffle_indexes = None # Which channels to switch between. + rotation = False # If true, a 90 degree rotation will be implemeneted. + rotation_not_90 = False # If true rotation based on provided angles with thetha will be implemeneted. + if rotation_not_90: + thetha = None # Rotate image by these angles for augmentation. + dir_train = None # Directory of training dataset with subdirectories having the names "images" and "labels". + dir_eval = None # Directory of validation dataset with subdirectories having the names "images" and "labels". + dir_output = None # Directory where the augmented training data and the model checkpoints will be saved. + pretraining = False # Set to true to (down)load pretrained weights of ResNet50 encoder. + save_interval = None # frequency for writing model checkpoints (nonzero integer for number of batches, or zero for epoch) + continue_training = False # Set to true if you would like to continue training an already trained a model. + dir_of_start_model = '' # Directory containing pretrained encoder to continue training the model. + data_is_provided = False # Only set this to true when you have already provided the input data and the train and eval data are in "dir_output". + if backbone_type == "transformer": + transformer_patchsize_x = None # Patch size of vision transformer patches in x direction. + transformer_patchsize_y = None # Patch size of vision transformer patches in y direction. + transformer_num_patches_xy = None # Number of patches for vision transformer in x and y direction respectively. + transformer_projection_dim = 64 # Transformer projection dimension. Default value is 64. + transformer_mlp_head_units = [128, 64] # Transformer Multilayer Perceptron (MLP) head units. Default value is [128, 64] + transformer_layers = 8 # transformer layers. Default value is 8. + transformer_num_heads = 4 # Transformer number of heads. Default value is 4. + transformer_cnn_first = True # We have two types of vision transformers: either the CNN is applied first, followed by the transformer, or reversed. @ex.automain -def run(_config, n_classes, n_epochs, input_height, - input_width, weight_decay, weighted_loss, - index_start, dir_of_start_model, is_loss_soft_dice, - n_batch, patches, augmentation, flip_aug, - blur_aug, padding_white, padding_black, scaling, shifting, degrading,channels_shuffling, - brightening, binarization, adding_rgb_background, adding_rgb_foreground, add_red_textlines, blur_k, scales, degrade_scales,shuffle_indexes, - brightness, dir_train, data_is_provided, scaling_bluring, - scaling_brightness, scaling_binarization, rotation, rotation_not_90, - thetha, scaling_flip, continue_training, transformer_projection_dim, - transformer_mlp_head_units, transformer_layers, transformer_num_heads, transformer_cnn_first, - transformer_patchsize_x, transformer_patchsize_y, - transformer_num_patches_xy, backbone_type, save_interval, flip_index, dir_eval, dir_output, - pretraining, learning_rate, task, f1_threshold_classification, classification_classes_name, dir_img_bin, number_of_backgrounds_per_image,dir_rgb_backgrounds, dir_rgb_foregrounds): +def run(_config, + _log, + task, + pretraining, + data_is_provided, + dir_train, + dir_eval, + dir_output, + n_classes, + n_epochs, + n_batch, + input_height, + input_width, + is_loss_soft_dice, + weighted_loss, + weight_decay, + learning_rate, + continue_training, + dir_of_start_model, + save_interval, + augmentation, + thetha, + backbone_type, + transformer_projection_dim, + transformer_mlp_head_units, + transformer_layers, + transformer_num_heads, + transformer_cnn_first, + transformer_patchsize_x, + transformer_patchsize_y, + transformer_num_patches_xy, + f1_threshold_classification, + classification_classes_name, +): if pretraining and not os.path.isfile(RESNET50_WEIGHTS_PATH): - print("downloading RESNET50 pretrained weights to", RESNET50_WEIGHTS_PATH) + _log.info("downloading RESNET50 pretrained weights to %s", RESNET50_WEIGHTS_PATH) download_file(RESNET50_WEIGHTS_URL, RESNET50_WEIGHTS_PATH) - - if dir_rgb_backgrounds: - list_all_possible_background_images = os.listdir(dir_rgb_backgrounds) - else: - list_all_possible_background_images = None - - if dir_rgb_foregrounds: - list_all_possible_foreground_rgbs = os.listdir(dir_rgb_foregrounds) - else: - list_all_possible_foreground_rgbs = None - + + # set the gpu configuration + configuration() + if task in ["segmentation", "enhancement", "binarization"]: - if data_is_provided: - dir_train_flowing = os.path.join(dir_output, 'train') - dir_eval_flowing = os.path.join(dir_output, 'eval') - - dir_flow_train_imgs = os.path.join(dir_train_flowing, 'images') - dir_flow_train_labels = os.path.join(dir_train_flowing, 'labels') + dir_train_flowing = os.path.join(dir_output, 'train') + dir_eval_flowing = os.path.join(dir_output, 'eval') - dir_flow_eval_imgs = os.path.join(dir_eval_flowing, 'images') - dir_flow_eval_labels = os.path.join(dir_eval_flowing, 'labels') + dir_flow_train_imgs = os.path.join(dir_train_flowing, 'images') + dir_flow_train_labels = os.path.join(dir_train_flowing, 'labels') - configuration() - - else: - dir_img, dir_seg = get_dirs_or_files(dir_train) - dir_img_val, dir_seg_val = get_dirs_or_files(dir_eval) - - # make first a directory in output for both training and evaluations in order to flow data from these directories. - dir_train_flowing = os.path.join(dir_output, 'train') - dir_eval_flowing = os.path.join(dir_output, 'eval') - - dir_flow_train_imgs = os.path.join(dir_train_flowing, 'images/') - dir_flow_train_labels = os.path.join(dir_train_flowing, 'labels/') - - dir_flow_eval_imgs = os.path.join(dir_eval_flowing, 'images/') - dir_flow_eval_labels = os.path.join(dir_eval_flowing, 'labels/') + dir_flow_eval_imgs = os.path.join(dir_eval_flowing, 'images') + dir_flow_eval_labels = os.path.join(dir_eval_flowing, 'labels') + if not data_is_provided: + # first create a directory in output for both training and evaluations + # in order to flow data from these directories. if os.path.isdir(dir_train_flowing): os.system('rm -rf ' + dir_train_flowing) - os.makedirs(dir_train_flowing) - else: - os.makedirs(dir_train_flowing) + os.makedirs(dir_train_flowing) if os.path.isdir(dir_eval_flowing): os.system('rm -rf ' + dir_eval_flowing) - os.makedirs(dir_eval_flowing) - else: - os.makedirs(dir_eval_flowing) + os.makedirs(dir_eval_flowing) os.mkdir(dir_flow_train_imgs) os.mkdir(dir_flow_train_labels) From 29a0f19cee579665d5edfaa8b3d2bbc8e3bb31b0 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Wed, 28 Jan 2026 13:53:11 +0100 Subject: [PATCH 09/21] =?UTF-8?q?training:=20simplify=20image=20preprocess?= =?UTF-8?q?ing=E2=80=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - `utils.provide_patches`: split up loop into * `utils.preprocess_img` (single img function) * `utils.preprocess_imgs` (top-level loop) - capture exceptions for all cases (not just some) at top level and with informative logging - avoid repeating / delegating config keys in several places: only as kwargs to `preprocess_img()` - read files into memory only once, then re-use - improve readability (avoiding long lines, repeated code) --- src/eynollah/training/train.py | 81 ++-- src/eynollah/training/utils.py | 799 ++++++++++++++++++++------------- 2 files changed, 510 insertions(+), 370 deletions(-) diff --git a/src/eynollah/training/train.py b/src/eynollah/training/train.py index e93281a..9c638ea 100644 --- a/src/eynollah/training/train.py +++ b/src/eynollah/training/train.py @@ -26,7 +26,7 @@ from eynollah.training.utils import ( generate_data_from_folder_evaluation, generate_data_from_folder_training, get_one_hot, - provide_patches, + preprocess_imgs, return_number_of_total_training_data ) @@ -240,9 +240,9 @@ def run(_config, os.mkdir(dir_flow_eval_imgs) os.mkdir(dir_flow_eval_labels) - # set the gpu configuration - configuration() - + dir_img, dir_seg = get_dirs_or_files(dir_train) + dir_img_val, dir_seg_val = get_dirs_or_files(dir_eval) + imgs_list=np.array(os.listdir(dir_img)) segs_list=np.array(os.listdir(dir_seg)) @@ -250,50 +250,21 @@ def run(_config, segs_list_test=np.array(os.listdir(dir_seg_val)) # writing patches into a sub-folder in order to be flowed from directory. - common_args = [input_height, input_width, - blur_k, blur_aug, - padding_white, padding_black, - flip_aug, binarization, - adding_rgb_background, - adding_rgb_foreground, - add_red_textlines, - channels_shuffling, - scaling, shifting, degrading, brightening, - scales, degrade_scales, brightness, - flip_index, shuffle_indexes, - scaling_bluring, scaling_brightness, scaling_binarization, - rotation, rotation_not_90, thetha, - scaling_flip, task, - ] - common_kwargs = dict(patches= - patches, - dir_img_bin= - dir_img_bin, - number_of_backgrounds_per_image= - number_of_backgrounds_per_image, - list_all_possible_background_images= - list_all_possible_background_images, - dir_rgb_backgrounds= - dir_rgb_backgrounds, - dir_rgb_foregrounds= - dir_rgb_foregrounds, - list_all_possible_foreground_rgbs= - list_all_possible_foreground_rgbs, - ) - provide_patches(imgs_list, segs_list, - dir_img, dir_seg, + preprocess_imgs(_config, + imgs_list, + segs_list, + dir_img, + dir_seg, dir_flow_train_imgs, - dir_flow_train_labels, - *common_args, - augmentation=augmentation, - **common_kwargs) - provide_patches(imgs_list_test, segs_list_test, - dir_img_val, dir_seg_val, + dir_flow_train_labels) + preprocess_imgs(_config, + imgs_list_test, + segs_list_test, + dir_img_val, + dir_seg_val, dir_flow_eval_imgs, dir_flow_eval_labels, - *common_args, - augmentation=False, - **common_kwargs) + augmentation=False) if weighted_loss: weights = np.zeros(n_classes) @@ -307,8 +278,8 @@ def run(_config, label_obj = cv2.imread(label_file) label_obj_one_hot = get_one_hot(label_obj, label_obj.shape[0], label_obj.shape[1], n_classes) weights += (label_obj_one_hot.sum(axis=0)).sum(axis=0) - except Exception as e: - print("error reading data file '%s': %s" % (label_file, e), file=sys.stderr) + except Exception: + _log.exception("error reading data file '%s'", label_file) weights = 1.00 / weights weights = weights / float(np.sum(weights)) @@ -340,7 +311,6 @@ def run(_config, custom_objects = {"PatchEncoder": PatchEncoder, "Patches": Patches}) else: - index_start = 0 if backbone_type == 'nontransformer': model = resnet50_unet(n_classes, input_height, @@ -391,7 +361,7 @@ def run(_config, pretraining) #if you want to see the model structure just uncomment model summary. - model.summary() + #model.summary() if task in ["segmentation", "binarization"]: if is_loss_soft_dice: @@ -423,7 +393,12 @@ def run(_config, SaveWeightsAfterSteps(0, dir_output, _config)] if save_interval: callbacks.append(SaveWeightsAfterSteps(save_interval, dir_output, _config)) - + + _log.info("training on %d batches in %d epochs", + len(os.listdir(dir_flow_train_imgs)) // n_batch - 1, + n_epochs) + _log.info("validating on %d batches", + len(os.listdir(dir_flow_eval_imgs)) // n_batch - 1) model.fit( train_gen, steps_per_epoch=len(os.listdir(dir_flow_train_imgs)) // n_batch - 1, @@ -439,7 +414,6 @@ def run(_config, #model.save(dir_output+'/'+'model'+'.h5') elif task=='classification': - configuration() model = resnet50_classifier(n_classes, input_height, input_width, @@ -474,7 +448,7 @@ def run(_config, usable_checkpoints = np.flatnonzero(np.array(history['val_f1']) > f1_threshold_classification) if len(usable_checkpoints) >= 1: - print("averaging over usable checkpoints", usable_checkpoints) + _log.info("averaging over usable checkpoints: %s", str(usable_checkpoints)) all_weights = [] for epoch in usable_checkpoints: cp_path = os.path.join(dir_output, 'model_{epoch:02d}'.format(epoch=epoch)) @@ -495,10 +469,9 @@ def run(_config, model.save(cp_path) with open(os.path.join(cp_path, "config.json"), "w") as fp: json.dump(_config, fp) # encode dict into JSON - print("ensemble model saved under", cp_path) + _log.info("ensemble model saved under '%s'", cp_path) elif task=='reading_order': - configuration() model = machine_based_reading_order_model( n_classes, input_height, input_width, weight_decay, pretraining) diff --git a/src/eynollah/training/utils.py b/src/eynollah/training/utils.py index 1278be5..61b2536 100644 --- a/src/eynollah/training/utils.py +++ b/src/eynollah/training/utils.py @@ -1,6 +1,7 @@ import os import math import random +from logging import getLogger import cv2 import numpy as np @@ -266,8 +267,9 @@ def generate_data_from_folder_training(path_classes, batchsize, height, width, n ret_y= np.zeros((batchsize, n_classes)).astype(np.int16) batchcount = 0 -def do_brightening(img_in_dir, factor): - im = Image.open(img_in_dir) +def do_brightening(img, factor): + img_rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) + im = Image.fromarray(img_rgb) enhancer = ImageEnhance.Brightness(im) out_img = enhancer.enhance(factor) out_img = out_img.convert('RGB') @@ -737,321 +739,486 @@ def get_patches_num_scale_new(dir_img_f, dir_seg_f, img, label, height, width, i return indexer -def provide_patches(imgs_list_train, segs_list_train, dir_img, dir_seg, dir_flow_train_imgs, - dir_flow_train_labels, input_height, input_width, blur_k, blur_aug, - padding_white, padding_black, flip_aug, binarization, adding_rgb_background, adding_rgb_foreground, add_red_textlines, channels_shuffling, scaling, shifting, degrading, - brightening, scales, degrade_scales, brightness, flip_index, shuffle_indexes, - scaling_bluring, scaling_brightness, scaling_binarization, rotation, - rotation_not_90, thetha, scaling_flip, task, augmentation=False, patches=False, dir_img_bin=None,number_of_backgrounds_per_image=None,list_all_possible_background_images=None, dir_rgb_backgrounds=None, dir_rgb_foregrounds=None, list_all_possible_foreground_rgbs=None): - +def preprocess_imgs(config, + imgs_list, + segs_list, + dir_img, + dir_seg, + dir_flow_imgs, + dir_flow_labels, + logger=None, + **kwargs, +): + if logger is None: + logger = getLogger('') + + # make a copy for this run + config = dict(config) + # add derived keys not part of config + if config.get('dir_rgb_backgrounds', None): + config['list_all_possible_background_images'] = \ + os.listdir(config['dir_rgb_backgrounds']) + if config.get('dir_rgb_foregrounds', None): + config['list_all_possible_foreground_rgbs'] = \ + os.listdir(config['dir_rgb_foregrounds']) + # override keys from call + config.update(kwargs) + indexer = 0 - for im, seg_i in tqdm(zip(imgs_list_train, segs_list_train)): + for im, seg_i in tqdm(zip(imgs_list, segs_list)): + img = cv2.imread(os.path.join(dir_img, im)) img_name = os.path.splitext(im)[0] - if task == "segmentation" or task == "binarization": - dir_of_label_file = os.path.join(dir_seg, img_name + '.png') - elif task=="enhancement": - dir_of_label_file = os.path.join(dir_seg, im) - - if not patches: - cv2.imwrite(dir_flow_train_imgs + '/img_' + str(indexer) + '.png', resize_image(cv2.imread(dir_img + '/' + im), input_height, input_width)) - cv2.imwrite(dir_flow_train_labels + '/img_' + str(indexer) + '.png', resize_image(cv2.imread(dir_of_label_file), input_height, input_width)) - indexer += 1 - - if augmentation: - if flip_aug: - for f_i in flip_index: - cv2.imwrite(dir_flow_train_imgs + '/img_' + str(indexer) + '.png', - resize_image(cv2.flip(cv2.imread(dir_img+'/'+im),f_i),input_height,input_width) ) - - cv2.imwrite(dir_flow_train_labels + '/img_' + str(indexer) + '.png', - resize_image(cv2.flip(cv2.imread(dir_of_label_file), f_i), input_height, input_width)) - indexer += 1 - - if blur_aug: - for blur_i in blur_k: - cv2.imwrite(dir_flow_train_imgs + '/img_' + str(indexer) + '.png', - (resize_image(bluring(cv2.imread(dir_img + '/' + im), blur_i), input_height, input_width))) - - cv2.imwrite(dir_flow_train_labels + '/img_' + str(indexer) + '.png', - resize_image(cv2.imread(dir_of_label_file), input_height, input_width)) - indexer += 1 - if brightening: - for factor in brightness: - try: - cv2.imwrite(dir_flow_train_imgs + '/img_' + str(indexer) + '.png', - (resize_image(do_brightening(dir_img + '/' +im, factor), input_height, input_width))) - - cv2.imwrite(dir_flow_train_labels + '/img_' + str(indexer) + '.png', - resize_image(cv2.imread(dir_of_label_file), input_height, input_width)) - indexer += 1 - except: - pass - - if binarization: - - if dir_img_bin: - img_bin_corr = cv2.imread(dir_img_bin + '/' + img_name+'.png') - - cv2.imwrite(dir_flow_train_imgs + '/img_' + str(indexer) + '.png', - resize_image(img_bin_corr, input_height, input_width)) - else: - cv2.imwrite(dir_flow_train_imgs + '/img_' + str(indexer) + '.png', - resize_image(otsu_copy(cv2.imread(dir_img + '/' + im)), input_height, input_width)) - - cv2.imwrite(dir_flow_train_labels + '/img_' + str(indexer) + '.png', - resize_image(cv2.imread(dir_of_label_file), input_height, input_width)) - indexer += 1 - - if degrading: - for degrade_scale_ind in degrade_scales: - cv2.imwrite(dir_flow_train_imgs + '/img_' + str(indexer) + '.png', - (resize_image(do_degrading(cv2.imread(dir_img + '/' + im), degrade_scale_ind), input_height, input_width))) - - cv2.imwrite(dir_flow_train_labels + '/img_' + str(indexer) + '.png', - resize_image(cv2.imread(dir_of_label_file), input_height, input_width)) - indexer += 1 - - if rotation_not_90: - for thetha_i in thetha: - img_max_rotated, label_max_rotated = rotation_not_90_func(cv2.imread(dir_img + '/'+im), - cv2.imread(dir_of_label_file), thetha_i) - - cv2.imwrite(dir_flow_train_imgs + '/img_' + str(indexer) + '.png', resize_image(img_max_rotated, input_height, input_width)) - - cv2.imwrite(dir_flow_train_labels + '/img_' + str(indexer) + '.png', resize_image(label_max_rotated, input_height, input_width)) - indexer += 1 - - if channels_shuffling: - for shuffle_index in shuffle_indexes: - cv2.imwrite(dir_flow_train_imgs + '/img_' + str(indexer) + '.png', - (resize_image(return_shuffled_channels(cv2.imread(dir_img + '/' + im), shuffle_index), input_height, input_width))) - - cv2.imwrite(dir_flow_train_labels + '/img_' + str(indexer) + '.png', - resize_image(cv2.imread(dir_of_label_file), input_height, input_width)) - indexer += 1 - - if scaling: - for sc_ind in scales: - img_scaled, label_scaled = scale_image_for_no_patch(cv2.imread(dir_img + '/'+im), - cv2.imread(dir_of_label_file), sc_ind) - - cv2.imwrite(dir_flow_train_imgs + '/img_' + str(indexer) + '.png', resize_image(img_scaled, input_height, input_width)) - cv2.imwrite(dir_flow_train_labels + '/img_' + str(indexer) + '.png', resize_image(label_scaled, input_height, input_width)) - indexer += 1 - if shifting: - shift_types = ['xpos', 'xmin', 'ypos', 'ymin', 'xypos', 'xymin'] - for st_ind in shift_types: - img_shifted, label_shifted = shift_image_and_label(cv2.imread(dir_img + '/'+im), - cv2.imread(dir_of_label_file), st_ind) - - cv2.imwrite(dir_flow_train_imgs + '/img_' + str(indexer) + '.png', resize_image(img_shifted, input_height, input_width)) - cv2.imwrite(dir_flow_train_labels + '/img_' + str(indexer) + '.png', resize_image(label_shifted, input_height, input_width)) - indexer += 1 - - - if adding_rgb_background: - img_bin_corr = cv2.imread(dir_img_bin + '/' + img_name+'.png') - for i_n in range(number_of_backgrounds_per_image): - background_image_chosen_name = random.choice(list_all_possible_background_images) - img_rgb_background_chosen = cv2.imread(dir_rgb_backgrounds + '/' + background_image_chosen_name) - img_with_overlayed_background = return_binary_image_with_given_rgb_background(img_bin_corr, img_rgb_background_chosen) - - cv2.imwrite(dir_flow_train_imgs + '/img_' + str(indexer) + '.png', resize_image(img_with_overlayed_background, input_height, input_width)) - cv2.imwrite(dir_flow_train_labels + '/img_' + str(indexer) + '.png', - resize_image(cv2.imread(dir_of_label_file), input_height, input_width)) - - indexer += 1 - - if adding_rgb_foreground: - img_bin_corr = cv2.imread(dir_img_bin + '/' + img_name+'.png') - for i_n in range(number_of_backgrounds_per_image): - background_image_chosen_name = random.choice(list_all_possible_background_images) - foreground_rgb_chosen_name = random.choice(list_all_possible_foreground_rgbs) - - img_rgb_background_chosen = cv2.imread(dir_rgb_backgrounds + '/' + background_image_chosen_name) - foreground_rgb_chosen = np.load(dir_rgb_foregrounds + '/' + foreground_rgb_chosen_name) - - img_with_overlayed_background = return_binary_image_with_given_rgb_background_and_given_foreground_rgb(img_bin_corr, img_rgb_background_chosen, foreground_rgb_chosen) - - cv2.imwrite(dir_flow_train_imgs + '/img_' + str(indexer) + '.png', resize_image(img_with_overlayed_background, input_height, input_width)) - cv2.imwrite(dir_flow_train_labels + '/img_' + str(indexer) + '.png', - resize_image(cv2.imread(dir_of_label_file), input_height, input_width)) - - indexer += 1 - - if add_red_textlines: - img_bin_corr = cv2.imread(dir_img_bin + '/' + img_name+'.png') - img_red_context = return_image_with_red_elements(cv2.imread(dir_img + '/'+im), img_bin_corr) - - cv2.imwrite(dir_flow_train_imgs + '/img_' + str(indexer) + '.png', resize_image(img_red_context, input_height, input_width)) - cv2.imwrite(dir_flow_train_labels + '/img_' + str(indexer) + '.png', - resize_image(cv2.imread(dir_of_label_file), input_height, input_width)) - - indexer += 1 - - - - - if patches: - indexer = get_patches(dir_flow_train_imgs, dir_flow_train_labels, - cv2.imread(dir_img + '/' + im), cv2.imread(dir_of_label_file), - input_height, input_width, indexer=indexer) - - if augmentation: - if rotation: - indexer = get_patches(dir_flow_train_imgs, dir_flow_train_labels, - rotation_90(cv2.imread(dir_img + '/' + im)), - rotation_90(cv2.imread(dir_of_label_file)), - input_height, input_width, indexer=indexer) - - if rotation_not_90: - for thetha_i in thetha: - img_max_rotated, label_max_rotated = rotation_not_90_func(cv2.imread(dir_img + '/'+im), - cv2.imread(dir_of_label_file), thetha_i) - indexer = get_patches(dir_flow_train_imgs, dir_flow_train_labels, - img_max_rotated, - label_max_rotated, - input_height, input_width, indexer=indexer) - - if channels_shuffling: - for shuffle_index in shuffle_indexes: - indexer = get_patches(dir_flow_train_imgs, dir_flow_train_labels, - return_shuffled_channels(cv2.imread(dir_img + '/' + im), shuffle_index), - cv2.imread(dir_of_label_file), - input_height, input_width, indexer=indexer) - - if adding_rgb_background: - img_bin_corr = cv2.imread(dir_img_bin + '/' + img_name+'.png') - for i_n in range(number_of_backgrounds_per_image): - background_image_chosen_name = random.choice(list_all_possible_background_images) - img_rgb_background_chosen = cv2.imread(dir_rgb_backgrounds + '/' + background_image_chosen_name) - img_with_overlayed_background = return_binary_image_with_given_rgb_background(img_bin_corr, img_rgb_background_chosen) - - indexer = get_patches(dir_flow_train_imgs, dir_flow_train_labels, - img_with_overlayed_background, - cv2.imread(dir_of_label_file), - input_height, input_width, indexer=indexer) - - - if adding_rgb_foreground: - img_bin_corr = cv2.imread(dir_img_bin + '/' + img_name+'.png') - for i_n in range(number_of_backgrounds_per_image): - background_image_chosen_name = random.choice(list_all_possible_background_images) - foreground_rgb_chosen_name = random.choice(list_all_possible_foreground_rgbs) - - img_rgb_background_chosen = cv2.imread(dir_rgb_backgrounds + '/' + background_image_chosen_name) - foreground_rgb_chosen = np.load(dir_rgb_foregrounds + '/' + foreground_rgb_chosen_name) - - img_with_overlayed_background = return_binary_image_with_given_rgb_background_and_given_foreground_rgb(img_bin_corr, img_rgb_background_chosen, foreground_rgb_chosen) - - indexer = get_patches(dir_flow_train_imgs, dir_flow_train_labels, - img_with_overlayed_background, - cv2.imread(dir_of_label_file), - input_height, input_width, indexer=indexer) - - - if add_red_textlines: - img_bin_corr = cv2.imread(dir_img_bin + '/' + img_name+'.png') - img_red_context = return_image_with_red_elements(cv2.imread(dir_img + '/'+im), img_bin_corr) - - indexer = get_patches(dir_flow_train_imgs, dir_flow_train_labels, - img_red_context, - cv2.imread(dir_of_label_file), - input_height, input_width, indexer=indexer) - - if flip_aug: - for f_i in flip_index: - indexer = get_patches(dir_flow_train_imgs, dir_flow_train_labels, - cv2.flip(cv2.imread(dir_img + '/' + im), f_i), - cv2.flip(cv2.imread(dir_of_label_file), f_i), - input_height, input_width, indexer=indexer) - if blur_aug: - for blur_i in blur_k: - indexer = get_patches(dir_flow_train_imgs, dir_flow_train_labels, - bluring(cv2.imread(dir_img + '/' + im), blur_i), - cv2.imread(dir_of_label_file), - input_height, input_width, indexer=indexer) - if padding_black: - indexer = get_patches(dir_flow_train_imgs, dir_flow_train_labels, - do_padding_black(cv2.imread(dir_img + '/' + im)), - do_padding_label(cv2.imread(dir_of_label_file)), - input_height, input_width, indexer=indexer) - - if padding_white: - indexer = get_patches(dir_flow_train_imgs, dir_flow_train_labels, - do_padding_white(cv2.imread(dir_img + '/'+im)), - do_padding_label(cv2.imread(dir_of_label_file)), - input_height, input_width, indexer=indexer) - - if brightening: - for factor in brightness: - try: - indexer = get_patches(dir_flow_train_imgs, dir_flow_train_labels, - do_brightening(dir_img + '/' +im, factor), - cv2.imread(dir_of_label_file), - input_height, input_width, indexer=indexer) - except: - pass - if scaling: - for sc_ind in scales: - indexer = get_patches_num_scale_new(dir_flow_train_imgs, dir_flow_train_labels, - cv2.imread(dir_img + '/' + im) , - cv2.imread(dir_of_label_file), - input_height, input_width, indexer=indexer, scaler=sc_ind) - - if degrading: - for degrade_scale_ind in degrade_scales: - indexer = get_patches(dir_flow_train_imgs, dir_flow_train_labels, - do_degrading(cv2.imread(dir_img + '/' + im), degrade_scale_ind), - cv2.imread(dir_of_label_file), - input_height, input_width, indexer=indexer) - - if binarization: - if dir_img_bin: - img_bin_corr = cv2.imread(dir_img_bin + '/' + img_name+'.png') - - indexer = get_patches(dir_flow_train_imgs, dir_flow_train_labels, - img_bin_corr, - cv2.imread(dir_of_label_file), - input_height, input_width, indexer=indexer) - - else: - indexer = get_patches(dir_flow_train_imgs, dir_flow_train_labels, - otsu_copy(cv2.imread(dir_img + '/' + im)), - cv2.imread(dir_of_label_file), - input_height, input_width, indexer=indexer) + if config['task'] in ["segmentation", "binarization"]: + lab = cv2.imread(os.path.join(dir_seg, img_name + '.png')) + elif config['task'] == "enhancement": + lab = cv2.imread(os.path.join(dir_seg, im)) + else: + lab = None - if scaling_brightness: - for sc_ind in scales: - for factor in brightness: - try: - indexer = get_patches_num_scale_new(dir_flow_train_imgs, - dir_flow_train_labels, - do_brightening(dir_img + '/' + im, factor) - ,cv2.imread(dir_of_label_file) - ,input_height, input_width, indexer=indexer, scaler=sc_ind) - except: - pass - - if scaling_bluring: - for sc_ind in scales: - for blur_i in blur_k: - indexer = get_patches_num_scale_new(dir_flow_train_imgs, dir_flow_train_labels, - bluring(cv2.imread(dir_img + '/' + im), blur_i), - cv2.imread(dir_of_label_file), - input_height, input_width, indexer=indexer, scaler=sc_ind) + try: + indexer = preprocess_img(indexer, img, img_name, lab, + dir_flow_imgs, + dir_flow_labels, + **config) - if scaling_binarization: - for sc_ind in scales: - indexer = get_patches_num_scale_new(dir_flow_train_imgs, dir_flow_train_labels, - otsu_copy(cv2.imread(dir_img + '/' + im)), - cv2.imread(dir_of_label_file), - input_height, input_width, indexer=indexer, scaler=sc_ind) - - if scaling_flip: - for sc_ind in scales: - for f_i in flip_index: - indexer = get_patches_num_scale_new(dir_flow_train_imgs, dir_flow_train_labels, - cv2.flip( cv2.imread(dir_img + '/' + im), f_i), - cv2.flip(cv2.imread(dir_of_label_file), f_i), - input_height, input_width, indexer=indexer, scaler=sc_ind) + except: + logger.exception("skipping image %s", img_name) + +def preprocess_img(indexer, + img, + img_name, + lab, + dir_flow_train_imgs, + dir_flow_train_labels, + input_height=None, + input_width=None, + augmentation=False, + flip_aug=False, + flip_index=None, + blur_aug=False, + blur_k=None, + padding_white=False, + padding_black=False, + scaling=False, + scaling_bluring=False, + scaling_brightness=False, + scaling_binarization=False, + scaling_flip=False, + scales=None, + shifting=False, + degrading=False, + degrade_scales=None, + brightening=False, + brightness=None, + binarization=False, + dir_img_bin=None, + add_red_textlines=False, + adding_rgb_background=False, + dir_rgb_backgrounds=None, + adding_rgb_foreground=False, + dir_rgb_foregrounds=None, + number_of_backgrounds_per_image=None, + channels_shuffling=False, + shuffle_indexes=None, + rotation=False, + rotation_not_90=False, + thetha=None, + patches=False, + list_all_possible_background_images=None, + list_all_possible_foreground_rgbs=None, + **kwargs, +): + if not patches: + cv2.imwrite(dir_flow_train_imgs + '/img_' + str(indexer) + '.png', + resize_image(img, + input_height, + input_width)) + cv2.imwrite(dir_flow_train_labels + '/img_' + str(indexer) + '.png', + resize_image(lab, + input_height, + input_width)) + indexer += 1 + if augmentation: + if flip_aug: + for f_i in flip_index: + cv2.imwrite(dir_flow_train_imgs + '/img_' + str(indexer) + '.png', + resize_image(cv2.flip(img, f_i), + input_height, + input_width)) + cv2.imwrite(dir_flow_train_labels + '/img_' + str(indexer) + '.png', + resize_image(cv2.flip(lab, f_i), + input_height, + input_width)) + indexer += 1 + if blur_aug: + for blur_i in blur_k: + cv2.imwrite(dir_flow_train_imgs + '/img_' + str(indexer) + '.png', + (resize_image(bluring(img, blur_i), + input_height, + input_width))) + cv2.imwrite(dir_flow_train_labels + '/img_' + str(indexer) + '.png', + resize_image(lab, + input_height, + input_width)) + indexer += 1 + if brightening: + for factor in brightness: + cv2.imwrite(dir_flow_train_imgs + '/img_' + str(indexer) + '.png', + (resize_image(do_brightening(img, factor), + input_height, + input_width))) + cv2.imwrite(dir_flow_train_labels + '/img_' + str(indexer) + '.png', + resize_image(lab, + input_height, + input_width)) + indexer += 1 + if binarization: + if dir_img_bin: + img_bin_corr = cv2.imread(dir_img_bin + '/' + img_name+'.png') + cv2.imwrite(dir_flow_train_imgs + '/img_' + str(indexer) + '.png', + resize_image(img_bin_corr, + input_height, + input_width)) + else: + cv2.imwrite(dir_flow_train_imgs + '/img_' + str(indexer) + '.png', + resize_image(otsu_copy(img), + input_height, + input_width)) + cv2.imwrite(dir_flow_train_labels + '/img_' + str(indexer) + '.png', + resize_image(lab, + input_height, + input_width)) + indexer += 1 + if degrading: + for degrade_scale_ind in degrade_scales: + cv2.imwrite(dir_flow_train_imgs + '/img_' + str(indexer) + '.png', + (resize_image(do_degrading(img, degrade_scale_ind), + input_height, + input_width))) + cv2.imwrite(dir_flow_train_labels + '/img_' + str(indexer) + '.png', + resize_image(lab, + input_height, + input_width)) + indexer += 1 + if rotation_not_90: + for thetha_i in thetha: + img_max_rotated, label_max_rotated = \ + rotation_not_90_func(img, lab, thetha_i) + cv2.imwrite(dir_flow_train_imgs + '/img_' + str(indexer) + '.png', + resize_image(img_max_rotated, + input_height, + input_width)) + cv2.imwrite(dir_flow_train_labels + '/img_' + str(indexer) + '.png', + resize_image(label_max_rotated, + input_height, + input_width)) + indexer += 1 + if channels_shuffling: + for shuffle_index in shuffle_indexes: + cv2.imwrite(dir_flow_train_imgs + '/img_' + str(indexer) + '.png', + (resize_image(return_shuffled_channels(img, shuffle_index), + input_height, + input_width))) + cv2.imwrite(dir_flow_train_labels + '/img_' + str(indexer) + '.png', + resize_image(lab, + input_height, + input_width)) + indexer += 1 + if scaling: + for sc_ind in scales: + img_scaled, label_scaled = \ + scale_image_for_no_patch(img, lab, sc_ind) + cv2.imwrite(dir_flow_train_imgs + '/img_' + str(indexer) + '.png', + resize_image(img_scaled, + input_height, + input_width)) + cv2.imwrite(dir_flow_train_labels + '/img_' + str(indexer) + '.png', + resize_image(label_scaled, + input_height, + input_width)) + indexer += 1 + if shifting: + shift_types = ['xpos', 'xmin', 'ypos', 'ymin', 'xypos', 'xymin'] + for st_ind in shift_types: + img_shifted, label_shifted = \ + shift_image_and_label(img, lab, st_ind) + cv2.imwrite(dir_flow_train_imgs + '/img_' + str(indexer) + '.png', + resize_image(img_shifted, + input_height, + input_width)) + cv2.imwrite(dir_flow_train_labels + '/img_' + str(indexer) + '.png', + resize_image(label_shifted, + input_height, + input_width)) + indexer += 1 + if adding_rgb_background: + img_bin_corr = cv2.imread(dir_img_bin + '/' + img_name+'.png') + for i_n in range(number_of_backgrounds_per_image): + background_image_chosen_name = random.choice(list_all_possible_background_images) + img_rgb_background_chosen = \ + cv2.imread(dir_rgb_backgrounds + '/' + background_image_chosen_name) + img_with_overlayed_background = \ + return_binary_image_with_given_rgb_background( + img_bin_corr, img_rgb_background_chosen) + cv2.imwrite(dir_flow_train_imgs + '/img_' + str(indexer) + '.png', + resize_image(img_with_overlayed_background, + input_height, + input_width)) + cv2.imwrite(dir_flow_train_labels + '/img_' + str(indexer) + '.png', + resize_image(lab, + input_height, + input_width)) + indexer += 1 + if adding_rgb_foreground: + img_bin_corr = cv2.imread(dir_img_bin + '/' + img_name+'.png') + for i_n in range(number_of_backgrounds_per_image): + background_image_chosen_name = random.choice(list_all_possible_background_images) + foreground_rgb_chosen_name = random.choice(list_all_possible_foreground_rgbs) + img_rgb_background_chosen = \ + cv2.imread(dir_rgb_backgrounds + '/' + background_image_chosen_name) + foreground_rgb_chosen = \ + np.load(dir_rgb_foregrounds + '/' + foreground_rgb_chosen_name) + img_with_overlayed_background = \ + return_binary_image_with_given_rgb_background_and_given_foreground_rgb( + img_bin_corr, img_rgb_background_chosen, foreground_rgb_chosen) + cv2.imwrite(dir_flow_train_imgs + '/img_' + str(indexer) + '.png', + resize_image(img_with_overlayed_background, + input_height, + input_width)) + cv2.imwrite(dir_flow_train_labels + '/img_' + str(indexer) + '.png', + resize_image(lab, + input_height, + input_width)) + indexer += 1 + if add_red_textlines: + img_bin_corr = cv2.imread(dir_img_bin + '/' + img_name+'.png') + img_red_context = \ + return_image_with_red_elements(img, img_bin_corr) + cv2.imwrite(dir_flow_train_imgs + '/img_' + str(indexer) + '.png', + resize_image(img_red_context, + input_height, + input_width)) + cv2.imwrite(dir_flow_train_labels + '/img_' + str(indexer) + '.png', + resize_image(lab, + input_height, + input_width)) + indexer += 1 + else: + indexer = get_patches(dir_flow_train_imgs, + dir_flow_train_labels, + img, + lab, + input_height, + input_width, + indexer=indexer) + if augmentation: + if rotation: + indexer = get_patches(dir_flow_train_imgs, + dir_flow_train_labels, + rotation_90(img), + rotation_90(lab), + input_height, + input_width, + indexer=indexer) + if rotation_not_90: + for thetha_i in thetha: + img_max_rotated, label_max_rotated = \ + rotation_not_90_func(img, lab, thetha_i) + indexer = get_patches(dir_flow_train_imgs, + dir_flow_train_labels, + img_max_rotated, + label_max_rotated, + input_height, + input_width, + indexer=indexer) + if channels_shuffling: + for shuffle_index in shuffle_indexes: + img_shuffled = \ + return_shuffled_channels(img, shuffle_index), + indexer = get_patches(dir_flow_train_imgs, + dir_flow_train_labels, + img_shuffled, + lab, + input_height, + input_width, + indexer=indexer) + if adding_rgb_background: + img_bin_corr = cv2.imread(dir_img_bin + '/' + img_name+'.png') + for i_n in range(number_of_backgrounds_per_image): + background_image_chosen_name = random.choice(list_all_possible_background_images) + img_rgb_background_chosen = \ + cv2.imread(dir_rgb_backgrounds + '/' + background_image_chosen_name) + img_with_overlayed_background = \ + return_binary_image_with_given_rgb_background( + img_bin_corr, img_rgb_background_chosen) + indexer = get_patches(dir_flow_train_imgs, + dir_flow_train_labels, + img_with_overlayed_background, + lab, + input_height, + input_width, + indexer=indexer) + if adding_rgb_foreground: + img_bin_corr = cv2.imread(dir_img_bin + '/' + img_name+'.png') + for i_n in range(number_of_backgrounds_per_image): + background_image_chosen_name = random.choice(list_all_possible_background_images) + foreground_rgb_chosen_name = random.choice(list_all_possible_foreground_rgbs) + img_rgb_background_chosen = \ + cv2.imread(dir_rgb_backgrounds + '/' + background_image_chosen_name) + foreground_rgb_chosen = \ + np.load(dir_rgb_foregrounds + '/' + foreground_rgb_chosen_name) + img_with_overlayed_background = \ + return_binary_image_with_given_rgb_background_and_given_foreground_rgb( + img_bin_corr, img_rgb_background_chosen, foreground_rgb_chosen) + indexer = get_patches(dir_flow_train_imgs, + dir_flow_train_labels, + img_with_overlayed_background, + lab, + input_height, + input_width, + indexer=indexer) + if add_red_textlines: + img_bin_corr = cv2.imread(dir_img_bin + '/' + img_name+'.png') + img_red_context = \ + return_image_with_red_elements(img, img_bin_corr) + indexer = get_patches(dir_flow_train_imgs, + dir_flow_train_labels, + img_red_context, + lab, + input_height, + input_width, + indexer=indexer) + if flip_aug: + for f_i in flip_index: + indexer = get_patches(dir_flow_train_imgs, + dir_flow_train_labels, + cv2.flip(img, f_i), + cv2.flip(lab, f_i), + input_height, + input_width, + indexer=indexer) + if blur_aug: + for blur_i in blur_k: + indexer = get_patches(dir_flow_train_imgs, + dir_flow_train_labels, + bluring(img, blur_i), + lab, + input_height, + input_width, + indexer=indexer) + if padding_black: + indexer = get_patches(dir_flow_train_imgs, + dir_flow_train_labels, + do_padding_black(img), + do_padding_label(lab), + input_height, + input_width, + indexer=indexer) + if padding_white: + indexer = get_patches(dir_flow_train_imgs, + dir_flow_train_labels, + do_padding_white(img), + do_padding_label(lab), + input_height, + input_width, + indexer=indexer) + if brightening: + for factor in brightness: + indexer = get_patches(dir_flow_train_imgs, + dir_flow_train_labels, + do_brightening(img, factor), + lab, + input_height, + input_width, + indexer=indexer) + if scaling: + for sc_ind in scales: + indexer = get_patches_num_scale_new( + dir_flow_train_imgs, + dir_flow_train_labels, + img , + lab, + input_height, + input_width, + indexer=indexer, + scaler=sc_ind) + if degrading: + for degrade_scale_ind in degrade_scales: + img_deg = \ + do_degrading(img, degrade_scale_ind), + indexer = get_patches(dir_flow_train_imgs, + dir_flow_train_labels, + img_deg, + lab, + input_height, + input_width, + indexer=indexer) + if binarization: + if dir_img_bin: + img_bin_corr = cv2.imread(dir_img_bin + '/' + img_name+'.png') + indexer = get_patches(dir_flow_train_imgs, + dir_flow_train_labels, + img_bin_corr, + lab, + input_height, + input_width, + indexer=indexer) + else: + indexer = get_patches(dir_flow_train_imgs, + dir_flow_train_labels, + otsu_copy(img), + lab, + input_height, + input_width, + indexer=indexer) + if scaling_brightness: + for sc_ind in scales: + for factor in brightness: + img_bright = do_brightening(img, factor) + indexer = get_patches_num_scale_new( + dir_flow_train_imgs, + dir_flow_train_labels, + img_bright, + lab, + input_height, + input_width, + indexer=indexer, + scaler=sc_ind) + if scaling_bluring: + for sc_ind in scales: + for blur_i in blur_k: + img_blur = bluring(img, blur_i), + indexer = get_patches_num_scale_new( + dir_flow_train_imgs, + dir_flow_train_labels, + img_blur, + lab, + input_height, + input_width, + indexer=indexer, + scaler=sc_ind) + if scaling_binarization: + for sc_ind in scales: + img_bin = otsu_copy(img), + indexer = get_patches_num_scale_new( + dir_flow_train_imgs, + dir_flow_train_labels, + img_bin, + lab, + input_height, + input_width, + indexer=indexer, + scaler=sc_ind) + if scaling_flip: + for sc_ind in scales: + for f_i in flip_index: + indexer = get_patches_num_scale_new( + dir_flow_train_imgs, + dir_flow_train_labels, + cv2.flip(img, f_i), + cv2.flip(lab, f_i), + input_height, + input_width, + indexer=indexer, + scaler=sc_ind) + return indexer From d1e8a02fd4a50d61d3101db8a9ae870201bde194 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Thu, 29 Jan 2026 03:01:14 +0100 Subject: [PATCH 10/21] training: fix epoch size calculation --- src/eynollah/training/train.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/eynollah/training/train.py b/src/eynollah/training/train.py index 9c638ea..1e2ab3e 100644 --- a/src/eynollah/training/train.py +++ b/src/eynollah/training/train.py @@ -394,17 +394,16 @@ def run(_config, if save_interval: callbacks.append(SaveWeightsAfterSteps(save_interval, dir_output, _config)) - _log.info("training on %d batches in %d epochs", - len(os.listdir(dir_flow_train_imgs)) // n_batch - 1, - n_epochs) - _log.info("validating on %d batches", - len(os.listdir(dir_flow_eval_imgs)) // n_batch - 1) + steps_train = len(os.listdir(dir_flow_train_imgs)) // n_batch # - 1 + steps_val = len(os.listdir(dir_flow_eval_imgs)) // n_batch + _log.info("training on %d batches in %d epochs", steps_train, n_epochs) + _log.info("validating on %d batches", steps_val) model.fit( train_gen, - steps_per_epoch=len(os.listdir(dir_flow_train_imgs)) // n_batch - 1, + steps_per_epoch=steps_train, validation_data=val_gen, #validation_steps=1, # rs: only one batch?? - validation_steps=len(os.listdir(dir_flow_eval_imgs)) // n_batch - 1, + validation_steps=steps_val, epochs=n_epochs, callbacks=callbacks) From 25153ad307a6ea658dee8d3be19250969530cdfc Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Thu, 29 Jan 2026 12:19:09 +0100 Subject: [PATCH 11/21] training: add IoU metric --- src/eynollah/training/train.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/eynollah/training/train.py b/src/eynollah/training/train.py index 1e2ab3e..344522a 100644 --- a/src/eynollah/training/train.py +++ b/src/eynollah/training/train.py @@ -34,6 +34,7 @@ os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' os.environ['TF_USE_LEGACY_KERAS'] = '1' # avoid Keras 3 after TF 2.15 import tensorflow as tf from tensorflow.keras.optimizers import SGD, Adam +from tensorflow.keras.metrics import MeanIoU from tensorflow.keras.models import load_model from tensorflow.keras.callbacks import ModelCheckpoint, TensorBoard from sacred import Experiment @@ -374,7 +375,11 @@ def run(_config, loss = 'mean_squared_error' model.compile(loss=loss, optimizer=Adam(learning_rate=learning_rate), - metrics=['accuracy']) + metrics=['accuracy', MeanIoU(n_classes, + name='iou', + ignore_class=0, + sparse_y_true=False, + sparse_y_pred=False)]) # generating train and evaluation data gen_kwargs = dict(batch_size=n_batch, From e85003db4a74d2a0b3f830c0338402368cb67d48 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Wed, 4 Feb 2026 17:32:24 +0100 Subject: [PATCH 12/21] training: re-instate `index_start`, reflect cfg dependency - `index_start`: re-introduce cfg key, pass to Keras `Model.fit` as `initial_epoch` - make config keys `index_start` and `dir_of_start_model` dependent on `continue_training` - improve description --- src/eynollah/training/train.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/eynollah/training/train.py b/src/eynollah/training/train.py index 344522a..de8cccd 100644 --- a/src/eynollah/training/train.py +++ b/src/eynollah/training/train.py @@ -157,10 +157,12 @@ def config_params(): dir_eval = None # Directory of validation dataset with subdirectories having the names "images" and "labels". dir_output = None # Directory where the augmented training data and the model checkpoints will be saved. pretraining = False # Set to true to (down)load pretrained weights of ResNet50 encoder. - save_interval = None # frequency for writing model checkpoints (nonzero integer for number of batches, or zero for epoch) - continue_training = False # Set to true if you would like to continue training an already trained a model. - dir_of_start_model = '' # Directory containing pretrained encoder to continue training the model. - data_is_provided = False # Only set this to true when you have already provided the input data and the train and eval data are in "dir_output". + save_interval = None # frequency for writing model checkpoints (positive integer for number of batches saved under "model_step_{batch:04d}", otherwise epoch saved under "model_{epoch:02d}") + continue_training = False # Whether to continue training an existing model. + if continue_training: + dir_of_start_model = '' # Directory of model checkpoint to load to continue training. (E.g. if you already trained for 3 epochs, set "dir_of_start_model=dir_output/model_03".) + index_start = 0 # Epoch counter initial value to continue training. (E.g. if you already trained for 3 epochs, set "index_start=3" to continue naming checkpoints model_04, model_05 etc.) + data_is_provided = False # Whether the preprocessed input data (subdirectories "images" and "labels" in both subdirectories "train" and "eval" of "dir_output") has already been generated (in the first epoch of a previous run). if backbone_type == "transformer": transformer_patchsize_x = None # Patch size of vision transformer patches in x direction. transformer_patchsize_y = None # Patch size of vision transformer patches in y direction. @@ -190,6 +192,7 @@ def run(_config, weight_decay, learning_rate, continue_training, + index_start, dir_of_start_model, save_interval, augmentation, @@ -312,6 +315,7 @@ def run(_config, custom_objects = {"PatchEncoder": PatchEncoder, "Patches": Patches}) else: + index_start = 0 if backbone_type == 'nontransformer': model = resnet50_unet(n_classes, input_height, @@ -410,7 +414,8 @@ def run(_config, #validation_steps=1, # rs: only one batch?? validation_steps=steps_val, epochs=n_epochs, - callbacks=callbacks) + callbacks=callbacks, + initial_epoch=index_start) #os.system('rm -rf '+dir_train_flowing) #os.system('rm -rf '+dir_eval_flowing) From 1581094141a2eb8892fa58b09de7fe8500e73e08 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Wed, 4 Feb 2026 17:35:12 +0100 Subject: [PATCH 13/21] training: extend `index_start` to tasks classification and RO --- src/eynollah/training/train.py | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/src/eynollah/training/train.py b/src/eynollah/training/train.py index de8cccd..168884a 100644 --- a/src/eynollah/training/train.py +++ b/src/eynollah/training/train.py @@ -423,11 +423,15 @@ def run(_config, #model.save(dir_output+'/'+'model'+'.h5') elif task=='classification': - model = resnet50_classifier(n_classes, - input_height, - input_width, - weight_decay, - pretraining) + if continue_training: + model = load_model(dir_of_start_model, compile=False) + else: + index_start = 0 + model = resnet50_classifier(n_classes, + input_height, + input_width, + weight_decay, + pretraining) model.compile(loss='categorical_crossentropy', optimizer=Adam(learning_rate=0.001), # rs: why not learning_rate? @@ -453,7 +457,8 @@ def run(_config, verbose=1, epochs=n_epochs, metrics=[F1Score(average='macro', name='f1')], - callbacks=callbacks) + callbacks=callbacks, + initial_epoch=index_start) usable_checkpoints = np.flatnonzero(np.array(history['val_f1']) > f1_threshold_classification) if len(usable_checkpoints) >= 1: @@ -481,8 +486,15 @@ def run(_config, _log.info("ensemble model saved under '%s'", cp_path) elif task=='reading_order': - model = machine_based_reading_order_model( - n_classes, input_height, input_width, weight_decay, pretraining) + if continue_training: + model = load_model(dir_of_start_model, compile=False) + else: + index_start = 0 + model = machine_based_reading_order_model(n_classes, + input_height, + input_width, + weight_decay, + pretraining) dir_flow_train_imgs = os.path.join(dir_train, 'images') dir_flow_train_labels = os.path.join(dir_train, 'labels') @@ -495,7 +507,6 @@ def run(_config, #ls_test = os.listdir(dir_flow_train_labels) #f1score_tot = [0] - indexer_start = 0 model.compile(loss="binary_crossentropy", #optimizer=SGD(learning_rate=0.01, momentum=0.9), optimizer=Adam(learning_rate=0.0001), # rs: why not learning_rate? @@ -515,7 +526,8 @@ def run(_config, steps_per_epoch=num_rows / n_batch, verbose=1, epochs=n_epochs, - callbacks=callbacks) + callbacks=callbacks, + initial_epoch=index_start) ''' if f1score>f1score_tot[0]: f1score_tot[0] = f1score From 7562317da5aa8f4a56c981848d23cb5eec7685d2 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Wed, 4 Feb 2026 17:35:38 +0100 Subject: [PATCH 14/21] training: fix+simplify `load_model` logic for `continue_training` - add missing combination `transformer` (w/ patch encoder and `weighted_loss`) - add assertion to prevent wrong loss type being configured --- src/eynollah/training/train.py | 36 ++++++++++++---------------------- 1 file changed, 13 insertions(+), 23 deletions(-) diff --git a/src/eynollah/training/train.py b/src/eynollah/training/train.py index 168884a..7ede551 100644 --- a/src/eynollah/training/train.py +++ b/src/eynollah/training/train.py @@ -290,30 +290,20 @@ def run(_config, weights = weights / float(np.min(weights)) weights = weights / float(np.sum(weights)) + if task == "enhancement": + assert not is_loss_soft_dice, "for enhancement, soft_dice loss does not apply" + assert not weighted_dice, "for enhancement, weighted loss does not apply" if continue_training: - if backbone_type == 'nontransformer': - if is_loss_soft_dice and task in ["segmentation", "binarization"]: - model = load_model(dir_of_start_model, compile=True, - custom_objects={'soft_dice_loss': soft_dice_loss}) - elif weighted_loss and task in ["segmentation", "binarization"]: - model = load_model(dir_of_start_model, compile=True, - custom_objects={'loss': weighted_categorical_crossentropy(weights)}) - else: - model = load_model(dir_of_start_model , compile=True) - - elif backbone_type == 'transformer': - if is_loss_soft_dice and task in ["segmentation", "binarization"]: - model = load_model(dir_of_start_model, compile=True, - custom_objects={"PatchEncoder": PatchEncoder, - "Patches": Patches, - 'soft_dice_loss': soft_dice_loss}) - elif weighted_loss and task in ["segmentation", "binarization"]: - model = load_model(dir_of_start_model, compile=True, - custom_objects={'loss': weighted_categorical_crossentropy(weights)}) - else: - model = load_model(dir_of_start_model, compile=True, - custom_objects = {"PatchEncoder": PatchEncoder, - "Patches": Patches}) + custom_objects = dict() + if is_loss_soft_dice: + custom_objects.update(soft_dice_loss=soft_dice_loss) + elif weighted_loss: + custom_objects.update(loss=weighted_categorical_crossentropy(weights)) + if backbone_type == 'transformer': + custom_objects.update(PatchEncoder=PatchEncoder, + Patches=Patches) + model = load_model(dir_of_start_model, compile=False, + custom_objects=custom_objects) else: index_start = 0 if backbone_type == 'nontransformer': From 4a65ee0c672640821ebb54dc647a3e027f21fc46 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Thu, 5 Feb 2026 11:53:19 +0100 Subject: [PATCH 15/21] =?UTF-8?q?training.train:=20more=20config=20depende?= =?UTF-8?q?ncies=E2=80=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - make more config_params keys dependent on each other - re-order accordingly - in main, initialise them (as kwarg), so sacred actually allows overriding them by named config file --- src/eynollah/training/train.py | 67 ++++++++++++++++++---------------- 1 file changed, 36 insertions(+), 31 deletions(-) diff --git a/src/eynollah/training/train.py b/src/eynollah/training/train.py index 7ede551..a21a34d 100644 --- a/src/eynollah/training/train.py +++ b/src/eynollah/training/train.py @@ -97,7 +97,17 @@ ex = Experiment(save_git_info=False) @ex.config def config_params(): task = "segmentation" # This parameter defines task of model which can be segmentation, enhancement or classification. - backbone_type = None # Type of image feature map network backbone. Either a vision transformer alongside a CNN we call "transformer", or only a CNN which we call "nontransformer" + if task in ["segmentation", "binarization", "enhancement"]: + backbone_type = "nontransformer" # Type of image feature map network backbone. Either a vision transformer alongside a CNN we call "transformer", or only a CNN which we call "nontransformer" + if backbone_type == "transformer": + transformer_patchsize_x = None # Patch size of vision transformer patches in x direction. + transformer_patchsize_y = None # Patch size of vision transformer patches in y direction. + transformer_num_patches_xy = None # Number of patches for vision transformer in x and y direction respectively. + transformer_projection_dim = 64 # Transformer projection dimension. Default value is 64. + transformer_mlp_head_units = [128, 64] # Transformer Multilayer Perceptron (MLP) head units. Default value is [128, 64] + transformer_layers = 8 # transformer layers. Default value is 8. + transformer_num_heads = 4 # Transformer number of heads. Default value is 4. + transformer_cnn_first = True # We have two types of vision transformers: either the CNN is applied first, followed by the transformer, or reversed. n_classes = None # Number of classes. In the case of binary classification this should be 2. n_epochs = 1 # Number of epochs to train. n_batch = 1 # Number of images per batch at each iteration. (Try as large as fits on VRAM.) @@ -105,10 +115,12 @@ def config_params(): input_width = 224 * 1 # Width of model's input in pixels. weight_decay = 1e-6 # Weight decay of l2 regularization of model layers. learning_rate = 1e-4 # Set the learning rate. - is_loss_soft_dice = False # Use soft dice as loss function. When set to true, "weighted_loss" must be false. - weighted_loss = False # Use weighted categorical cross entropy as loss fucntion. When set to true, "is_loss_soft_dice" must be false. - f1_threshold_classification = None # This threshold is used to consider models with an evaluation f1 scores bigger than it. The selected model weights undergo a weights ensembling. And avreage ensembled model will be written to output. - classification_classes_name = None # Dictionary of classification classes names. + if task in ["segmentation", "binarization"]: + is_loss_soft_dice = False # Use soft dice as loss function. When set to true, "weighted_loss" must be false. + weighted_loss = False # Use weighted categorical cross entropy as loss fucntion. When set to true, "is_loss_soft_dice" must be false. + elif task == "classification": + f1_threshold_classification = None # This threshold is used to consider models with an evaluation f1 scores bigger than it. The selected model weights undergo a weights ensembling. And avreage ensembled model will be written to output. + classification_classes_name = None # Dictionary of classification classes names. patches = False # Divides input image into smaller patches (input size of the model) when set to true. For the model to see the full image, like page extraction, set this to false. augmentation = False # To apply any kind of augmentation, this parameter must be set to true. if augmentation: @@ -163,17 +175,8 @@ def config_params(): dir_of_start_model = '' # Directory of model checkpoint to load to continue training. (E.g. if you already trained for 3 epochs, set "dir_of_start_model=dir_output/model_03".) index_start = 0 # Epoch counter initial value to continue training. (E.g. if you already trained for 3 epochs, set "index_start=3" to continue naming checkpoints model_04, model_05 etc.) data_is_provided = False # Whether the preprocessed input data (subdirectories "images" and "labels" in both subdirectories "train" and "eval" of "dir_output") has already been generated (in the first epoch of a previous run). - if backbone_type == "transformer": - transformer_patchsize_x = None # Patch size of vision transformer patches in x direction. - transformer_patchsize_y = None # Patch size of vision transformer patches in y direction. - transformer_num_patches_xy = None # Number of patches for vision transformer in x and y direction respectively. - transformer_projection_dim = 64 # Transformer projection dimension. Default value is 64. - transformer_mlp_head_units = [128, 64] # Transformer Multilayer Perceptron (MLP) head units. Default value is [128, 64] - transformer_layers = 8 # transformer layers. Default value is 8. - transformer_num_heads = 4 # Transformer number of heads. Default value is 4. - transformer_cnn_first = True # We have two types of vision transformers: either the CNN is applied first, followed by the transformer, or reversed. -@ex.automain +@ex.main def run(_config, _log, task, @@ -187,27 +190,29 @@ def run(_config, n_batch, input_height, input_width, - is_loss_soft_dice, - weighted_loss, weight_decay, learning_rate, continue_training, - index_start, - dir_of_start_model, save_interval, augmentation, - thetha, - backbone_type, - transformer_projection_dim, - transformer_mlp_head_units, - transformer_layers, - transformer_num_heads, - transformer_cnn_first, - transformer_patchsize_x, - transformer_patchsize_y, - transformer_num_patches_xy, - f1_threshold_classification, - classification_classes_name, + # dependent config keys need a default, + # otherwise yields sacred.utils.ConfigAddedError + thetha=None, + is_loss_soft_dice=False, + weighted_loss=False, + index_start=0, + dir_of_start_model=None, + backbone_type=None, + transformer_projection_dim=None, + transformer_mlp_head_units=None, + transformer_layers=None, + transformer_num_heads=None, + transformer_cnn_first=None, + transformer_patchsize_x=None, + transformer_patchsize_y=None, + transformer_num_patches_xy=None, + f1_threshold_classification=None, + classification_classes_name=None, ): if pretraining and not os.path.isfile(RESNET50_WEIGHTS_PATH): From 5c7801a1d6273cd88b64548edf41507e5c0235d6 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Thu, 5 Feb 2026 11:56:11 +0100 Subject: [PATCH 16/21] training.train: simplify config args for model builder --- src/eynollah/training/models.py | 67 +++++++++++++++++++++++---------- src/eynollah/training/train.py | 33 ++++++++-------- 2 files changed, 63 insertions(+), 37 deletions(-) diff --git a/src/eynollah/training/models.py b/src/eynollah/training/models.py index 011c614..f053447 100644 --- a/src/eynollah/training/models.py +++ b/src/eynollah/training/models.py @@ -400,9 +400,21 @@ def resnet50_unet(n_classes, input_height=224, input_width=224, task="segmentati return model -def vit_resnet50_unet(n_classes, patch_size_x, patch_size_y, num_patches, mlp_head_units=None, transformer_layers=8, num_heads =4, projection_dim = 64, input_height=224, input_width=224, task="segmentation", weight_decay=1e-6, pretraining=False): - if mlp_head_units is None: - mlp_head_units = [128, 64] +def vit_resnet50_unet(num_patches, + n_classes, + transformer_patchsize_x, + transformer_patchsize_y, + transformer_mlp_head_units=None, + transformer_layers=8, + transformer_num_heads=4, + transformer_projection_dim=64, + input_height=224, + input_width=224, + task="segmentation", + weight_decay=1e-6, + pretraining=False): + if transformer_mlp_head_units is None: + transformer_mlp_head_units = [128, 64] inputs = layers.Input(shape=(input_height, input_width, 3)) #transformer_units = [ @@ -449,30 +461,30 @@ def vit_resnet50_unet(n_classes, patch_size_x, patch_size_y, num_patches, mlp_he #num_patches = x.shape[1]*x.shape[2] - #patch_size_y = input_height / x.shape[1] - #patch_size_x = input_width / x.shape[2] - #patch_size = patch_size_x * patch_size_y - patches = Patches(patch_size_x, patch_size_y)(x) + patches = Patches(transformer_patchsize_x, transformer_patchsize_y)(x) # Encode patches. - encoded_patches = PatchEncoder(num_patches, projection_dim)(patches) + encoded_patches = PatchEncoder(num_patches, transformer_projection_dim)(patches) for _ in range(transformer_layers): # Layer normalization 1. x1 = layers.LayerNormalization(epsilon=1e-6)(encoded_patches) # Create a multi-head attention layer. attention_output = layers.MultiHeadAttention( - num_heads=num_heads, key_dim=projection_dim, dropout=0.1 + num_heads=transformer_num_heads, key_dim=transformer_projection_dim, dropout=0.1 )(x1, x1) # Skip connection 1. x2 = layers.Add()([attention_output, encoded_patches]) # Layer normalization 2. x3 = layers.LayerNormalization(epsilon=1e-6)(x2) # MLP. - x3 = mlp(x3, hidden_units=mlp_head_units, dropout_rate=0.1) + x3 = mlp(x3, hidden_units=transformer_mlp_head_units, dropout_rate=0.1) # Skip connection 2. encoded_patches = layers.Add()([x3, x2]) - encoded_patches = tf.reshape(encoded_patches, [-1, x.shape[1], x.shape[2] , int( projection_dim / (patch_size_x * patch_size_y) )]) + encoded_patches = tf.reshape(encoded_patches, + [-1, x.shape[1], x.shape[2], + transformer_projection_dim // (transformer_patchsize_x * + transformer_patchsize_y)]) v1024_2048 = Conv2D( 1024 , (1, 1), padding='same', data_format=IMAGE_ORDERING,kernel_regularizer=l2(weight_decay))(encoded_patches) v1024_2048 = (BatchNormalization(axis=bn_axis))(v1024_2048) @@ -524,9 +536,21 @@ def vit_resnet50_unet(n_classes, patch_size_x, patch_size_y, num_patches, mlp_he return model -def vit_resnet50_unet_transformer_before_cnn(n_classes, patch_size_x, patch_size_y, num_patches, mlp_head_units=None, transformer_layers=8, num_heads =4, projection_dim = 64, input_height=224, input_width=224, task="segmentation", weight_decay=1e-6, pretraining=False): - if mlp_head_units is None: - mlp_head_units = [128, 64] +def vit_resnet50_unet_transformer_before_cnn(num_patches, + n_classes, + transformer_patchsize_x, + transformer_patchsize_y, + transformer_mlp_head_units=None, + transformer_layers=8, + transformer_num_heads=4, + transformer_projection_dim=64, + input_height=224, + input_width=224, + task="segmentation", + weight_decay=1e-6, + pretraining=False): + if transformer_mlp_head_units is None: + transformer_mlp_head_units = [128, 64] inputs = layers.Input(shape=(input_height, input_width, 3)) ##transformer_units = [ @@ -536,27 +560,32 @@ def vit_resnet50_unet_transformer_before_cnn(n_classes, patch_size_x, patch_size IMAGE_ORDERING = 'channels_last' bn_axis=3 - patches = Patches(patch_size_x, patch_size_y)(inputs) + patches = Patches(transformer_patchsize_x, transformer_patchsize_y)(inputs) # Encode patches. - encoded_patches = PatchEncoder(num_patches, projection_dim)(patches) + encoded_patches = PatchEncoder(num_patches, transformer_projection_dim)(patches) for _ in range(transformer_layers): # Layer normalization 1. x1 = layers.LayerNormalization(epsilon=1e-6)(encoded_patches) # Create a multi-head attention layer. attention_output = layers.MultiHeadAttention( - num_heads=num_heads, key_dim=projection_dim, dropout=0.1 + num_heads=transformer_num_heads, key_dim=transformer_projection_dim, dropout=0.1 )(x1, x1) # Skip connection 1. x2 = layers.Add()([attention_output, encoded_patches]) # Layer normalization 2. x3 = layers.LayerNormalization(epsilon=1e-6)(x2) # MLP. - x3 = mlp(x3, hidden_units=mlp_head_units, dropout_rate=0.1) + x3 = mlp(x3, hidden_units=transformer_mlp_head_units, dropout_rate=0.1) # Skip connection 2. encoded_patches = layers.Add()([x3, x2]) - encoded_patches = tf.reshape(encoded_patches, [-1, input_height, input_width , int( projection_dim / (patch_size_x * patch_size_y) )]) + encoded_patches = tf.reshape(encoded_patches, + [-1, + input_height, + input_width, + transformer_projection_dim // (transformer_patchsize_x * + transformer_patchsize_y)]) encoded_patches = Conv2D(3, (1, 1), padding='same', data_format=IMAGE_ORDERING, kernel_regularizer=l2(weight_decay), name='convinput')(encoded_patches) diff --git a/src/eynollah/training/train.py b/src/eynollah/training/train.py index a21a34d..4aafcf2 100644 --- a/src/eynollah/training/train.py +++ b/src/eynollah/training/train.py @@ -38,6 +38,7 @@ from tensorflow.keras.metrics import MeanIoU from tensorflow.keras.models import load_model from tensorflow.keras.callbacks import ModelCheckpoint, TensorBoard from sacred import Experiment +from sacred.config import create_captured_function from tqdm import tqdm from sklearn.metrics import f1_score @@ -318,7 +319,7 @@ def run(_config, task, weight_decay, pretraining) - elif backbone_type == 'transformer': + else: num_patches_x = transformer_num_patches_xy[0] num_patches_y = transformer_num_patches_xy[1] num_patches = num_patches_x * num_patches_y @@ -330,35 +331,31 @@ def run(_config, model_builder = vit_resnet50_unet_transformer_before_cnn multiple_of_32 = False - assert input_height == num_patches_y * transformer_patchsize_y * (32 if multiple_of_32 else 1), \ + assert input_height == (num_patches_y * + transformer_patchsize_y * + (32 if multiple_of_32 else 1)), \ "transformer_patchsize_y or transformer_num_patches_xy height value error: " \ "input_height should be equal to " \ "(transformer_num_patches_xy height value * transformer_patchsize_y%s)" % \ " * 32" if multiple_of_32 else "" - assert input_width == num_patches_x * transformer_patchsize_x * (32 if multiple_of_32 else 1), \ + assert input_width == (num_patches_x * + transformer_patchsize_x * + (32 if multiple_of_32 else 1)), \ "transformer_patchsize_x or transformer_num_patches_xy width value error: " \ "input_width should be equal to " \ "(transformer_num_patches_xy width value * transformer_patchsize_x%s)" % \ " * 32" if multiple_of_32 else "" - assert 0 == transformer_projection_dim % (transformer_patchsize_y * transformer_patchsize_x), \ + assert 0 == (transformer_projection_dim % + (transformer_patchsize_y * + transformer_patchsize_x)), \ "transformer_projection_dim error: " \ "The remainder when parameter transformer_projection_dim is divided by " \ "(transformer_patchsize_y*transformer_patchsize_x) should be zero" - model = model_builder( - n_classes, - transformer_patchsize_x, - transformer_patchsize_y, - num_patches, - transformer_mlp_head_units, - transformer_layers, - transformer_num_heads, - transformer_projection_dim, - input_height, - input_width, - task, - weight_decay, - pretraining) + model_builder = create_captured_function(model_builder) + model_builder.config = _config + model_builder.logger = _log + model = model_builder(num_patches) #if you want to see the model structure just uncomment model summary. #model.summary() From 82d649061a7d932df25828081c01b25a6acae012 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Thu, 5 Feb 2026 11:57:38 +0100 Subject: [PATCH 17/21] training.train: fix F1 metric score setup --- src/eynollah/training/train.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/eynollah/training/train.py b/src/eynollah/training/train.py index 4aafcf2..effc920 100644 --- a/src/eynollah/training/train.py +++ b/src/eynollah/training/train.py @@ -34,7 +34,7 @@ os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' os.environ['TF_USE_LEGACY_KERAS'] = '1' # avoid Keras 3 after TF 2.15 import tensorflow as tf from tensorflow.keras.optimizers import SGD, Adam -from tensorflow.keras.metrics import MeanIoU +from tensorflow.keras.metrics import MeanIoU, F1Score from tensorflow.keras.models import load_model from tensorflow.keras.callbacks import ModelCheckpoint, TensorBoard from sacred import Experiment @@ -427,8 +427,8 @@ def run(_config, model.compile(loss='categorical_crossentropy', optimizer=Adam(learning_rate=0.001), # rs: why not learning_rate? - metrics=['accuracy']) - + metrics=['accuracy', F1Score(average='macro', name='f1')]) + list_classes = list(classification_classes_name.values()) trainXY = generate_data_from_folder_training( dir_train, n_batch, input_height, input_width, n_classes, list_classes) @@ -440,7 +440,8 @@ def run(_config, callbacks = [TensorBoard(os.path.join(dir_output, 'logs'), write_graph=False), SaveWeightsAfterSteps(0, dir_output, _config, monitor='val_f1', - save_best_only=True, mode='max')] + #save_best_only=True, # we need all for ensembling + mode='max')] history = model.fit(trainXY, steps_per_epoch=num_rows / n_batch, @@ -448,17 +449,17 @@ def run(_config, validation_data=testXY, verbose=1, epochs=n_epochs, - metrics=[F1Score(average='macro', name='f1')], callbacks=callbacks, initial_epoch=index_start) - usable_checkpoints = np.flatnonzero(np.array(history['val_f1']) > f1_threshold_classification) + usable_checkpoints = np.flatnonzero(np.array(history.history['val_f1']) > + f1_threshold_classification) if len(usable_checkpoints) >= 1: _log.info("averaging over usable checkpoints: %s", str(usable_checkpoints)) all_weights = [] for epoch in usable_checkpoints: - cp_path = os.path.join(dir_output, 'model_{epoch:02d}'.format(epoch=epoch)) - assert os.path.isdir(cp_path) + cp_path = os.path.join(dir_output, 'model_{epoch:02d}'.format(epoch=epoch + 1)) + assert os.path.isdir(cp_path), cp_path model = load_model(cp_path, compile=False) all_weights.append(model.get_weights()) From f03124f747db7edef03d968e1b10db0e7638850d Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Thu, 5 Feb 2026 11:58:50 +0100 Subject: [PATCH 18/21] =?UTF-8?q?training.train:=20simplify+fix=20classifi?= =?UTF-8?q?cation=20data=20loaders=E2=80=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - unify `generate_data_from_folder_training` w/ `..._evaluation` - instead of recreating array after every batch, just zero out - cast image results to uint8 instead of uint16 - cast categorical results to float instead of int --- src/eynollah/training/train.py | 15 ++++--- src/eynollah/training/utils.py | 78 ++++++++-------------------------- 2 files changed, 25 insertions(+), 68 deletions(-) diff --git a/src/eynollah/training/train.py b/src/eynollah/training/train.py index effc920..0f8d0e9 100644 --- a/src/eynollah/training/train.py +++ b/src/eynollah/training/train.py @@ -430,13 +430,13 @@ def run(_config, metrics=['accuracy', F1Score(average='macro', name='f1')]) list_classes = list(classification_classes_name.values()) - trainXY = generate_data_from_folder_training( - dir_train, n_batch, input_height, input_width, n_classes, list_classes) - testXY = generate_data_from_folder_evaluation( - dir_eval, input_height, input_width, n_classes, list_classes) + trainXY = generate_data_from_folder( + dir_train, n_batch, input_height, input_width, n_classes, list_classes, shuffle=True) + testXY = generate_data_from_folder( + dir_eval, n_batch, input_height, input_width, n_classes, list_classes) + epoch_size_train = return_number_of_total_training_data(dir_train) + epoch_size_eval = return_number_of_total_training_data(dir_eval) - y_tot = np.zeros((testX.shape[0], n_classes)) - num_rows = return_number_of_total_training_data(dir_train) callbacks = [TensorBoard(os.path.join(dir_output, 'logs'), write_graph=False), SaveWeightsAfterSteps(0, dir_output, _config, monitor='val_f1', @@ -444,9 +444,10 @@ def run(_config, mode='max')] history = model.fit(trainXY, - steps_per_epoch=num_rows / n_batch, + steps_per_epoch=epoch_size_train // n_batch, #class_weight=weights) validation_data=testXY, + validation_steps=epoch_size_eval // n_batch, verbose=1, epochs=n_epochs, callbacks=callbacks, diff --git a/src/eynollah/training/utils.py b/src/eynollah/training/utils.py index 61b2536..5b25a4f 100644 --- a/src/eynollah/training/utils.py +++ b/src/eynollah/training/utils.py @@ -166,50 +166,7 @@ def return_number_of_total_training_data(path_classes): -def generate_data_from_folder_evaluation(path_classes, height, width, n_classes, list_classes): - #sub_classes = os.listdir(path_classes) - #n_classes = len(sub_classes) - all_imgs = [] - labels = [] - #dicts =dict() - #indexer= 0 - for indexer, sub_c in enumerate(list_classes): - sub_files = os.listdir(os.path.join(path_classes,sub_c )) - sub_files = [os.path.join(path_classes,sub_c )+'/' + x for x in sub_files] - #print( os.listdir(os.path.join(path_classes,sub_c )) ) - all_imgs = all_imgs + sub_files - sub_labels = list( np.zeros( len(sub_files) ) +indexer ) - - #print( len(sub_labels) ) - labels = labels + sub_labels - #dicts[sub_c] = indexer - #indexer +=1 - - - categories = to_categorical(range(n_classes)).astype(np.int16)#[ [1 , 0, 0 , 0 , 0 , 0] , [0 , 1, 0 , 0 , 0 , 0] , [0 , 0, 1 , 0 , 0 , 0] , [0 , 0, 0 , 1 , 0 , 0] , [0 , 0, 0 , 0 , 1 , 0] , [0 , 0, 0 , 0 , 0 , 1] ] - ret_x= np.zeros((len(labels), height,width, 3)).astype(np.int16) - ret_y= np.zeros((len(labels), n_classes)).astype(np.int16) - - #print(all_imgs) - for i in range(len(all_imgs)): - row = all_imgs[i] - #####img = cv2.imread(row, 0) - #####img= resize_image (img, height, width) - #####img = img.astype(np.uint16) - #####ret_x[i, :,:,0] = img[:,:] - #####ret_x[i, :,:,1] = img[:,:] - #####ret_x[i, :,:,2] = img[:,:] - - img = cv2.imread(row) - img= resize_image (img, height, width) - img = img.astype(np.uint16) - ret_x[i, :,:] = img[:,:,:] - - ret_y[i, :] = categories[ int( labels[i] ) ][:] - - return ret_x/255., ret_y - -def generate_data_from_folder_training(path_classes, batchsize, height, width, n_classes, list_classes): +def generate_data_from_folder(path_classes, batchsize, height, width, n_classes, list_classes, shuffle=False): #sub_classes = os.listdir(path_classes) #n_classes = len(sub_classes) @@ -228,43 +185,42 @@ def generate_data_from_folder_training(path_classes, batchsize, height, width, n labels = labels + sub_labels #dicts[sub_c] = indexer #indexer +=1 - - ids = np.array(range(len(labels))) - random.shuffle(ids) - - shuffled_labels = np.array(labels)[ids] - shuffled_files = np.array(all_imgs)[ids] + + if shuffle: + ids = np.array(range(len(labels))) + random.shuffle(ids) + labels = np.array(labels)[ids] + all_imgs = np.array(all_imgs)[ids] + categories = to_categorical(range(n_classes)).astype(np.int16)#[ [1 , 0, 0 , 0 , 0 , 0] , [0 , 1, 0 , 0 , 0 , 0] , [0 , 0, 1 , 0 , 0 , 0] , [0 , 0, 0 , 1 , 0 , 0] , [0 , 0, 0 , 0 , 1 , 0] , [0 , 0, 0 , 0 , 0 , 1] ] - ret_x= np.zeros((batchsize, height,width, 3)).astype(np.int16) - ret_y= np.zeros((batchsize, n_classes)).astype(np.int16) + ret_x= np.zeros((batchsize, height,width, 3)).astype(np.uint8) + ret_y= np.zeros((batchsize, n_classes)).astype(float) batchcount = 0 while True: - for i in range(len(shuffled_files)): - row = shuffled_files[i] - #print(row) - ###img = cv2.imread(row, 0) + for lab, img in zip(labels, all_imgs): + ###img = cv2.imread(img, 0) ###img= resize_image (img, height, width) ###img = img.astype(np.uint16) ###ret_x[batchcount, :,:,0] = img[:,:] ###ret_x[batchcount, :,:,1] = img[:,:] ###ret_x[batchcount, :,:,2] = img[:,:] - img = cv2.imread(row) + img = cv2.imread(img) img= resize_image (img, height, width) img = img.astype(np.uint16) ret_x[batchcount, :,:,:] = img[:,:,:] #print(int(shuffled_labels[i]) ) #print( categories[int(shuffled_labels[i])] ) - ret_y[batchcount, :] = categories[ int( shuffled_labels[i] ) ][:] + ret_y[batchcount, :] = categories[int(lab)][:] batchcount+=1 if batchcount>=batchsize: - ret_x = ret_x/255. + ret_x = ret_x//255 yield ret_x, ret_y - ret_x= np.zeros((batchsize, height,width, 3)).astype(np.int16) - ret_y= np.zeros((batchsize, n_classes)).astype(np.int16) + ret_x[:] = 0 + ret_y[:] = 0 batchcount = 0 def do_brightening(img, factor): From 5d0c26b629dc0f7368c7d2058a2efbd0ac27a911 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Thu, 5 Feb 2026 12:02:58 +0100 Subject: [PATCH 19/21] training.train: use std Keras data loader for classification (much more efficient, works with std F1 metric) --- src/eynollah/training/train.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/src/eynollah/training/train.py b/src/eynollah/training/train.py index 0f8d0e9..7cf7536 100644 --- a/src/eynollah/training/train.py +++ b/src/eynollah/training/train.py @@ -23,8 +23,6 @@ from eynollah.training.models import ( from eynollah.training.utils import ( data_gen, generate_arrays_from_folder_reading_order, - generate_data_from_folder_evaluation, - generate_data_from_folder_training, get_one_hot, preprocess_imgs, return_number_of_total_training_data @@ -37,6 +35,7 @@ from tensorflow.keras.optimizers import SGD, Adam from tensorflow.keras.metrics import MeanIoU, F1Score from tensorflow.keras.models import load_model from tensorflow.keras.callbacks import ModelCheckpoint, TensorBoard +from tensorflow.keras.utils import image_dataset_from_directory from sacred import Experiment from sacred.config import create_captured_function from tqdm import tqdm @@ -430,13 +429,13 @@ def run(_config, metrics=['accuracy', F1Score(average='macro', name='f1')]) list_classes = list(classification_classes_name.values()) - trainXY = generate_data_from_folder( - dir_train, n_batch, input_height, input_width, n_classes, list_classes, shuffle=True) - testXY = generate_data_from_folder( - dir_eval, n_batch, input_height, input_width, n_classes, list_classes) - epoch_size_train = return_number_of_total_training_data(dir_train) - epoch_size_eval = return_number_of_total_training_data(dir_eval) - + data_args = dict(label_mode="categorical", + class_names=list_classes, + batch_size=n_batch, + image_size=(input_height, input_width), + interpolation="nearest") + trainXY = image_dataset_from_directory(dir_train, shuffle=True, **data_args) + testXY = image_dataset_from_directory(dir_eval, shuffle=False, **data_args) callbacks = [TensorBoard(os.path.join(dir_output, 'logs'), write_graph=False), SaveWeightsAfterSteps(0, dir_output, _config, monitor='val_f1', @@ -444,10 +443,8 @@ def run(_config, mode='max')] history = model.fit(trainXY, - steps_per_epoch=epoch_size_train // n_batch, #class_weight=weights) validation_data=testXY, - validation_steps=epoch_size_eval // n_batch, verbose=1, epochs=n_epochs, callbacks=callbacks, From b1633dfc7cf9cdfd84586b2fe367a8bd239fc2cf Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Thu, 5 Feb 2026 14:53:26 +0100 Subject: [PATCH 20/21] training.generate_gt: for RO, skip files if regionRefs are missing --- .../training/generate_gt_for_training.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/eynollah/training/generate_gt_for_training.py b/src/eynollah/training/generate_gt_for_training.py index 693cab8..f71614c 100644 --- a/src/eynollah/training/generate_gt_for_training.py +++ b/src/eynollah/training/generate_gt_for_training.py @@ -205,14 +205,20 @@ def machine_based_reading_order(dir_xml, dir_out_modal_image, dir_out_classes, i img_header_and_sep = np.zeros((y_len,x_len), dtype='uint8') for j in range(len(cy_main)): - img_header_and_sep[int(y_max_main[j]):int(y_max_main[j])+12,int(x_min_main[j]):int(x_max_main[j]) ] = 1 + img_header_and_sep[int(y_max_main[j]):int(y_max_main[j])+12, + int(x_min_main[j]):int(x_max_main[j]) ] = 1 - texts_corr_order_index = [index_tot_regions[tot_region_ref.index(i)] for i in id_all_text ] - texts_corr_order_index_int = [int(x) for x in texts_corr_order_index] - + try: + texts_corr_order_index_int = [int(index_tot_regions[tot_region_ref.index(i)]) + for i in id_all_text] + except ValueError as e: + print("incomplete ReadingOrder in", xml_file, "- skipping:", str(e)) + continue - co_text_all, texts_corr_order_index_int, regions_ar_less_than_early_min = filter_contours_area_of_image(img_poly, co_text_all, texts_corr_order_index_int, max_area, min_area, min_area_early) + co_text_all, texts_corr_order_index_int, regions_ar_less_than_early_min = \ + filter_contours_area_of_image(img_poly, co_text_all, texts_corr_order_index_int, + max_area, min_area, min_area_early) arg_array = np.array(range(len(texts_corr_order_index_int))) From 0d3a8eacba67f6fc6b8bec0fbe6ea12d4d1b948f Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Thu, 5 Feb 2026 14:54:08 +0100 Subject: [PATCH 21/21] improve/update docs/train.md --- docs/train.md | 110 +++++++++++++++++++++++++++++--------------------- 1 file changed, 64 insertions(+), 46 deletions(-) diff --git a/docs/train.md b/docs/train.md index 4e76740..3c64ab9 100644 --- a/docs/train.md +++ b/docs/train.md @@ -343,51 +343,17 @@ The following parameter configuration can be applied to all segmentation use cas its sub-parameters, and continued training are defined only for segmentation use cases and enhancements, not for classification and machine-based reading order, as you can see in their example config files. -* `backbone_type`: For segmentation tasks (such as text line, binarization, and layout detection) and enhancement, we - offer two backbone options: a "nontransformer" and a "transformer" backbone. For the "transformer" backbone, we first - apply a CNN followed by a transformer. In contrast, the "nontransformer" backbone utilizes only a CNN ResNet-50. -* `task`: The task parameter can have values such as "segmentation", "enhancement", "classification", and "reading_order". -* `patches`: If you want to break input images into smaller patches (input size of the model) you need to set this -* parameter to `true`. In the case that the model should see the image once, like page extraction, patches should be - set to ``false``. -* `n_batch`: Number of batches at each iteration. -* `n_classes`: Number of classes. In the case of binary classification this should be 2. In the case of reading_order it - should set to 1. And for the case of layout detection just the unique number of classes should be given. -* `n_epochs`: Number of epochs. -* `input_height`: This indicates the height of model's input. -* `input_width`: This indicates the width of model's input. -* `weight_decay`: Weight decay of l2 regularization of model layers. -* `pretraining`: Set to `true` to load pretrained weights of ResNet50 encoder. The downloaded weights should be saved - in a folder named "pretrained_model" in the same directory of "train.py" script. -* `augmentation`: If you want to apply any kind of augmentation this parameter should first set to `true`. -* `flip_aug`: If `true`, different types of filp will be applied on image. Type of flips is given with "flip_index" parameter. -* `blur_aug`: If `true`, different types of blurring will be applied on image. Type of blurrings is given with "blur_k" parameter. -* `scaling`: If `true`, scaling will be applied on image. Scale of scaling is given with "scales" parameter. -* `degrading`: If `true`, degrading will be applied to the image. The amount of degrading is defined with "degrade_scales" parameter. -* `brightening`: If `true`, brightening will be applied to the image. The amount of brightening is defined with "brightness" parameter. -* `rotation_not_90`: If `true`, rotation (not 90 degree) will be applied on image. Rotation angles are given with "thetha" parameter. -* `rotation`: If `true`, 90 degree rotation will be applied on image. -* `binarization`: If `true`,Otsu thresholding will be applied to augment the input data with binarized images. -* `scaling_bluring`: If `true`, combination of scaling and blurring will be applied on image. -* `scaling_binarization`: If `true`, combination of scaling and binarization will be applied on image. -* `scaling_flip`: If `true`, combination of scaling and flip will be applied on image. -* `flip_index`: Type of flips. -* `blur_k`: Type of blurrings. -* `scales`: Scales of scaling. -* `brightness`: The amount of brightenings. -* `thetha`: Rotation angles. -* `degrade_scales`: The amount of degradings. -* `continue_training`: If `true`, it means that you have already trained a model and you would like to continue the - training. So it is needed to providethe dir of trained model with "dir_of_start_model" and index for naming - themodels. For example if you have already trained for 3 epochs then your lastindex is 2 and if you want to continue - from model_1.h5, you can set `index_start` to 3 to start naming model with index 3. -* `weighted_loss`: If `true`, this means that you want to apply weighted categorical_crossentropy as loss fucntion. Be carefull if you set to `true`the parameter "is_loss_soft_dice" should be ``false`` -* `data_is_provided`: If you have already provided the input data you can set this to `true`. Be sure that the train - and eval data are in"dir_output".Since when once we provide training data we resize and augmentthem and then wewrite - them in sub-directories train and eval in "dir_output". -* `dir_train`: This is the directory of "images" and "labels" (dir_train should include two subdirectories with names of images and labels ) for raw images and labels. Namely they are not prepared (not resized and not augmented) yet for training the model. When we run this tool these raw data will be transformed to suitable size needed for the model and they will be written in "dir_output" in train and eval directories. Each of train and eval include "images" and "labels" sub-directories. -* `index_start`: Starting index for saved models in the case that "continue_training" is `true`. -* `dir_of_start_model`: Directory containing pretrained model to continue training the model in the case that "continue_training" is `true`. +* `task`: The task parameter must be one of the following values: + - `binarization`, + - `enhancement`, + - `segmentation`, + - `classification`, + - `reading_order`. +* `backbone_type`: For the tasks `segmentation` (such as text line, and region layout detection), + `binarization` and `enhancement`, we offer two backbone options: + - `nontransformer` (only a CNN ResNet-50). + - `transformer` (first apply a CNN, followed by a transformer) +* `transformer_cnn_first`: Whether to apply the CNN first (followed by the transformer) when using `transformer` backbone. * `transformer_num_patches_xy`: Number of patches for vision transformer in x and y direction respectively. * `transformer_patchsize_x`: Patch size of vision transformer patches in x direction. * `transformer_patchsize_y`: Patch size of vision transformer patches in y direction. @@ -395,7 +361,59 @@ classification and machine-based reading order, as you can see in their example * `transformer_mlp_head_units`: Transformer Multilayer Perceptron (MLP) head units. Default value is [128, 64]. * `transformer_layers`: transformer layers. Default value is 8. * `transformer_num_heads`: Transformer number of heads. Default value is 4. -* `transformer_cnn_first`: We have two types of vision transformers. In one type, a CNN is applied first, followed by a transformer. In the other type, this order is reversed. If transformer_cnn_first is true, it means the CNN will be applied before the transformer. Default value is true. +* `patches`: Whether to break up (tile) input images into smaller patches (input size of the model). + If `false`, the model will see the image once (resized to the input size of the model). + Should be set to `false` for cases like page extraction. +* `n_batch`: Number of batches at each iteration. +* `n_classes`: Number of classes. In the case of binary classification this should be 2. In the case of reading_order it + should set to 1. And for the case of layout detection just the unique number of classes should be given. +* `n_epochs`: Number of epochs (iterations over the data) to train. +* `input_height`: the image height for the model's input. +* `input_width`: the image width for the model's input. +* `weight_decay`: Weight decay of l2 regularization of model layers. +* `weighted_loss`: If `true`, this means that you want to apply weighted categorical crossentropy as loss function. + (Mutually exclusive with `is_loss_soft_dice`, and only applies for `segmentation` and `binarization` tasks.) +* `pretraining`: Set to `true` to (download and) initialise pretrained weights of ResNet50 encoder. +* `dir_train`: Path to directory of raw training data (as extracted via `pagexml2labels`, i.e. with subdirectories + `images` and `labels` for input images and output labels. + (These are not prepared for training the model, yet. Upon first run, the raw data will be transformed to suitable size + needed for the model, and written in `dir_output` under `train` and `eval` subdirectories. See `data_is_provided`.) +* `dir_eval`: Ditto for raw evaluation data. +* `dir_output`: Directory to write model checkpoints, logs (for Tensorboard) and precomputed images to. +* `data_is_provided`: If you have already trained at least one complete epoch (using the same data settings) before, + you can set this to `true` to avoid computing the resized / patched / augmented image files again. + Be sure that there are subdirectories `train` and `eval` data are in `dir_output` (each with subdirectories `images` + and `labels`, respectively). +* `continue_training`: If `true`, continue training a model checkpoint from a previous run. + This requires providing the directory of the model checkpoint to load via `dir_of_start_model` + and setting `index_start` counter for naming new checkpoints. + For example if you have already trained for 3 epochs, then your last index is 2, so if you want + to continue with `model_04`, `model_05` etc., set `index_start=3`. +* `index_start`: Starting index for saving models in the case that `continue_training` is `true`. + (Existing checkpoints above this will be overwritten.) +* `dir_of_start_model`: Directory containing existing model checkpoint to initialise model weights from when `continue_training=true`. + (Can be an epoch-interval checkpoint, or batch-interval checkpoint from `save_interval`.) +* `augmentation`: If you want to apply any kind of augmentation this parameter should first set to `true`. + The remaining settings pertain to that... +* `flip_aug`: If `true`, different types of flipping over the image arrays. Requires `flip_index` parameter. +* `flip_index`: List of flip codes (as in `cv2.flip`, i.e. 0 for vertical, positive for horizontal shift, negative for vertical and horizontal shift). +* `blur_aug`: If `true`, different types of blurring will be applied on image. Requires `blur_k` parameter. +* `blur_k`: Method of blurring (`gauss`, `median` or `blur`). +* `scaling`: If `true`, scaling will be applied on image. Requires `scales` parameter. +* `scales`: List of scale factors for scaling. +* `scaling_bluring`: If `true`, combination of scaling and blurring will be applied on image. +* `scaling_binarization`: If `true`, combination of scaling and binarization will be applied on image. +* `scaling_flip`: If `true`, combination of scaling and flip will be applied on image. +* `degrading`: If `true`, degrading will be applied to the image. Requires `degrade_scales` parameter. +* `degrade_scales`: List of intensity factors for degrading. +* `brightening`: If `true`, brightening will be applied to the image. Requires `brightness` parameter. +* `brightness`: List of intensity factors for brightening. +* `binarization`: If `true`, Otsu thresholding will be applied to augment the input data with binarized images. +* `dir_img_bin`: With `binarization`, use this directory to read precomputed binarized images instead of ad-hoc Otsu. + (Base names should correspond to the files in `dir_train/images`.) +* `rotation`: If `true`, 90° rotation will be applied on images. +* `rotation_not_90`: If `true`, random rotation (other than 90°) will be applied on image. Requires `thetha` parameter. +* `thetha`: List of rotation angles (in degrees). In case of segmentation and enhancement the train and evaluation data should be organised as follows.