qurator-spk/eynollah
1
0
Fork 0
mirror of https://github.com/qurator-spk/eynollah.git synced 2026-02-20 16:32:03 +01:00
Code Issues Projects Releases Wiki Activity

training.train.config_params: re-organise to reflect dependencies

Browse source 
- re-order keys belonging together logically
- make keys dependent on each other
This commit is contained in:
Robert Sachunsky 2026-01-28 13:49:23 +01:00
parent 0372fd7a1e
commit e69b35b49c
1 changed files with 115 additions and 107 deletions
Download patch file Download diff file Expand all files Collapse all files

166
src/eynollah/training/train.py
View file

@ -95,46 +95,72 @@ ex = Experiment(save_git_info=False)
@ex.config
def config_params():
task = "segmentation" # This parameter defines task of model which can be segmentation, enhancement or classification.
backbone_type = None # Type of image feature map network backbone. Either a vision transformer alongside a CNN we call "transformer", or only a CNN which we call "nontransformer"
n_classes = None # Number of classes. In the case of binary classification this should be 2.
n_epochs = 1 # Number of epochs.
n_epochs = 1 # Number of epochs to train.
n_batch = 1 # Number of images per batch at each iteration. (Try as large as fits on VRAM.)
input_height = 224 * 1 # Height of model's input in pixels.
input_width = 224 * 1 # Width of model's input in pixels.
weight_decay = 1e-6 # Weight decay of l2 regularization of model layers.
n_batch = 1 # Number of batches at each iteration.
learning_rate = 1e-4 # Set the learning rate.
is_loss_soft_dice = False # Use soft dice as loss function. When set to true, "weighted_loss" must be false.
weighted_loss = False # Use weighted categorical cross entropy as loss fucntion. When set to true, "is_loss_soft_dice" must be false.
f1_threshold_classification = None # This threshold is used to consider models with an evaluation f1 scores bigger than it. The selected model weights undergo a weights ensembling. And avreage ensembled model will be written to output.
classification_classes_name = None # Dictionary of classification classes names.
patches = False # Divides input image into smaller patches (input size of the model) when set to true. For the model to see the full image, like page extraction, set this to false.
augmentation = False # To apply any kind of augmentation, this parameter must be set to true.
if augmentation:
flip_aug = False # If true, different types of flipping will be applied to the image. Types of flips are defined with "flip_index" in config_params.json.
if flip_aug:
flip_index = None # Flip image for augmentation.
blur_aug = False # If true, different types of blurring will be applied to the image. Types of blur are defined with "blur_k" in config_params.json.
if blur_aug:
blur_k = None # Blur image for augmentation.
padding_white = False # If true, white padding will be applied to the image.
padding_black = False # If true, black padding will be applied to the image.
scaling = False # If true, scaling will be applied to the image. The amount of scaling is defined with "scales" in config_params.json.
shifting = False
degrading = False # If true, degrading will be applied to the image. The amount of degrading is defined with "degrade_scales" in config_params.json.
brightening = False # If true, brightening will be applied to the image. The amount of brightening is defined with "brightness" in config_params.json.
binarization = False # If true, Otsu thresholding will be applied to augment the input with binarized images.
adding_rgb_background = False
adding_rgb_foreground = False
add_red_textlines = False
channels_shuffling = False
dir_train = None # Directory of training dataset with subdirectories having the names "images" and "labels".
dir_eval = None # Directory of validation dataset with subdirectories having the names "images" and "labels".
dir_output = None # Directory where the output model will be saved.
pretraining = False # Set to true to load pretrained weights of ResNet50 encoder.
scaling_bluring = False # If true, a combination of scaling and blurring will be applied to the image.
scaling_binarization = False # If true, a combination of scaling and binarization will be applied to the image.
rotation = False # If true, a 90 degree rotation will be implemeneted.
rotation_not_90 = False # If true rotation based on provided angles with thetha will be implemeneted.
scaling_brightness = False # If true, a combination of scaling and brightening will be applied to the image.
scaling_flip = False # If true, a combination of scaling and flipping will be applied to the image.
thetha = None # Rotate image by these angles for augmentation.
shuffle_indexes = None
blur_k = None # Blur image for augmentation.
if scaling or scaling_brightness or scaling_bluring or scaling_binarization or scaling_flip:
scales = None # Scale patches for augmentation.
shifting = False
degrading = False # If true, degrading will be applied to the image. The amount of degrading is defined with "degrade_scales" in config_params.json.
if degrading:
degrade_scales = None # Degrade image for augmentation.
brightening = False # If true, brightening will be applied to the image. The amount of brightening is defined with "brightness" in config_params.json.
if brightening:
brightness = None # Brighten image for augmentation.
flip_index = None # Flip image for augmentation.
binarization = False # If true, Otsu thresholding will be applied to augment the input with binarized images.
if binarization:
dir_img_bin = None # Directory of training dataset subdirectory of binarized images
add_red_textlines = False
adding_rgb_background = False
if adding_rgb_background:
dir_rgb_backgrounds = None # Directory of texture images for synthetic background
adding_rgb_foreground = False
if adding_rgb_foreground:
dir_rgb_foregrounds = None # Directory of texture images for synthetic foreground
if adding_rgb_background or adding_rgb_foreground:
number_of_backgrounds_per_image = 1
channels_shuffling = False # Re-arrange color channels.
if channels_shuffling:
shuffle_indexes = None # Which channels to switch between.
rotation = False # If true, a 90 degree rotation will be implemeneted.
rotation_not_90 = False # If true rotation based on provided angles with thetha will be implemeneted.
if rotation_not_90:
thetha = None # Rotate image by these angles for augmentation.
dir_train = None # Directory of training dataset with subdirectories having the names "images" and "labels".
dir_eval = None # Directory of validation dataset with subdirectories having the names "images" and "labels".
dir_output = None # Directory where the augmented training data and the model checkpoints will be saved.
pretraining = False # Set to true to (down)load pretrained weights of ResNet50 encoder.
save_interval = None # frequency for writing model checkpoints (nonzero integer for number of batches, or zero for epoch)
continue_training = False # Set to true if you would like to continue training an already trained a model.
dir_of_start_model = '' # Directory containing pretrained encoder to continue training the model.
data_is_provided = False # Only set this to true when you have already provided the input data and the train and eval data are in "dir_output".
if backbone_type == "transformer":
transformer_patchsize_x = None # Patch size of vision transformer patches in x direction.
transformer_patchsize_y = None # Patch size of vision transformer patches in y direction.
transformer_num_patches_xy = None # Number of patches for vision transformer in x and y direction respectively.
@ -142,53 +168,52 @@ def config_params():
transformer_mlp_head_units = [128, 64] # Transformer Multilayer Perceptron (MLP) head units. Default value is [128, 64]
transformer_layers = 8 # transformer layers. Default value is 8.
transformer_num_heads = 4 # Transformer number of heads. Default value is 4.
transformer_cnn_first = True # We have two types of vision transformers. In one type, a CNN is applied first, followed by a transformer. In the other type, this order is reversed. If transformer_cnn_first is true, it means the CNN will be applied before the transformer. Default value is true.
index_start = 0 # Index of model to continue training from. E.g. if you trained for 3 epochs and last index is 2, to continue from model_1.h5, set "index_start" to 3 to start naming model with index 3.
dir_of_start_model = '' # Directory containing pretrained encoder to continue training the model.
is_loss_soft_dice = False # Use soft dice as loss function. When set to true, "weighted_loss" must be false.
weighted_loss = False # Use weighted categorical cross entropy as loss fucntion. When set to true, "is_loss_soft_dice" must be false.
data_is_provided = False # Only set this to true when you have already provided the input data and the train and eval data are in "dir_output".
task = "segmentation" # This parameter defines task of model which can be segmentation, enhancement or classification.
f1_threshold_classification = None # This threshold is used to consider models with an evaluation f1 scores bigger than it. The selected model weights undergo a weights ensembling. And avreage ensembled model will be written to output.
classification_classes_name = None # Dictionary of classification classes names.
backbone_type = None # As backbone we have 2 types of backbones. A vision transformer alongside a CNN and we call it "transformer" and only CNN called "nontransformer"
save_interval = None
dir_img_bin = None
number_of_backgrounds_per_image = 1
dir_rgb_backgrounds = None
dir_rgb_foregrounds = None
transformer_cnn_first = True # We have two types of vision transformers: either the CNN is applied first, followed by the transformer, or reversed.
@ex.automain
def run(_config, n_classes, n_epochs, input_height,
input_width, weight_decay, weighted_loss,
index_start, dir_of_start_model, is_loss_soft_dice,
n_batch, patches, augmentation, flip_aug,
blur_aug, padding_white, padding_black, scaling, shifting, degrading,channels_shuffling,
brightening, binarization, adding_rgb_background, adding_rgb_foreground, add_red_textlines, blur_k, scales, degrade_scales,shuffle_indexes,
brightness, dir_train, data_is_provided, scaling_bluring,
scaling_brightness, scaling_binarization, rotation, rotation_not_90,
thetha, scaling_flip, continue_training, transformer_projection_dim,
transformer_mlp_head_units, transformer_layers, transformer_num_heads, transformer_cnn_first,
transformer_patchsize_x, transformer_patchsize_y,
transformer_num_patches_xy, backbone_type, save_interval, flip_index, dir_eval, dir_output,
pretraining, learning_rate, task, f1_threshold_classification, classification_classes_name, dir_img_bin, number_of_backgrounds_per_image,dir_rgb_backgrounds, dir_rgb_foregrounds):
def run(_config,
_log,
task,
pretraining,
data_is_provided,
dir_train,
dir_eval,
dir_output,
n_classes,
n_epochs,
n_batch,
input_height,
input_width,
is_loss_soft_dice,
weighted_loss,
weight_decay,
learning_rate,
continue_training,
dir_of_start_model,
save_interval,
augmentation,
thetha,
backbone_type,
transformer_projection_dim,
transformer_mlp_head_units,
transformer_layers,
transformer_num_heads,
transformer_cnn_first,
transformer_patchsize_x,
transformer_patchsize_y,
transformer_num_patches_xy,
f1_threshold_classification,
classification_classes_name,
):
if pretraining and not os.path.isfile(RESNET50_WEIGHTS_PATH):
print("downloading RESNET50 pretrained weights to", RESNET50_WEIGHTS_PATH)
_log.info("downloading RESNET50 pretrained weights to %s", RESNET50_WEIGHTS_PATH)
download_file(RESNET50_WEIGHTS_URL, RESNET50_WEIGHTS_PATH)
if dir_rgb_backgrounds:
list_all_possible_background_images = os.listdir(dir_rgb_backgrounds)
else:
list_all_possible_background_images = None
if dir_rgb_foregrounds:
list_all_possible_foreground_rgbs = os.listdir(dir_rgb_foregrounds)
else:
list_all_possible_foreground_rgbs = None
# set the gpu configuration
configuration()
if task in ["segmentation", "enhancement", "binarization"]:
if data_is_provided:
dir_train_flowing = os.path.join(dir_output, 'train')
dir_eval_flowing = os.path.join(dir_output, 'eval')
@ -198,33 +223,16 @@ def run(_config, n_classes, n_epochs, input_height,
dir_flow_eval_imgs = os.path.join(dir_eval_flowing, 'images')
dir_flow_eval_labels = os.path.join(dir_eval_flowing, 'labels')
configuration()
else:
dir_img, dir_seg = get_dirs_or_files(dir_train)
dir_img_val, dir_seg_val = get_dirs_or_files(dir_eval)
# make first a directory in output for both training and evaluations in order to flow data from these directories.
dir_train_flowing = os.path.join(dir_output, 'train')
dir_eval_flowing = os.path.join(dir_output, 'eval')
dir_flow_train_imgs = os.path.join(dir_train_flowing, 'images/')
dir_flow_train_labels = os.path.join(dir_train_flowing, 'labels/')
dir_flow_eval_imgs = os.path.join(dir_eval_flowing, 'images/')
dir_flow_eval_labels = os.path.join(dir_eval_flowing, 'labels/')
if not data_is_provided:
# first create a directory in output for both training and evaluations
# in order to flow data from these directories.
if os.path.isdir(dir_train_flowing):
os.system('rm -rf ' + dir_train_flowing)
os.makedirs(dir_train_flowing)
else:
os.makedirs(dir_train_flowing)
if os.path.isdir(dir_eval_flowing):
os.system('rm -rf ' + dir_eval_flowing)
os.makedirs(dir_eval_flowing)
else:
os.makedirs(dir_eval_flowing)
os.mkdir(dir_flow_train_imgs)
os.mkdir(dir_flow_train_labels)